Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Cluster


   * @param jobTrackAddr the job tracker to connect to.
   * @param conf configuration.
   */
  public JobClient(InetSocketAddress jobTrackAddr,
                   Configuration conf) throws IOException {
    cluster = new Cluster(jobTrackAddr, conf);
    clientUgi = UserGroupInformation.getCurrentUser();
  }
View Full Code Here


   * @param conf the job configuration.
   * @throws IOException
   */
  public void init(JobConf conf) throws IOException {
    setConf(conf);
    cluster = new Cluster(conf);
    clientUgi = UserGroupInformation.getCurrentUser();
  }
View Full Code Here

   * @param jobTrackAddr the job tracker to connect to.
   * @param conf configuration.
   */
  public JobClient(InetSocketAddress jobTrackAddr,
                   Configuration conf) throws IOException {
    cluster = new Cluster(jobTrackAddr, conf);
    clientUgi = UserGroupInformation.getCurrentUser();
  }
View Full Code Here

    historyService.init(conf);
    historyService.start(conf);

    LOG.info("services started");

    Cluster cluster = new Cluster(conf);
    org.apache.hadoop.mapreduce.JobID jobID =
      new org.apache.hadoop.mapred.JobID("201103121733", 1);
    org.apache.hadoop.mapreduce.Counters counters =
        cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);

    LOG.info("Sleeping for 5 seconds before stop for" +
    " the client socket to not get EOF immediately..");
    Thread.sleep(5000);

    //bring down the AM service
    amService.stop();

    LOG.info("Sleeping for 5 seconds after stop for" +
        " the server to exit cleanly..");
    Thread.sleep(5000);

    amRestarting = true;

    // Same client
    //results are returned from fake (not started job)
    counters = cluster.getJob(jobID).getCounters();
    Assert.assertEquals(0, counters.countCounters());
    Job job = cluster.getJob(jobID);
    org.apache.hadoop.mapreduce.TaskID taskId =
      new org.apache.hadoop.mapreduce.TaskID(jobID, TaskType.MAP, 0);
    TaskAttemptID tId = new TaskAttemptID(taskId, 0);

    //invoke all methods to check that no exception is thrown
    job.killJob();
    job.killTask(tId);
    job.failTask(tId);
    job.getTaskCompletionEvents(0, 100);
    job.getStatus();
    job.getTaskDiagnostics(tId);
    job.getTaskReports(TaskType.MAP);
    job.getTrackingURL();

    amRestarting = false;
    amService = new AMService();
    amService.init(conf);
    amService.start(conf);
    amContact = false; //reset

    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);

    // Stop the AM. It is not even restarting. So it should be treated as
    // completed.
    amService.stop();

    // Same client
    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(hsContact);

    rmService.stop();
    historyService.stop();
View Full Code Here

      displayUsage(cmd);
      return exitCode;
    }

    // initialize cluster
    cluster = new Cluster(getConf());
       
    // Submit the request
    try {
      if (submitJobFile != null) {
        Job job = Job.getInstance(new JobConf(submitJobFile));
View Full Code Here

    @Override
    public synchronized void init(Job job) throws IOException {
      final Configuration conf = job.getConfiguration();
      if (cluster == null) {
        String jobTrackerStr = conf.get("mapreduce.jobtracker.address", "localhost:8012");
        cluster = new Cluster(NetUtils.createSocketAddr(jobTrackerStr), conf);

      }
      chooseMachine(conf).init(job);
    }
View Full Code Here

      // Framework history log file location
      Path logFile = new Path(doneDir, logFileName);
      FileSystem fileSys = logFile.getFileSystem(conf);

      Cluster cluster = new Cluster(conf);
      assertEquals("Client returned wrong history url", logFile.toString(),
          cluster.getJobHistoryUrl(id));
  
      // Check if the history file exists
      assertTrue("History file does not exist", fileSys.exists(logFile));

      // check if the corresponding conf file exists
View Full Code Here

        mr = new MiniMRCluster(0, "file:///", 1, null, null, new JobConf(conf));
        jobTracker = mr.getJobTrackerRunner().getJobTracker();
        for (String tracker : trackers) {
          FakeObjectUtilities.establishFirstContact(jobTracker, tracker);
        }
        cluster = new Cluster(mr.createJobConf());
      }

      protected void tearDown() throws Exception {
        cluster.close();
        mr.shutdown();
View Full Code Here

    if (shouldComplete) {
      job.waitForCompletion(false);
    } else {
      job.submit();
      // miniMRCluster.getJobTrackerRunner().getJobTracker().jobsToComplete()[]
      Cluster cluster = new Cluster(miniMRCluster.createJobConf());
      JobStatus[] status = miniMRCluster.getJobTrackerRunner().getJobTracker()
          .jobsToComplete();
      JobID id = status[status.length -1].getJobID();
      Job newJob = cluster.getJob(id);
      cluster.close();
      return newJob;
    }
    return job;
  }
View Full Code Here

  @Test
  public void testCleanup() throws Exception {

      Configuration conf = getConf();

      Path stagingDir = JobSubmissionFiles.getStagingDir(new Cluster(conf),
          conf);
      stagingDir.getFileSystem(conf).mkdirs(stagingDir);
      Path soure = createFile("tmp.txt");
      Path target = createFile("target.txt");
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.Cluster

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.