Package org.apache.hadoop.mapreduce

Examples of org.apache.hadoop.mapreduce.Job.submit()

289
290
291
292
293
294
295
296
297
298
299
    BasicTableOutputFormat.setOutputPath(job, outPath);
    BasicTableOutputFormat.setSchema(job, "fileName, wordPos, lineNo");

    // set map-only job.
    job.setNumReduceTasks(0);
    job.submit();
    job.waitForCompletion(true);
    BasicTableOutputFormat.close( job );
  }

  /**
 
View Full Code Here
568
569
570
571
572
573
574
575
576
577
578
    job.setCombinerClass(InvertedIndexGen.CombinerClass.class);
    BasicTableOutputFormat.setOutputPath(job, invIndexTablePath);
    BasicTableOutputFormat.setSchema(job, "count, index");
    job.setNumReduceTasks(options.numReducer);

    job.submit();
    job.waitForCompletion(true);
    BasicTableOutputFormat.close( job );
  }

  void reduce(Summary sum, Summary delta) {
View Full Code Here
907
908
909
910
911
912
913
914
915
916
917
    job.setCombinerClass(FreqWords.CombinerClass.class);
    BasicTableOutputFormat.setOutputPath(job, freqWordTablePath);
    BasicTableOutputFormat.setSchema(job, "count");
    job.setNumReduceTasks(1);

    job.submit();
    job.waitForCompletion(true);
    BasicTableOutputFormat.close( job );
  }

  void printFreqWords() throws IOException, ParseException {
View Full Code Here
311
312
313
314
315
316
317
318
319
320
321
    createShardDirectories(outDir, 1);

    BlurOutputFormat.setupJob(job, tableDescriptor);
    BlurOutputFormat.setReducerMultiplier(job, 2);
    job.setNumReduceTasks(4);
    job.submit();

  }
 
  // @TODO this test to fail sometimes due to issues in the MR MiniCluster
//  @Test
View Full Code Here
345
346
347
348
349
350
351
352
353
354
355
    createShardDirectories(outDir, 2);

    BlurOutputFormat.setupJob(job, tableDescriptor);
    BlurOutputFormat.setIndexLocally(job, false);

    job.submit();
    boolean killCalled = false;
    while (!job.isComplete()) {
      Thread.sleep(1000);
      System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100,
          job.reduceProgress() * 100);
 
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
    job.addFileToClassPath(second);
    job.addArchiveToClassPath(third);
    job.addCacheArchive(fourth.toUri());
    job.setMaxMapAttempts(1); // speed up failures

    job.submit();
    assertTrue(job.waitForCompletion(false));
  }

  /** Tests using the local job runner. */
  public void testLocalJobRunner() throws Exception {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(jobConf);
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.submit();
    boolean completed = job.waitForCompletion(true);
    Assert.assertTrue("Job should be completed", completed);
    Assert.assertEquals("Job should be finished successfully",
                    JobStatus.State.SUCCEEDED, job.getJobState());
  }
View Full Code Here
157
158
159
160
161
162
163
164
165
166
167
        job = createJob();
      }
      createInputFileListing(job);

      job.submit();
      submitted = true;
    } finally {
      if (!submitted) {
        cleanup();
      }
View Full Code Here
252
253
254
255
256
257
258
259
260
261
262
       
    // Submit the request
    try {
      if (submitJobFile != null) {
        Job job = Job.getInstance(new JobConf(submitJobFile));
        job.submit();
        System.out.println("Created job " + job.getJobID());
        exitCode = 0;
      } else if (getStatus) {
        Job job = cluster.getJob(JobID.forName(jobid));
        if (job == null) {
View Full Code Here
612
613
614
615
616
617
618
619
620
621
622
      Job job = clientUgi.doAs(new PrivilegedExceptionAction<Job> () {
        @Override
        public Job run() throws IOException, ClassNotFoundException,
          InterruptedException {
          Job job = Job.getInstance(conf);
          job.submit();
          return job;
        }
      });
      // update our Cluster instance with the one created by Job for submission
      // (we can't pass our Cluster instance to Job, since Job wraps the config
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.