Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

392
393
394
395
396
397
398
399
400
401
402
    ArrayList<String> seenEvents = new ArrayList<String>(10);
    RunningJob rJob = null;
   
    try {
      // construct a job with 1 map and 1 reduce task.
      rJob = UtilsForTests.runJob(mrCluster.createJobConf(), inDir, outDir, 1,
                                  1);
      rJob.waitForCompletion();
      assertTrue("Job failed", rJob.isSuccessful());
     
      JobID id = rJob.getID();
View Full Code Here
167
168
169
170
171
172
173
174
175
176
177
      }
      System.out.println("args=" + Arrays.asList(args).toString().replace(",", ",\n  "));
      System.out.println("newstatus=" + Arrays.asList(newstatus).toString().replace(",", ",\n  "));

      //run DistCh
      new DistCh(mr.createJobConf()).run(args);
      runLsr(shell, tree.root, 0);

      //check results
      for(int i = 0; i < NUN_SUBS; i++) {
        Path sub = new Path(tree.root + "/sub" + i);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
  public void testWithLocal() throws Exception {
    MiniMRCluster mr = null;
    try {
      mr = new MiniMRCluster(2, "file:///", 3);
      Configuration conf = mr.createJobConf();
      runWordCount(conf);
    } finally {
      if (mr != null) { mr.shutdown(); }
    }
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
      createInput(fileSys, numMappers);
      Path output1 = new Path("/testlazy/output1");

      // Test 1.
      runTestLazyOutput(mr.createJobConf(), output1,
          numReducers, true);

      Path[] fileList =
        FileUtil.stat2Paths(fileSys.listStatus(output1,
            new Utils.OutputFileUtils.OutputFilesFilter()));
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
      }
      assertTrue(fileList.length == (numReducers - 1));

      // Test 2. 0 Reducers, maps directly write to the output files
      Path output2 = new Path("/testlazy/output2");
      runTestLazyOutput(mr.createJobConf(), output2, 0, true);

      fileList =
        FileUtil.stat2Paths(fileSys.listStatus(output2,
            new Utils.OutputFileUtils.OutputFilesFilter()));
      for(int i=0; i < fileList.length; ++i) {
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
      assertTrue(fileList.length == numMappers - 1);

      // Test 3. 0 Reducers, but flag is turned off
      Path output3 = new Path("/testlazy/output3");
      runTestLazyOutput(mr.createJobConf(), output3, 0, false);

      fileList =
        FileUtil.stat2Paths(fileSys.listStatus(output3,
            new Utils.OutputFileUtils.OutputFilesFilter()));
      for(int i=0; i < fileList.length; ++i) {
View Full Code Here
687
688
689
690
691
692
693
694
695
696
697
    RewindableInputStream ris = null;
    ArrayList<String> seenEvents = new ArrayList<String>(10);
    RunningJob rJob = null;
   
    try {
      JobConf jobConf = mrCluster.createJobConf();
      jobConf.setQueueName(queueName);
      // construct a job with 1 map and 1 reduce task.
      rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1,
                                  1);
      rJob.waitForCompletion();
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    FileSystem fs = null;
    try {
      mr = new MiniMRCluster(1, "file:///", 3);

      Path inFile = new Path(INPUT_FILE);
      fs = inFile.getFileSystem(mr.createJobConf());
      clean(fs);
      makeInput(fs);
     
      StreamJob job = new StreamJob();
      int failed = job.run(genArgs(mr.getJobTrackerPort()));
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
      FileSystem fileSys = dfs.getFileSystem();
      String namenode = fileSys.getUri().toString();
      mr  = new MiniMRCluster(1, namenode, 3);

      List<String> args = new ArrayList<String>();
      for (Map.Entry<String, String> entry : mr.createJobConf()) {
        args.add("-jobconf");
        args.add(entry.getKey() + "=" + entry.getValue());
      }

      // During tests, the default Configuration will use a local mapred
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      String namenode = fileSys.getUri().toString();

      mr  = new MiniMRCluster(1, namenode, 3);

      List<String> args = new ArrayList<String>();
      for (Map.Entry<String, String> entry : mr.createJobConf()) {
        args.add("-jobconf");
        args.add(entry.getKey() + "=" + entry.getValue());
      }

      String argv[] = new String[] {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.