Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

153
154
155
156
157
158
159
160
161
162
163
      }
      assertTrue(fileList.length == (numReducers - 1));

      // Test 2. 0 Reducers, maps directly write to the output files
      Path output2 = new Path("/testlazy/output2");
      runTestLazyOutput(mr.createJobConf(), output2, 0, true);

      fileList =
        FileUtil.stat2Paths(fileSys.listStatus(output2,
            new Utils.OutputFileUtils.OutputFilesFilter()));
      for(int i=0; i < fileList.length; ++i) {
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
      assertTrue(fileList.length == numMappers - 1);

      // Test 3. 0 Reducers, but flag is turned off
      Path output3 = new Path("/testlazy/output3");
      runTestLazyOutput(mr.createJobConf(), output3, 0, false);

      fileList =
        FileUtil.stat2Paths(fileSys.listStatus(output3,
            new Utils.OutputFileUtils.OutputFilesFilter()));
      for(int i=0; i < fileList.length; ++i) {
View Full Code Here
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100",
                        "-copybychunk",
                        "-log",
View Full Code Here
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100",
                        "-log",
                        namenode+"/logs",
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
        .toString(), 1);
    String[] archiveArgs = {INPUT_DIR.toString(),
        fileSys.getUri().toString() +OUTPUT_DIR.toString() };
   
   
    JobConf jc = mr.createJobConf(new JobConf(conf));
    assertEquals("true", jc.get("archive.groupByClusterName"));
    assertEquals(1, jc.getInt("io.sort.mb", 5));
   
    int returnVal = ToolRunner.run(jc,  new XtrExtract(), archiveArgs);
    assertEquals(0, returnVal);
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
      System.out.println("wrote "
          + fileSys.getFileStatus(DEMUX_INPUT_PATH).getLen()
          + " bytes of temp test data");
      long ts_start = System.currentTimeMillis();
      runDemux(mr.createJobConf(), DEMUX_INPUT_PATH, DEMUX_OUTPUT_PATH);

      long time = (System.currentTimeMillis() - ts_start);
      long bytes = fileSys.getContentSummary(DEMUX_OUTPUT_PATH).getLength();
      System.out.println("result was " + bytes + " bytes long");
      System.out.println("processing took " + time + " milliseconds");
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
        option = "-usefastcopy";
      } else if (type.equals(DistcpType.COPYBYCHUNK)) {
        option = "-copybychunk";
      }
     
      Configuration job = mr.createJobConf();
      List<String> args = new ArrayList<String>();
      args.add("-m");
      args.add("100");
      args.add("-skipunderconstruction");
      if (!option.equals("")) {
View Full Code Here
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100", "-r", "3",
                        "-copybychunk",
                        "-log",
View Full Code Here
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
      FileSystem fs = dfs.getFileSystem();
      namenode = fs.getUri().toString();
      mr = new MiniMRCluster(3, namenode, 1);
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
     
      Configuration job = mr.createJobConf();
      for (int i = 0; i < tosValue.length; i++) {
        ToolRunner.run(new DistCp(job),
            new String[] {"-m", "100",
                          "-tos", String.valueOf(tosValue[i]),
                          "-log",
View Full Code Here
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100",
                        "-log",
                        namenode+"/logs",
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.