Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.MiniMRCluster.createJobConf()

571
572
573
574
575
576
577
578
579
580
581
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      JobConf job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100",
                        "-log",
                        namenode+"/logs",
View Full Code Here
687
688
689
690
691
692
693
694
695
696
697
    RewindableInputStream ris = null;
    ArrayList<String> seenEvents = new ArrayList<String>(10);
    RunningJob rJob = null;
   
    try {
      JobConf jobConf = mrCluster.createJobConf();
      jobConf.setQueueName(queueName);
      // construct a job with 1 map and 1 reduce task.
      rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1,
                                  1);
      rJob.waitForCompletion();
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  public void testWithLocal() throws Exception {
    MiniMRCluster mr = null;
    try {
      mr = new MiniMRCluster(2, "file:///", 3);
      Configuration conf = mr.createJobConf();
      runWordCount(conf);
      runMultiFileWordCount(conf);
    } finally {
      if (mr != null) { mr.shutdown(); }
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
    conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
    try {
      mr = new MiniMRCluster(1, "file:///", 3, null , null, conf);

      Path inFile = new Path(INPUT_FILE);
      fs = inFile.getFileSystem(mr.createJobConf());
      clean(fs);
      makeInput(fs);
     
      StreamJob job = new StreamJob();
      int failed = job.run(genArgs(mr.getJobTrackerPort()));
View Full Code Here
749
750
751
752
753
754
755
756
757
758
759
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }
      Configuration job = mr.createJobConf();
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(new DistCp(job),
          new String[] {"-m", "100",
                        "-log",
                        namenode+"/logs",
View Full Code Here
74
75
76
77
78
79
80
81
82
83
84
      String namenode = fileSys.getUri().toString();

      mr  = new MiniMRCluster(1, namenode, 3);

      List<String> args = new ArrayList<String>();
      for (Map.Entry<String, String> entry : mr.createJobConf()) {
        args.add("-jobconf");
        args.add(entry.getKey() + "=" + entry.getValue());
      }

      String argv[] = new String[] {
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
      FileSystem fileSys = dfs.getFileSystem();
      String namenode = fileSys.getUri().toString();
      mr  = new MiniMRCluster(1, namenode, 3);

      List<String> args = new ArrayList<String>();
      for (Map.Entry<String, String> entry : mr.createJobConf()) {
        args.add("-jobconf");
        args.add(entry.getKey() + "=" + entry.getValue());
      }

      // During tests, the default Configuration will use a local mapred
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
        } else {
            // HBase 0.95.x returns a MR cluster, and we have to manually
            // copy the job tracker address into our configuration
            MiniMRCluster mrCluster = (MiniMRCluster)startMrClusterMethod.invoke(hbaseTestUtil);
            Configuration conf = hbaseTestUtil.getConfiguration();
            conf.set("mapred.job.tracker", mrCluster.createJobConf().get("mapred.job.tracker"));
        }
    }
   
    public static File substituteZkHost(File file, String zkConnectString) throws IOException {
      String str = Files.toString(file, Charsets.UTF_8);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
      // make cleanup inline sothat validation of existence of these directories
      // can be done
      mr.setInlineCleanupThreads();
     
      // Run examples
      TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf));
      TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf));
    } finally {
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
View Full Code Here
63
64
65
66
67
68
69
70
71
72
73
      // can be done
      mr.setInlineCleanupThreads();
     
      // Run examples
      TestMiniMRWithDFS.runPI(mr, mr.createJobConf(mrConf));
      TestMiniMRWithDFS.runWordCount(mr, mr.createJobConf(mrConf));
    } finally {
      if (dfs != null) { dfs.shutdown(); }
      if (mr != null) { mr.shutdown();
      }
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.