Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addInputPath()

59
60
61
62
63
64
65
66
67
68
69
  }

  public void merge(Path output, Path[] dbs, boolean normalize, boolean filter) throws Exception {
    JobConf job = LinkDb.createMergeJob(getConf(), output, normalize, filter);
    for (int i = 0; i < dbs.length; i++) {
      job.addInputPath(new Path(dbs[i], LinkDb.CURRENT_NAME));     
    }
    JobClient.runJob(job);
    FileSystem fs = FileSystem.get(getConf());
    fs.mkdirs(output);
    fs.rename(job.getOutputPath(), new Path(output, LinkDb.CURRENT_NAME));
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
    }
    theJob.setJobName("ValueAggregatorJob: " + jobName);

    String[] inputDirsSpecs = inputDir.split(",");
    for (int i = 0; i < inputDirsSpecs.length; i++) {
      theJob.addInputPath(new Path(inputDirsSpecs[i]));
    }

    theJob.setInputFormat(theInputFormat);
   
    theJob.setMapperClass(ValueAggregatorMapper.class);
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
    theJob.setJobName("DataMoveJob");

    theJob.setInputPath((Path) indirs.get(0));
    if (indirs.size() > 1) {
      for (int i = 1; i < indirs.size(); i++) {
        theJob.addInputPath((Path) indirs.get(i));
      }
    }
    theJob.setMapperClass(DataCopy.class);
    theJob.setOutputPath(outdir);
    theJob.setOutputKeyClass(Text.class);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
   * @return
   */
  private JobConf setupJob(){
    JobConf job = new JobConf() ;
   
    job.addInputPath(new Path(context + "/input"));
   
    job.setInputFormat(TextInputFormat.class);
    job.setOutputFormat(TextOutputFormat.class);
   
    job.setInputKeyClass(LongWritable.class);
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
        theJob.setJobName("DataMoveJob");

        theJob.setInputPath((Path) indirs.get(0));
        if (indirs.size() > 1) {
            for (int i = 1; i < indirs.size(); i++) {
                theJob.addInputPath((Path) indirs.get(i));
            }
        }
        theJob.setMapperClass(DataCopy.class);
        theJob.setOutputPath(outdir);
        theJob.setOutputKeyClass(Text.class);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
   * @return
   */
  private JobConf setupJob(){
    JobConf job = new JobConf() ;
   
    job.addInputPath(new Path(context + "/input"));
   
    job.setInputFormat(TextInputFormat.class);
    job.setOutputFormat(TextOutputFormat.class);
   
    job.setInputKeyClass(LongWritable.class);
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
    FileSystem fs = FileSystem.get(theJob);
    theJob.setJobName("ValueAggregatorJob");

    String[] inputDirsSpecs = inputDir.split(",");
    for (int i = 0; i < inputDirsSpecs.length; i++) {
      theJob.addInputPath(new Path(inputDirsSpecs[i]));
    }

    theJob.setInputFormat(theInputFormat);
   
    theJob.setMapperClass(ValueAggregatorMapper.class);
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
      job.setReducerClass(SleepJob.class);
      job.setOutputFormat(NullOutputFormat.class);
      job.setInputFormat(SequenceFileInputFormat.class);
      job.setSpeculativeExecution(false);
      job.setJobName("Sleep job");
      job.addInputPath(tempPath);
      job.setLong("sleep.job.map.sleep.time", mapSleepTime);
      job.setLong("sleep.job.reduce.sleep.time", reduceSleepTime);
      job.setLong("sleep.job.map.sleep.count", mapSleepCount);
      job.setLong("sleep.job.reduce.sleep.count", reduceSleepCount);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
   * @return
   */
  private JobConf setupJob(){
    JobConf job = new JobConf() ;
   
    job.addInputPath(new Path(context + "/input"));
   
    job.setInputFormat(TextInputFormat.class);
    job.setOutputFormat(TextOutputFormat.class);
   
    job.setInputKeyClass(LongWritable.class);
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
        theJob.setJobName("DataMoveJob");

        theJob.setInputPath((Path) indirs.get(0));
        if (indirs.size() > 1) {
            for (int i = 1; i < indirs.size(); i++) {
                theJob.addInputPath((Path) indirs.get(i));
            }
        }
        theJob.setMapperClass(DataCopy.class);
        theJob.setOutputPath(outdir);
        theJob.setOutputKeyClass(Text.class);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.