Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addResource()

64
65
66
67
68
69
70
71
72
73
74
    }

    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(PageRankJob.class);
        conf.setJobName("PageRank");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }
View Full Code Here
65
66
67
68
69
70
71
72
73
74
75
    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(PageRankJob.class);
        conf.setJobName("PageRank");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }

    public static String scaleFloat(float f) {// 保留6位小数
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
    public static JobConf config() {// Hadoop集群的远程配置信息
        JobConf conf = new JobConf(PageRankJob.class);
        conf.setJobName("PageRank");
        conf.addResource("classpath:/hadoop/core-site.xml");
        conf.addResource("classpath:/hadoop/hdfs-site.xml");
        conf.addResource("classpath:/hadoop/mapred-site.xml");
        return conf;
    }

    public static String scaleFloat(float f) {// 保留6位小数
        DecimalFormat df = new DecimalFormat("##0.000000");
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
        // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation.
        // Now add the settings from "properties" object to override any existing properties
        // All of the above is accomplished in the method call below
          
        JobConf jobConf = new JobConf();
        jobConf.addResource("pig-cluster-hadoop-site.xml");
           
        //the method below alters the properties object by overriding the
        //hadoop properties with the values from properties and recomputing
        //the properties
        recomputeProperties(jobConf, properties);
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
                  "If you plan to use local mode, please put -x local option in command line",
                  4010);
            }

            jc = new JobConf();
            jc.addResource("pig-cluster-hadoop-site.xml");
           
            // Trick to invoke static initializer of DistributedFileSystem to add hdfs-default.xml
            // into configuration
            new DistributedFileSystem();
           
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
            //the properties
            recomputeProperties(jc, properties);
        } else {
            // If we are running in local mode we dont read the hadoop conf file
            jc = new JobConf(false);
            jc.addResource("core-default.xml");
            jc.addResource("mapred-default.xml");
            recomputeProperties(jc, properties);
           
            properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
            properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
            recomputeProperties(jc, properties);
        } else {
            // If we are running in local mode we dont read the hadoop conf file
            jc = new JobConf(false);
            jc.addResource("core-default.xml");
            jc.addResource("mapred-default.xml");
            recomputeProperties(jc, properties);
           
            properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
            properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
        }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
          "test.build.data", "/tmp"));
   
      String[] sortArgs = { DEMUX_INPUT_PATH.toString(), DEMUX_OUTPUT_PATH.toString() };
      //      JobConf job = mr.createJobConf();
      JobConf job = new JobConf(new ChukwaConfiguration(), Demux.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-Demux_" + day.format(new Date()));
      job.setInputFormat(SequenceFileInputFormat.class);
      job.setMapperClass(Demux.MapClass.class);
      job.setPartitionerClass(ChukwaRecordPartitioner.class);
      job.setReducerClass(Demux.ReduceClass.class);
View Full Code Here
321
322
323
324
325
326
327
328
329
330
331
    log.info("Testing FSMBuilder (Job History only)");
    System.out.println("In JobHistory020");
    // Run FSMBuilder on Demux output
    try {
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(JobHistoryTaskDataMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
      job.setMapOutputValueClass(FSMIntermedEntry.class);
View Full Code Here
450
451
452
453
454
455
456
457
458
459
460
    System.out.println("In ClientTrace020");
    // Run FSMBuilder on Demux output
    try {
      // Process TaskTracker shuffle clienttrace entries first
      JobConf job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(TaskTrackerClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
      job.setMapOutputValueClass(FSMIntermedEntry.class);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.