Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addResource()

112
113
114
115
116
117
118
119
120
121
122
    // Loads S3 properties from core-site.xml including aws keys that are needed
    // for both local and non-local mode.
    public JobConf getS3Conf() throws ExecException {
        JobConf jc = new JobConf();
        jc.addResource(CORE_SITE);
        Iterator<Entry<String, String>> i = jc.iterator();
        while (i.hasNext()) {
            Entry<String, String> e = i.next();
            String key = e.getKey();
            String value = e.getValue();
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
    }

    public JobConf getLocalConf() {
        JobConf jc = new JobConf(false);

        jc.addResource(CORE_DEFAULT_SITE);
        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);

        return jc;
    }
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
    public JobConf getLocalConf() {
        JobConf jc = new JobConf(false);

        jc.addResource(CORE_DEFAULT_SITE);
        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);

        return jc;
    }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    public JobConf getLocalConf() {
        JobConf jc = new JobConf(false);

        jc.addResource(CORE_DEFAULT_SITE);
        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);

        return jc;
    }

    public JobConf getExecConf(Properties properties) throws ExecException {
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
                                + " If you plan to use local mode, please put -x local option in command line",
                                4010);
            }
            jc = new JobConf();
        }
        jc.addResource("pig-cluster-hadoop-site.xml");
        jc.addResource(YARN_SITE);
        return jc;
    }

    @SuppressWarnings("resource")
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
                                4010);
            }
            jc = new JobConf();
        }
        jc.addResource("pig-cluster-hadoop-site.xml");
        jc.addResource(YARN_SITE);
        return jc;
    }

    @SuppressWarnings("resource")
    private void init(Properties properties) throws ExecException {
View Full Code Here
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
              Integer.parseInt(nodeHttpPortString), appSubmitTime,
              Integer.parseInt(maxAppAttempts));
      ShutdownHookManager.get().addShutdownHook(
        new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
      JobConf conf = new JobConf(new YarnConfiguration());
      conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
     
      // Explicitly disabling SSL for map reduce task as we can't allow MR users
      // to gain access to keystore file for opening SSL listener. We can trust
      // RM/NM to issue SSL certificates but definitely not MR-AM as it is
      // running in user-land.
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
            // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation.
            // Now add the settings from "properties" object to override any existing properties
            // All of the above is accomplished in the method call below
          
            JobConf jobConf = new JobConf();
            jobConf.addResource("pig-cluster-hadoop-site.xml");
           
            //the method below alters the properties object by overriding the
            //hadoop properties with the values from properties and recomputing
            //the properties
            recomputeProperties(jobConf, properties);
View Full Code Here
387
388
389
390
391
392
393
394
395
396
397
            String hadoopConf = hodConfDir + "/hadoop-site.xml";

            log.info ("Hadoop configuration file: " + hadoopConf);

            JobConf jobConf = new JobConf(hadoopConf);
            jobConf.addResource("pig-cluster-hadoop-site.xml");

            //the method below alters the properties object by overriding the
            //hod properties with the values from properties and recomputing
            //the properties
            recomputeProperties(jobConf, properties);
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    }

    public JobConf getLocalConf(Properties properties) {
        JobConf jc = new JobConf(false);

        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);
        jc.addResource(CORE_DEFAULT_SITE);

        return jc;
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.