Package azkaban.common.utils

Examples of azkaban.common.utils.Props


            }

        }

        public void configure(JobConf job) {
            Props props = HadoopUtils.getPropsFromJob(job);

            try {
                _position = 0;
                _conf = job;

                _taskId = job.get("mapred.task.id");

                storeName = props.get("voldemort.store.name");
                taskIndexFileName = new Path(FileOutputFormat.getOutputPath(_conf),
                                             _conf.get("voldemort.index.filename") + "_" + _taskId);
                taskValueFileName = new Path(FileOutputFormat.getOutputPath(_conf),
                                             _conf.get("voldemort.data.filename") + "_" + _taskId);
View Full Code Here


                                           e);
            }
        }

        public void configure(JobConf conf) {
            Props props = HadoopUtils.getPropsFromJob(conf);

            // get the voldemort cluster.xml and store.xml files.
            try {
                _cluster = HadoopUtils.readCluster(props.get("voldemort.cluster.file"), conf);
                _storeDef = HadoopUtils.readStoreDef(props.get("voldemort.store.file"),
                                                     props.get("voldemort.store.name"),
                                                     conf);

                _checkPercent = conf.getInt("input.data.check.percent", 0);
                _routingStrategy = new ConsistentRoutingStrategy(_cluster,
                                                                 _storeDef.getReplicationFactor());
View Full Code Here

        config.addResource(new Path(new File(hadoopConfDir, "hadoop-default.xml").getAbsolutePath()));
        config.addResource(new Path(new File(hadoopConfDir, "hadoop-site.xml").getAbsolutePath()));

        // copy to props
        Props props = new Props(parent);
        for(Entry<String, String> entry: config)
            props.put(entry.getKey(), config.get(entry.getKey()));

        return props;
    }
View Full Code Here

            throw new UndefinedPropertyException("The required property azkaban.props was not found in the Configuration.");
        try {
            ByteArrayInputStream input = new ByteArrayInputStream(propsString.getBytes("UTF-8"));
            Properties properties = new Properties();
            properties.load(input);
            return new Props(null, properties);
        } catch(IOException e) {
            throw new RuntimeException("This is not possible!", e);
        }
    }
View Full Code Here

        if(fs.exists(path)) {
            InputStream input = fs.open(path);
            try {
                // wrap it up in another layer so that the user can override
                // properties
                Props p = new Props(null, input);
                return new Props(p);
            } finally {
                input.close();
            }
        } else {
            return new Props();
        }
    }
View Full Code Here

    }

    @Override
    public void configure(JobConf conf) {
        super.configure(conf);
        Props props = HadoopUtils.getPropsFromJob(conf);

        _keySelection = props.getString("key.selection", null);
        _valSelection = props.getString("value.selection", null);
        _inputKeySerializer = getSchemaFromJob(conf, "mapper.input.key.schema");
        _inputValueSerializer = getSchemaFromJob(conf, "mapper.input.value.schema");
        String _keyTransClass = props.getString("key.transformation.class", null);
        String _valueTransClass = props.getString("value.transformation.class", null);

        if(_keyTransClass != null)
            _keyTrans = (StoreBuilderTransformation) Utils.callConstructor(_keyTransClass);
        if(_valueTransClass != null)
            _valTrans = (StoreBuilderTransformation) Utils.callConstructor(_valueTransClass);
View Full Code Here

        valueCompressor = new CompressionStrategyFactory().get(valueSerializerDefinition.getCompression());

        routingStrategy = new ConsistentRoutingStrategy(getCluster(),
                                                        getStoreDef().getReplicationFactor());

        Props props = HadoopUtils.getPropsFromJob(conf);

    }
View Full Code Here

        this._jobDirs = jobDirs;
        this._baseClassLoader = classLoader;
    }

    public Job loadJob(String jobName, boolean ignoreDep) {
        return loadJob(jobName, new Props(), ignoreDep);
    }
View Full Code Here

                } catch (Exception e) {
                    logger.info("Ignoring unknown directory found in logs:" + execDir.getAbsolutePath(), e);
                    continue;
                }
                if(runProps.canRead()) {
                    Props props = new Props(null, runProps.getAbsolutePath());
                    start = new DateTime(props.getLong("start"));
                    end = new DateTime(props.getLong("end"));
                    succeeded = props.getBoolean("succeeded");
                }
                String logFile = jobName + File.separator + execDir.getName() + File.separator
                                 + jobName + "." + execDir.getName() + ".log";
                execs.add(new JobExecution(jobName,
                                           start == null ? dirDate : start,
View Full Code Here

                                              propsOverrides,
                                              pathOverrides,
                                              _baseClassLoader);

        if(pathOverrides != null) {
            Props dirProps = loadLocalNonJobProps(jobDir, _defaultProps);
            for(Map.Entry<File, File> override: pathOverrides.entrySet()) {
                loadJobDescriptorsWithoutDependencies(m,
                                                      jobDir,
                                                      override.getValue(),
                                                      dirProps,
View Full Code Here

TOP

Related Classes of azkaban.common.utils.Props

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.