Examples of org.apache.crunch.types.Converter

org.apache.crunch.types.Converter
Converts the input key/value from a MapReduce task into the input to a {@link DoFn}, or takes the output of a {@code DoFn} and write it to theoutput key/values.

    this.fileNamingScheme = fileNamingScheme;
  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, outputFormatClass, outputPath, name);
  }

View Full Code Here

  }
  
  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {

View Full Code Here

    return "Text(" + path + ")";
  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, FormatBundle.forOutput(getOutputFormat(ptype)), outputPath, name);
  }

View Full Code Here

  }


  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {

View Full Code Here

    this.fileNamingScheme = fileNamingScheme;
  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, outputFormatClass, outputPath, name);
  }

View Full Code Here

        }
        for (Target t : targets) {
          Configuration conf = new Configuration(getConfiguration());
          getRuntimeContext().setConf(sparkContext.broadcast(WritableUtils.toByteArray(conf)));
          if (t instanceof MapReduceTarget) { //TODO: check this earlier
            Converter c = t.getConverter(ptype);
            JavaPairRDD<?, ?> outRDD;
            if (rdd instanceof JavaRDD) {
              outRDD = ((JavaRDD) rdd)
                  .map(new MapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            } else {
              outRDD = ((JavaPairRDD) rdd)
                  .map(new PairMapFunction(ptype.getOutputMapFn(), ctxt))
                  .map(new OutputConverterFunction(c));
            }
            try {
              Job job = new Job(conf);
              if (t instanceof PathTarget) {
                PathTarget pt = (PathTarget) t;
                pt.configureForMapReduce(job, ptype, pt.getPath(), null);
                Path tmpPath = pipeline.createTempPath();
                outRDD.saveAsNewAPIHadoopFile(
                    tmpPath.toString(),
                    c.getKeyClass(),
                    c.getValueClass(),
                    job.getOutputFormatClass(),
                    job.getConfiguration());
                pt.handleOutputs(job.getConfiguration(), tmpPath, -1);
              } else if (t instanceof MapReduceTarget) {
                MapReduceTarget mrt = (MapReduceTarget) t;

View Full Code Here

    return this;
  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = getConverter(ptype);
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    configureForMapReduce(job, keyClass, valueClass, formatBundle, outputPath, name);
  }

View Full Code Here

    return this;
  }


  @Override
  public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String name) {
    Converter converter = ptype.getConverter();
    Class keyClass = converter.getKeyClass();
    Class valueClass = converter.getValueClass();
    FormatBundle fb = FormatBundle.forOutput(getOutputFormat(ptype));
    for (Map.Entry<String, String> e : extraConf.entrySet()) {
      fb.set(e.getKey(), e.getValue());
    }
    configureForMapReduce(job, keyClass, valueClass, fb, outputPath, name);

View Full Code Here

    fn.configure(conf);
    for (DoNode child : children) {
      childRTNodes.add(child.toRTNode(false, conf, nodeContext));
    }


    Converter inputConverter = null;
    if (inputNode) {
      if (nodeContext == NodeContext.MAP) {
        inputConverter = source.getConverter();
      } else {
        inputConverter = ((PGroupedTableType<?, ?>) ptype).getGroupingConverter();

View Full Code Here

  }


  private void writeSequenceFileFromPCollection(final FileSystem fs, final Path path,
      final PCollection collection) throws IOException {
    final PType pType = collection.getPType();
    final Converter converter = pType.getConverter();
    final Class valueClass = converter.getValueClass();


    final SequenceFile.Writer writer = new SequenceFile.Writer(fs, fs.getConf(), path,
        NullWritable.class, valueClass);


    for (final Object o : collection.materialize()) {

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.crunch.types.Converter

com.twitter.elephantbird.crunch.LzoProtobufTarget

com.twitter.elephantbird.crunch.LzoThriftTarget

org.apache.crunch.impl.mem.MemPipeline

org.apache.crunch.impl.mr.plan.DoNode

org.apache.crunch.impl.spark.collect.InputCollection

org.apache.crunch.impl.spark.collect.InputTable

org.apache.crunch.impl.spark.SparkRuntime

org.apache.crunch.io.impl.FileTargetImpl

org.apache.crunch.io.text.TextFileTarget

org.kitesdk.data.crunch.DatasetTarget

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.