Package org.apache.crunch.io

Examples of org.apache.crunch.io.FormatBundle.configure()

46
47
48
49
50
51
52
53
54
55
56
    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, true)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(jobCopy);
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    AvroType<Person> avroSpecificType = Avros.records(Person.class);
    AvroFileSource<Person> personFileSource = new AvroFileSource<Person>(new Path(tempFile.getAbsolutePath()),
        avroSpecificType);

    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());

    assertFalse(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true));
    assertEquals(Person.SCHEMA$.toString(), job.getConfiguration().get(AvroJob.INPUT_SCHEMA));
  }
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
    AvroType<Record> avroGenericType = Avros.generics(Person.SCHEMA$);
    AvroFileSource<Record> personFileSource = new AvroFileSource<Record>(new Path(tempFile.getAbsolutePath()),
        avroGenericType);

    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());

    assertFalse(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, true));

  }
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    AvroType<StringWrapper> avroReflectType = Avros.reflects(StringWrapper.class);
    AvroFileSource<StringWrapper> personFileSource = new AvroFileSource<StringWrapper>(new Path(
        tempFile.getAbsolutePath()), avroReflectType);

    FormatBundle bundle = personFileSource.getBundle();
    bundle.configure(job.getConfiguration());
    assertTrue(job.getConfiguration().getBoolean(AvroJob.INPUT_IS_REFLECT, false));

  }

}
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      if (format instanceof FileInputFormat && !conf.getBoolean(RuntimeParameters.DISABLE_COMBINE_FILE, false)) {
        format = new CrunchCombineFileInputFormat<Object, Object>(job);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
    // First, build a map of InputFormats to Paths
    for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
      FormatBundle inputBundle = entry.getKey();
      Configuration conf = new Configuration(base);
      inputBundle.configure(conf);
      Job jobCopy = new Job(conf);
      InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
          jobCopy.getConfiguration());
      for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
        Integer nodeIndex = nodeEntry.getKey();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.