Examples of tableOf()

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

222
223
224
225
226
227
228
229
230
231
232
          public void cleanup(Emitter<Pair<Boolean, S>> emitter) {
            if (max != null) {
              emitter.emit(Pair.of(true, max));
            }
          }
        }, tf.tableOf(tf.booleans(), collect.getPType())).groupByKey(1)
        .combineValues(new CombineFn<Boolean, S>() {
          public void process(Pair<Boolean, Iterable<S>> input, Emitter<Pair<Boolean, S>> emitter) {
            S max = null;
            for (S v : input.second()) {
              if (max == null || ((Comparable<S>) max).compareTo(v) < 0) {
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

262
263
264
265
266
267
268
269
270
271
272
          public void cleanup(Emitter<Pair<Boolean, S>> emitter) {
            if (min != null) {
              emitter.emit(Pair.of(false, min));
            }
          }
        }, tf.tableOf(tf.booleans(), collect.getPType())).groupByKey(1)
        .combineValues(new CombineFn<Boolean, S>() {
          public void process(Pair<Boolean, Iterable<S>> input, Emitter<Pair<Boolean, S>> emitter) {
            S min = null;
            for (S v : input.second()) {
              if (min == null || ((Comparable<S>) min).compareTo(v) > 0) {
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

303
304
305
306
307
308
309
310
311
312
313
    PTypeFamily tf = collect.getTypeFamily();
    return collect.parallelDo("Aggregate.aggregator", new MapFn<S, Pair<Boolean, S>>() {
      public Pair<Boolean, S> map(S input) {
        return Pair.of(false, input);
      }
    }, tf.tableOf(tf.booleans(), collect.getPType()))
    .groupByKey(1)
    .combineValues(aggregator)
    .values();
  }
}
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

477
478
479
480
481
482
483
484
485
486
487
   * @param valueType The {@code PType} for the value of the SequenceFile entry
   * @return A new {@code SourceTable<K, V>} instance
   */
  public static <K, V> TableSource<K, V> sequenceFile(Path path, PType<K> keyType, PType<V> valueType) {
    PTypeFamily ptf = keyType.getFamily();
    return new SeqFileTableSource<K, V>(path, ptf.tableOf(keyType, valueType));
  }

  /**
   * Creates a {@code TableSource<K, V>} instance for the SequenceFile(s) at the given {@code Path}s.
   *
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

490
491
492
493
494
495
496
497
498
499
500
   * @param valueType The {@code PType} for the value of the SequenceFile entry
   * @return A new {@code SourceTable<K, V>} instance
   */
  public static <K, V> TableSource<K, V> sequenceFile(List<Path> paths, PType<K> keyType, PType<V> valueType) {
    PTypeFamily ptf = keyType.getFamily();
    return new SeqFileTableSource<K, V>(paths, ptf.tableOf(keyType, valueType));
  }

  /**
   * Creates a {@code Source<String>} instance for the text file(s) at the given path name.
   *
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

247
248
249
250
251
252
253
254
255
256
257
   * @param valueType The {@code PType} for the value of the SequenceFile entry
   * @return A new {@code TableSourceTarget<K, V>} instance
   */
  public static <K, V> TableSourceTarget<K, V> sequenceFile(Path path, PType<K> keyType, PType<V> valueType) {
    PTypeFamily ptf = keyType.getFamily();
    return new SeqFileTableSourceTarget<K, V>(path, ptf.tableOf(keyType, valueType));
  }

  /**
   * Creates a {@code SourceTarget<String>} instance for the text file(s) at the given path name.
   *
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

64
65
66
67
68
69
70
71
72
73
74
    PTypeFamily tf = collect.getTypeFamily();
    return collect.parallelDo("Aggregate.count", new MapFn<S, Pair<S, Long>>() {
      public Pair<S, Long> map(S input) {
        return Pair.of(input, 1L);
      }
    }, tf.tableOf(collect.getPType(), tf.longs()))
        .groupByKey(numPartitions)
        .combineValues(Aggregators.SUM_LONGS());
  }
 
  /**
 
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

83
84
85
86
87
88
89
90
91
92
93
    PTable<Integer, Long> countTable = collect
        .parallelDo("Aggregate.count", new MapFn<S, Pair<Integer, Long>>() {
          public Pair<Integer, Long> map(S input) {
            return Pair.of(1, 1L);
          }
        }, tf.tableOf(tf.ints(), tf.longs()))
        .groupByKey(GroupingOptions.builder().numReducers(1).build())
        .combineValues(Aggregators.SUM_LONGS());
    PCollection<Long> count = countTable.values();
    return new FirstElementPObject<Long>(count);
  }
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

117
118
119
120
121
122
123
124
125
126
127
   *
   * @return a {@code PCollection} representing the sorted collection.
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, int numReducers, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
View Full Code Here

Examples of org.apache.crunch.types.PTypeFamily.tableOf()

76
77
78
79
80
81
82
83
84
85
86
      ParallelDoOptions.Builder optionsBuilder = ParallelDoOptions.builder();
      if (mi.isSourceTarget()) {
        optionsBuilder.sourceTargets((SourceTarget) mi.getSource());
      }
      return left.parallelDo("mapjoin", mapJoinDoFn,
          tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())),
          optionsBuilder.build());
    } else { // in-memory pipeline
      return left.parallelDo(new InMemoryJoinFn<K, U, V>(iterable),
          tf.tableOf(left.getKeyType(), tf.pairs(left.getValueType(), right.getValueType())));
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.