Package org.apache.crunch

Examples of org.apache.crunch.GroupingOptions$Builder

153
154
155
156
157
158
159
160
   *
   * @return a {@code PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, int numReducers, Order key) {
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(table, conf, numReducers, key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here
213
214
215
216
217
218
219
220
      ColumnOrder... columnOrders) {
    PType<T> pType = collection.getPType();
    SortFns.KeyExtraction<T> ke = new SortFns.KeyExtraction<T>(pType, columnOrders);
    PTable<Object, T> pt = collection.by(ke.getByFn(), ke.getKeyType());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(pt, conf, numReducers, columnOrders);
    return pt.groupByKey(options).ungroup().values();
  }
View Full Code Here
389
390
391
392
393
394
395
396
397
398
399
      }
    }, tableOf(writables(KeyValue.class), nulls()));
    List <KeyValue> splitPoints = getSplitPoints(table);
    Path partitionFile = new Path(((MRPipeline) kvs.getPipeline()).createTempPath(), "partition");
    writePartitionInfo(conf, partitionFile, splitPoints);
    GroupingOptions options = GroupingOptions.builder()
        .partitionerClass(TotalOrderPartitioner.class)
        .conf(TotalOrderPartitioner.PARTITIONER_PATH, partitionFile.toString())
        .numReducers(splitPoints.size() + 1)
        .sortComparatorClass(KeyValueComparator.class)
        .build();
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
   */
  public static <T> PCollection<T> sort(PCollection<T> collection, Order order) {
    PTypeFamily tf = collection.getTypeFamily();
    PTableType<T, Void> type = tf.tableOf(collection.getPType(), tf.nulls());
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, collection.getPType(), order);
    PTable<T, Void> pt = collection.parallelDo("sort-pre", new DoFn<T, Pair<T, Void>>() {
      @Override
      public void process(T input, Emitter<Pair<T, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
View Full Code Here
145
146
147
148
149
150
151
152
   * @return a {@link PTable} representing the sorted collection.
   */
  public static <K, V> PTable<K, V> sort(PTable<K, V> table, Order key) {
    PTypeFamily tf = table.getTypeFamily();
    Configuration conf = table.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, table.getKeyType(), key);
    return table.groupByKey(options).ungroup();
  }
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
      public void process(Pair<U, V> input, Emitter<Pair<Pair<U, V>, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, pType, columnOrders);
    PTable<Pair<U, V>, Void> sortedPt = pt.groupByKey(options).ungroup();
    return sortedPt.parallelDo(new DoFn<Pair<Pair<U, V>, Void>, Pair<U, V>>() {
      @Override
      public void process(Pair<Pair<U, V>, Void> input, Emitter<Pair<U, V>> emitter) {
        emitter.emit(input.first());
View Full Code Here
202
203
204
205
206
207
208
209
210
211
212
          public void process(Tuple3<V1, V2, V3> input, Emitter<Pair<Tuple3<V1, V2, V3>, Void>> emitter) {
            emitter.emit(Pair.of(input, (Void) null));
          }
        }, type);
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, pType, columnOrders);
    PTable<Tuple3<V1, V2, V3>, Void> sortedPt = pt.groupByKey(options).ungroup();
    return sortedPt.parallelDo(new DoFn<Pair<Tuple3<V1, V2, V3>, Void>, Tuple3<V1, V2, V3>>() {
      @Override
      public void process(Pair<Tuple3<V1, V2, V3>, Void> input, Emitter<Tuple3<V1, V2, V3>> emitter) {
        emitter.emit(input.first());
View Full Code Here
233
234
235
236
237
238
239
240
241
242
243
          public void process(Tuple4<V1, V2, V3, V4> input, Emitter<Pair<Tuple4<V1, V2, V3, V4>, Void>> emitter) {
            emitter.emit(Pair.of(input, (Void) null));
          }
        }, type);
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, pType, columnOrders);
    PTable<Tuple4<V1, V2, V3, V4>, Void> sortedPt = pt.groupByKey(options).ungroup();
    return sortedPt.parallelDo(new DoFn<Pair<Tuple4<V1, V2, V3, V4>, Void>, Tuple4<V1, V2, V3, V4>>() {
      @Override
      public void process(Pair<Tuple4<V1, V2, V3, V4>, Void> input, Emitter<Tuple4<V1, V2, V3, V4>> emitter) {
        emitter.emit(input.first());
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
      public void process(TupleN input, Emitter<Pair<TupleN, Void>> emitter) {
        emitter.emit(Pair.of(input, (Void) null));
      }
    }, type);
    Configuration conf = collection.getPipeline().getConfiguration();
    GroupingOptions options = buildGroupingOptions(conf, tf, pType, columnOrders);
    PTable<TupleN, Void> sortedPt = pt.groupByKey(options).ungroup();
    return sortedPt.parallelDo(new DoFn<Pair<TupleN, Void>, TupleN>() {
      @Override
      public void process(Pair<TupleN, Void> input, Emitter<TupleN> emitter) {
        emitter.emit(input.first());
View Full Code Here
TOP

Related Classes of org.apache.crunch.GroupingOptions$Builder

  • ch.grengine.code.groovy.DefaultGroovyCompilerFactory
  • co.cask.tigon.sql.internal.DefaultGDATField
  • com.asakusafw.utils.java.model.syntax.Expression
  • com.asakusafw.utils.java.model.syntax.ModelFactory
  • com.asakusafw.utils.java.model.util.ImportBuilder
  • com.asakusafw.utils.java.model.util.TypeBuilder
  • com.basho.riak.client.api.cap.Quorum
  • com.basho.riak.client.cap.Quorum
  • com.basho.riak.client.core.query.Location
  • com.facebook.presto.serde.BlocksFileWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.