Package org.apache.crunch.types

Examples of org.apache.crunch.types.PTypeFamily.pairs()

128
129
130
131
132
133
134
135
136
137
138
    PTypeFamily rtf = right.getTypeFamily();

    PTable<Pair<Integer, Integer>, Pair<K1, U>> leftCross = left.parallelDo(new GFCross<Pair<K1, U>>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), ltf.pairs(left.getKeyType(), left.getValueType())));
    PTable<Pair<Integer, Integer>, Pair<K2, V>> rightCross = right.parallelDo(new GFCross<Pair<K2, V>>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), rtf.pairs(right.getKeyType(), right.getValueType())));

    PTable<Pair<Integer, Integer>, Pair<Collection<Pair<K1, U>>, Collection<Pair<K2, V>>>> cg = leftCross
        .cogroup(rightCross);

    PTypeFamily ctf = cg.getTypeFamily();
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
                  for (Pair<K2, V> r : input.second().second()) {
                    emitter.emit(Pair.of(Pair.of(l.first(), r.first()), Pair.of(l.second(), r.second())));
                  }
                }
              }
            }, ctf.tableOf(ctf.pairs(left.getKeyType(), right.getKeyType()),
                ctf.pairs(left.getValueType(), right.getValueType())));
  }

  /**
   * Performs a full cross join on the specified {@link PCollection}s (using the
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
                    emitter.emit(Pair.of(Pair.of(l.first(), r.first()), Pair.of(l.second(), r.second())));
                  }
                }
              }
            }, ctf.tableOf(ctf.pairs(left.getKeyType(), right.getKeyType()),
                ctf.pairs(left.getValueType(), right.getValueType())));
  }

  /**
   * Performs a full cross join on the specified {@link PCollection}s (using the
   * same strategy as Pig's CROSS operator).
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
      throw new Error();

    PTable<Pair<Integer, Integer>, U> leftCross = left.parallelDo(new GFCross<U>(0, parallelism),
        ltf.tableOf(ltf.pairs(ltf.ints(), ltf.ints()), left.getPType()));
    PTable<Pair<Integer, Integer>, V> rightCross = right.parallelDo(new GFCross<V>(1, parallelism),
        rtf.tableOf(rtf.pairs(rtf.ints(), rtf.ints()), right.getPType()));

    PTable<Pair<Integer, Integer>, Pair<Collection<U>, Collection<V>>> cg = leftCross.cogroup(rightCross);

    PTypeFamily ctf = cg.getTypeFamily();
View Full Code Here
217
218
219
220
221
222
223
224
225
          for (V r : input.second().second()) {
            emitter.emit(Pair.of(l, r));
          }
        }
      }
    }, ctf.pairs(left.getPType(), right.getPType()));
  }

}
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
  }

  public static <K, V> PTable<K, V> top(PTable<K, V> ptable, int limit, boolean maximize) {
    PTypeFamily ptf = ptable.getTypeFamily();
    PTableType<K, V> base = ptable.getPTableType();
    PType<Pair<K, V>> pairType = ptf.pairs(base.getKeyType(), base.getValueType());
    PTableType<Integer, Pair<K, V>> inter = ptf.tableOf(ptf.ints(), pairType);
    return ptable.parallelDo("top" + limit + "map", new TopKFn<K, V>(limit, maximize), inter).groupByKey(1)
        .combineValues(new TopKCombineFn<K, V>(limit, maximize))
        .parallelDo("top" + limit + "reduce", new DoFn<Pair<Integer, Pair<K, V>>, Pair<K, V>>() {
          public void process(Pair<Integer, Pair<K, V>> input, Emitter<Pair<K, V>> emitter) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.