Package com.tdunning.plume.local.lazy.op

Examples of com.tdunning.plume.local.lazy.op.GroupByKey


    List<DeferredOp> groupBys = OptimizerTools.getAll(outputs, GroupByKey.class);
    int mscrId = 1;
    Set<MSCR> mscrs = new HashSet<MSCR>();
    // For all found GroupByKey blocks
    for(DeferredOp gBK: groupBys) {
      GroupByKey groupBy = (GroupByKey<?,?>)gBK;
      // Gather all information needed for MSCR from this GBK
      Set<PCollection<?>> inputs = new HashSet<PCollection<?>>();
      Set<GroupByKey<?, ?>> outputChannels = new HashSet<GroupByKey<?, ?>>();
      Set<Flatten<?>> unGroupedOutputChannels = new HashSet<Flatten<?>>();
      Set<PCollection<?>> bypassChannels = new HashSet<PCollection<?>>();
      Stack<LazyCollection<?>> toVisit = new Stack<LazyCollection<?>>();
      Set<LazyCollection<?>> visited = new HashSet<LazyCollection<?>>();
      LazyCollection<?> origin = (LazyCollection<?>)groupBy.getOrigin();
      toVisit.push(origin);
      outputChannels.add(groupBy);
      while(!toVisit.isEmpty()) {
        LazyCollection<?> current = toVisit.pop();
        visited.add(current);
View Full Code Here


        for (Object obj : parent) {
          function.process(obj, emitter);
        }
      // GroupByKey
      } else if(op instanceof GroupByKey) {
        GroupByKey gBK = (GroupByKey) op;
        parent = execute(gBK.getOrigin());
        Map<Object, List> groupMap = Maps.newHashMap();
        // Perform in-memory group by operation
        for (Object obj : parent) {
          Pair p = (Pair)obj;
          List list = groupMap.get(p.getKey());
View Full Code Here

TOP

Related Classes of com.tdunning.plume.local.lazy.op.GroupByKey

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.