Package com.tdunning.plume.local.lazy.op

Examples of com.tdunning.plume.local.lazy.op.ParallelDo


      for(PCollection coll: flatten.getOrigins()) {
        LazyCollection lCol = (LazyCollection)coll;
        if(lCol.isMaterialized()) {
          mscr.addInput(coll);
        } else if(lCol.deferredOp instanceof ParallelDo) {
          ParallelDo pDo = (ParallelDo)lCol.deferredOp;
          if(((LazyCollection)pDo.getOrigin()).isMaterialized()) {
            mscr.addInput(pDo.getOrigin());
          } else if(pDo instanceof MultipleParallelDo) {
            mscr.addInput(pDo.getOrigin());
          } else {
            mscr.addInput(coll);
          }
        } else {
          mscr.addInput(coll);
View Full Code Here


   
    PCollection col  = mscr.getChannelByNumber().get(arg0.sourceId);
    OutputChannel oC = mscr.getOutputChannels().get(col);
    if(oC.reducer != null) {
      // apply reducer
      ParallelDo pDo = oC.reducer;
      DoFn reducer = pDo.getFunction(); // TODO how to check / report this
      List<WritableComparable> vals = Lists.newArrayList();
      for(PlumeObject val: values) {
        vals.add(val.obj);
      }
      reducer.process(Pair.create(arg0.obj, vals), new EmitFn() {
View Full Code Here

          result.add(v);
        }
      };
      // ParallelDo
      if (op instanceof ParallelDo) {
        ParallelDo pDo = (ParallelDo) op;
        parent = execute((LazyCollection)pDo.getOrigin());
        for (Object obj : parent) {
          pDo.getFunction().process(obj, emitter);
        }
      // MultipleParallelDo -> parallel operations that read the same collection
      // In this version of executor, we will only compute the current collection, not its neighbors
      } else if(op instanceof MultipleParallelDo) {
        MultipleParallelDo mPDo = (MultipleParallelDo) op;
View Full Code Here

      // Recursively apply this function to this flatten's origin
      LazyCollection<T> fInput = (LazyCollection<T>)col;
      sinkFlattens(fInput);
      // Sink
      LazyCollection<?> newInput = new LazyCollection();
      newInput.deferredOp = new ParallelDo(op.getFunction(), fInput, newInput);
      newInput.type = ((LazyCollection)flatten.getDest()).getType();
      fInput.downOps.remove(0);
      fInput.addDownOp(newInput.deferredOp);
      newOrigins.add(newInput);
    }
View Full Code Here

      }
      if(dOp instanceof MultipleParallelDo) {
        return;
      }
    }
    ParallelDo pDo = (ParallelDo)output.getDeferredOp();
    LazyCollection<T> orig = (LazyCollection<T>)pDo.getOrigin();
    int willAdd = 0;
    for(DeferredOp op: orig.getDownOps()) {
      if(op instanceof ParallelDo) {
        willAdd++;
      }
    }
    if(willAdd == 1) { // Parent doesn't have more ParallelDos to fuse
      // Recursively apply this function to parent
      fuseSiblingParallelDos(orig);
      return;
    }
    // MultipleParallelDo is viable, create it
    MultipleParallelDo<T> mPDo = new MultipleParallelDo<T>(orig);
    mPDo.addDest(pDo.getFunction(), output);
    orig.downOps.remove(pDo);
    output.deferredOp = mPDo;
    List<DeferredOp> newList = new ArrayList<DeferredOp>();
    for(DeferredOp op: orig.getDownOps()) {
      if(op instanceof ParallelDo) {
        ParallelDo thisPDo = (ParallelDo)op;
        mPDo.addDest(thisPDo.getFunction(), thisPDo.getDest());
        LazyCollection thisDest = (LazyCollection)thisPDo.getDest();
        thisDest.deferredOp = mPDo;
      } else {
        newList.add(op);
      }
    }
View Full Code Here

          fuseParallelDos(col);
        }
        return;
      }
    }
    ParallelDo p1 = (ParallelDo)output.getDeferredOp();
    LazyCollection orig1 = (LazyCollection)p1.getOrigin();
    if(orig1.isMaterialized()) {
      return;
    }
    if(!(orig1.getDeferredOp() instanceof ParallelDo)) {
      // Recursively apply this function to parent node
      fuseParallelDos(orig1);
      return;
    }
    // At this point we know ParallelDo fusion can be done -> Perform it
    ParallelDo p2 = (ParallelDo)orig1.getDeferredOp();
    // Lift combine values
    if(p2 instanceof CombineValues) {
      LazyCollection lCol = (LazyCollection)p2.getOrigin();
      if(!lCol.isMaterialized() && lCol.getDeferredOp() instanceof GroupByKey) {
        // Upper parallel do is CombineValues and follows a GroupByKey -> don't join
        fuseParallelDos(orig1);
        return;
      }
    }
    final DoFn f1 = p1.getFunction();
    final DoFn f2 = p2.getFunction();
    // Define the joined function
    DoFn newFn = new DoFn() {
      @Override
      public void process(Object v, final EmitFn emitter) {
        f2.process(v, new EmitFn() {
          @Override
          public void emit(Object v) {
            f1.process(v, emitter);
          }
        });
      }
    };
    LazyCollection orig2 = (LazyCollection)p2.getOrigin();
    ParallelDo newPDo = new ParallelDo(newFn, orig2, output);
    // Clean & change pointers
    orig2.downOps.remove(p2);
    orig1.downOps.remove(p1);
    orig2.addDownOp(newPDo);
    output.deferredOp = newPDo;
View Full Code Here

    assertTrue(lOutput.getDeferredOp() instanceof Flatten);
    Flatten flatten = (Flatten)lOutput.getDeferredOp();
    assertEquals(flatten.getOrigins().size(), 2);
    for(int i = 0; i < 2; i++) {
      LazyCollection<Integer> origin = (LazyCollection<Integer>) flatten.getOrigins().get(i);
      ParallelDo newPDo = (ParallelDo)origin.getDeferredOp();
      assertEquals(newPDo.getFunction(), plusOne);
      assertTrue(newPDo.getOrigin() == input1 || newPDo.getOrigin() == input2);
    }
  }
View Full Code Here

TOP

Related Classes of com.tdunning.plume.local.lazy.op.ParallelDo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.