Examples of POSplit

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit
The MapReduce Split operator.
The assumption here is that the logical to physical translation will create this dummy operator with just the filename using which the input branch will be stored and used for loading Also the translation should make sure that appropriate filter operators are configured as outputs of this operator using the conditions specified in the LOSplit. So LOSplit will be converted into: | | | Filter1 Filter2 ... Filter3 | | ... | | | ... | ---- POSplit -... ---- This is different than the existing implementation where the POSplit writes to sidefiles after filtering and then loads the appropriate file.
The approach followed here is as good as the old approach if not better in many cases because of the availability of attachinInputs. An optimization that can ensue is if there are multiple loads that load the same file, they can be merged into one and then the operators that take input from the load can be stored. This can be used when the mapPlan executes to read the file only once and attach the resulting tuple as inputs to all the operators that take input from this load. In some cases where the conditions are exclusive and some outputs are ignored, this approach can be worse. But this leads to easier management of the Split and also allows to reuse this data stored from the split job whenever necessary.
org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit
org.apache.pig.pen.physicalOperators.POSplit

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

    @Test
    public void testSpl2() throws Exception {
        PhysicalPlan php = new PhysicalPlan();


        POLoad lA = GenPhyOp.topLoadOp();
        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();
        POFilter fl2 = GenPhyOp.topFilterOp();
        php.add(fl1);
        php.add(fl2);
        php.connect(spl, fl1);
        php.connect(spl, fl2);


        POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
        php.add(lr1);
        php.add(lr2);
        php.connect(fl1, lr1);
        php.connect(fl2, lr2);


        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.addAsLeaf(gr);


        POPackage pk = GenPhyOp.topPackageOp();
        php.addAsLeaf(pk);


        POSplit sp2 = GenPhyOp.topSplitOp();
        php.addAsLeaf(sp2);


        POFilter fl3 = GenPhyOp.topFilterOp();
        POFilter fl4 = GenPhyOp.topFilterOp();
        php.add(fl3);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

    @Test
    public void testSpl3() throws Exception {
        PhysicalPlan php = new PhysicalPlan();


        POLoad lA = GenPhyOp.topLoadOp();
        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();
        fl1.setRequestedParallelism(10);
        POFilter fl2 = GenPhyOp.topFilterOp();
        fl2.setRequestedParallelism(20);
        php.add(fl1);
        php.add(fl2);
        php.connect(spl, fl1);
        php.connect(spl, fl2);


        POSplit sp11 = GenPhyOp.topSplitOp();
        POSplit sp21 = GenPhyOp.topSplitOp();
        php.add(sp11);
        php.add(sp21);
        php.connect(fl1, sp11);
        php.connect(fl2, sp21);


        POFilter fl11 = GenPhyOp.topFilterOp();
        fl11.setRequestedParallelism(10);
        POFilter fl21 = GenPhyOp.topFilterOp();
        fl21.setRequestedParallelism(20);
        POFilter fl22 = GenPhyOp.topFilterOp();
        fl22.setRequestedParallelism(30);
        php.add(fl11);
        php.add(fl21);
        php.add(fl22);
        php.connect(sp11, fl11);
        php.connect(sp21, fl21);
        php.connect(sp21, fl22);


        POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
        lr1.setRequestedParallelism(40);
        POLocalRearrange lr21 = GenPhyOp.topLocalRearrangeOp();
        lr21.setRequestedParallelism(15);
        POLocalRearrange lr22 = GenPhyOp.topLocalRearrangeOp();
        lr22.setRequestedParallelism(35);
        php.add(lr1);
        php.add(lr21);
        php.add(lr22);
        php.connect(fl11, lr1);
        php.connect(fl21, lr21);
        php.connect(fl22, lr22);


        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.addAsLeaf(gr);


        POPackage pk = GenPhyOp.topPackageOp();
        pk.setRequestedParallelism(25);
        php.addAsLeaf(pk);


        POSplit sp2 = GenPhyOp.topSplitOp();
        php.addAsLeaf(sp2);


        POFilter fl3 = GenPhyOp.topFilterOp();
        fl3.setRequestedParallelism(100);
        POFilter fl4 = GenPhyOp.topFilterOp();

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

    public void testMRCompilerErr1() throws Exception {
        PhysicalPlan pp = new PhysicalPlan();
        PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
        pp.merge(ldFil1);


        POSplit op = GenPhyOp.topSplitOp();
        pp.addAsLeaf(op);


      try {
        Util.buildMRPlan(pp, pc);
      } catch (MRCompilerException mrce) {

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

            checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
            PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
            
            Assert.assertTrue(leaf instanceof POSplit);


            POSplit split = (POSplit)leaf;
            
            int i = 0;
            for (PhysicalPlan p: split.getPlans()) {
                checkPhysicalPlan(p, 1, 1, 1);
                ++i;
            }


            Assert.assertEquals(i,2);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

            checkPhysicalPlan(mo.mapPlan, 1, 1, 4);
            PhysicalOperator leaf = mo.mapPlan.getLeaves().get(0);
            
            Assert.assertTrue(leaf instanceof POSplit);
            
            POSplit split = (POSplit)leaf;


            int i = 0;
            for (PhysicalPlan p: split.getPlans()) {
                checkPhysicalPlan(p, 1, 1, 1);
                ++i;
            }


            Assert.assertEquals(i,3);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

      Tuple tuple = TupleFactory.getInstance().newTuple(1);
      tuple.set(0, "1.2345");
      proj.attachInput(tuple);


      // Create POSplit and attach the above cast op as input
      POSplit split = new POSplit(new OperatorKey("", r.nextLong()), -1);
      inputs = new ArrayList<PhysicalOperator>();
      inputs.add(cast);
      split.setInputs(inputs);


      PhysicalPlan pplan = new PhysicalPlan();
      POStore store = GenPhyOp.dummyPigStorageOp();
      pplan.add(store);


      split.addPlan(pplan);


      // setting parentPlan since POSplit access
      // for "if (this.parentPlan.endOfAllInput)"
      split.setParentPlan(pplan);


      // Calling POSplit which in turn calls POCast and should fail.
      Result res = split.getNextTuple();


      assertEquals("POSplit should fail when input processing fails.",
                   res.returnStatus, POStatus.STATUS_ERR);
  }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit

    @Test
    public void testSpl1() throws Exception {
        PhysicalPlan php = new PhysicalPlan();


        POLoad lA = GenPhyOp.topLoadOp();
        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();

View Full Code Here

Examples of org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit

    }
    
    @Override
    public void visit(LOSplit split) throws VisitorException {
  String scope = split.getOperatorKey().scope;
        PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), split.getRequestedParallelism());
        
        LogToPhyMap.put(split, physOp);


        currentPlan.add(physOp);

View Full Code Here

Examples of org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit

    }
    
    @Override
    public void visit(LOSplit split) throws VisitorException {
  String scope = split.getOperatorKey().scope;
        PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), split.getRequestedParallelism());
        
        logToPhyMap.put(split, physOp);


        currentPlan.add(physOp);

View Full Code Here

Examples of org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit

    }
    
    @Override
    public void visit(LOSplit split) throws VisitorException {
  String scope = split.getOperatorKey().scope;
        PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), split.getRequestedParallelism());
        
        LogToPhyMap.put(split, physOp);


        currentPlan.add(physOp);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.