Examples of POFilter

org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter
This is an implementation of the Filter operator. It has an Expression Plan that decides whether the input tuple should be filtered or passed through. To avoid many function calls, the filter operator, stores the Comparison Operator that is the root of the Expression Plan and uses its getNext directly. Since the filter is supposed to return tuples only, getNext is not supported on any other data type.

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter


    @Override
    public void visit(LOSplitOutput loSplitOutput) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        //        System.err.println("Entering Filter");
        POFilter poFilter = new POFilter(new OperatorKey(scope, nodeGen
                .getNextNodeId(scope)), loSplitOutput.getRequestedParallelism());
        poFilter.addOriginalLocation(loSplitOutput.getAlias(), loSplitOutput.getLocation());
        poFilter.setResultType(DataType.BAG);
        currentPlan.add(poFilter);
        logToPhyMap.put(loSplitOutput, poFilter);
        currentPlans.push(currentPlan);


        currentPlan = new PhysicalPlan();


//        PlanWalker childWalker = currentWalker
//                .spawnChildWalker(filter.getFilterPlan());
        PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(loSplitOutput.getFilterPlan());
        pushWalker(childWalker);
        //currentWalker.walk(this);
        currentWalker.walk(
                new ExpToPhyTranslationVisitor( currentWalker.getPlan(), 
                        childWalker, loSplitOutput, currentPlan, logToPhyMap) );
        popWalker();


        poFilter.setPlan(currentPlan);
        currentPlan = currentPlans.pop();


        List<Operator> op = loSplitOutput.getPlan().getPredecessors(loSplitOutput);


        PhysicalOperator from;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

    public void testRun1() throws Exception {
        PhysicalPlan php = new PhysicalPlan();


        PhysicalPlan part1 = new PhysicalPlan();
        POLoad lC = GenPhyOp.topLoadOp();
        POFilter fC = GenPhyOp.topFilterOp();
        fC.setRequestedParallelism(20);
        POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
        lrC.setRequestedParallelism(10);
        POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
        POPackage pkC = GenPhyOp.topPackageOp();
        part1.add(lC);
        part1.add(fC);
        part1.connect(lC, fC);
        part1.add(lrC);
        part1.connect(fC, lrC);
        part1.add(grC);
        part1.connect(lrC, grC);
        part1.add(pkC);
        part1.connect(grC, pkC);


        POPackage pkD = GenPhyOp.topPackageOp();
        pkD.setRequestedParallelism(20);
        POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
        lrD.setRequestedParallelism(30);
        POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
        POLoad lD = GenPhyOp.topLoadOp();
        part1.add(lD);
        part1.add(lrD);
        part1.connect(lD, lrD);


        part1.add(grD);
        part1.connect(lrD, grD);
        part1.add(pkD);
        part1.connect(grD, pkD);


        POPackage pkCD = GenPhyOp.topPackageOp();
        POLocalRearrange lrCD1 = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrCD2 = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grCD = GenPhyOp.topGlobalRearrangeOp();
        part1.add(lrCD1);
        part1.add(lrCD2);
        part1.connect(pkC, lrCD1);
        part1.connect(pkD, lrCD2);
        part1.add(grCD);
        part1.connect(lrCD1, grCD);
        part1.connect(lrCD2, grCD);
        part1.add(pkCD);
        part1.connect(grCD, pkCD);


        POLoad lA = GenPhyOp.topLoadOp();
        POLoad lB = GenPhyOp.topLoadOp();


        // POLoad lC = lA;
        POFilter fA = GenPhyOp.topFilterOp();


        POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();


        POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();


        POPackage pkAB = GenPhyOp.topPackageOp();


        POFilter fAB = GenPhyOp.topFilterOp();
        POUnion unABC = GenPhyOp.topUnionOp();


        php.add(lA);
        php.add(lB);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

    public void testRun2() throws Exception {
        PhysicalPlan php = new PhysicalPlan();


        PhysicalPlan part1 = new PhysicalPlan();
        POLoad lC = GenPhyOp.topLoadOp();
        POFilter fC = GenPhyOp.topFilterOp();
        POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
        POPackage pkC = GenPhyOp.topPackageOp();
        part1.add(lC);
        part1.add(fC);
        part1.connect(lC, fC);
        part1.add(lrC);
        part1.connect(fC, lrC);
        part1.add(grC);
        part1.connect(lrC, grC);
        part1.add(pkC);
        part1.connect(grC, pkC);


        POPackage pkD = GenPhyOp.topPackageOp();
        POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
        POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
        POLoad lD = GenPhyOp.topLoadOp();
        part1.add(lD);
        part1.add(lrD);
        part1.connect(lD, lrD);


        part1.add(grD);
        part1.connect(lrD, grD);
        part1.add(pkD);
        part1.connect(grD, pkD);
        part1.connect(pkD, grC);


        POLoad lA = GenPhyOp.topLoadOp();
        POLoad lB = GenPhyOp.topLoadOp();


        // POLoad lC = lA;
        POFilter fA = GenPhyOp.topFilterOp();


        POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();


        POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();


        POPackage pkAB = GenPhyOp.topPackageOp();


        POFilter fAB = GenPhyOp.topFilterOp();
        POUnion unABC = GenPhyOp.topUnionOp();


        php.add(lA);
        php.add(lB);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();
        POFilter fl2 = GenPhyOp.topFilterOp();
        php.add(fl1);
        php.add(fl2);
        php.connect(spl, fl1);
        php.connect(spl, fl2);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();
        POFilter fl2 = GenPhyOp.topFilterOp();
        php.add(fl1);
        php.add(fl2);
        php.connect(spl, fl1);
        php.connect(spl, fl2);


        POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
        POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
        php.add(lr1);
        php.add(lr2);
        php.connect(fl1, lr1);
        php.connect(fl2, lr2);


        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.addAsLeaf(gr);


        POPackage pk = GenPhyOp.topPackageOp();
        php.addAsLeaf(pk);


        POSplit sp2 = GenPhyOp.topSplitOp();
        php.addAsLeaf(sp2);


        POFilter fl3 = GenPhyOp.topFilterOp();
        POFilter fl4 = GenPhyOp.topFilterOp();
        php.add(fl3);
        php.add(fl4);
        php.connect(sp2, fl3);
        php.connect(sp2, fl4);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

        POSplit spl = GenPhyOp.topSplitOp();
        php.add(lA);
        php.add(spl);
        php.connect(lA, spl);


        POFilter fl1 = GenPhyOp.topFilterOp();
        fl1.setRequestedParallelism(10);
        POFilter fl2 = GenPhyOp.topFilterOp();
        fl2.setRequestedParallelism(20);
        php.add(fl1);
        php.add(fl2);
        php.connect(spl, fl1);
        php.connect(spl, fl2);


        POSplit sp11 = GenPhyOp.topSplitOp();
        POSplit sp21 = GenPhyOp.topSplitOp();
        php.add(sp11);
        php.add(sp21);
        php.connect(fl1, sp11);
        php.connect(fl2, sp21);


        POFilter fl11 = GenPhyOp.topFilterOp();
        fl11.setRequestedParallelism(10);
        POFilter fl21 = GenPhyOp.topFilterOp();
        fl21.setRequestedParallelism(20);
        POFilter fl22 = GenPhyOp.topFilterOp();
        fl22.setRequestedParallelism(30);
        php.add(fl11);
        php.add(fl21);
        php.add(fl22);
        php.connect(sp11, fl11);
        php.connect(sp21, fl21);
        php.connect(sp21, fl22);


        POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
        lr1.setRequestedParallelism(40);
        POLocalRearrange lr21 = GenPhyOp.topLocalRearrangeOp();
        lr21.setRequestedParallelism(15);
        POLocalRearrange lr22 = GenPhyOp.topLocalRearrangeOp();
        lr22.setRequestedParallelism(35);
        php.add(lr1);
        php.add(lr21);
        php.add(lr22);
        php.connect(fl11, lr1);
        php.connect(fl21, lr21);
        php.connect(fl22, lr22);


        POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
        php.addAsLeaf(gr);


        POPackage pk = GenPhyOp.topPackageOp();
        pk.setRequestedParallelism(25);
        php.addAsLeaf(pk);


        POSplit sp2 = GenPhyOp.topSplitOp();
        php.addAsLeaf(sp2);


        POFilter fl3 = GenPhyOp.topFilterOp();
        fl3.setRequestedParallelism(100);
        POFilter fl4 = GenPhyOp.topFilterOp();
        fl4.setRequestedParallelism(80);
        php.add(fl3);
        php.add(fl4);
        php.connect(sp2, fl3);
        php.connect(sp2, fl4);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

        php.merge(grpChain2);


        php.connect(leaf, grpChain2.getRoots().get(0));


        leaf = php.getLeaves().get(0);
        POFilter fl = GenPhyOp.topFilterOp();
        php.add(fl);


        php.connect(leaf, fl);


        POStore st = GenPhyOp.topStoreOp();

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

        ld1.setLFile(fSpec);


        POLoad ld2 = GenPhyOp.topLoadOp();
        ld2.setLFile(fSpec);


        POFilter fl1 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.LTE);


        POFilter fl2 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.GT);


        int[] flds = {0,2};
        Tuple sample = new DefaultTuple();
        sample.append(new String("S"));
        sample.append(new String("x"));

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

                t.set(1, new Integer(ints[j + 1]));
                inbag.add(t);
            }


            PORead read = GenPhyOp.topReadOp(inbag);
            POFilter filter = GenPhyOp.connectedFilterOp(read);
            filter.setPlan(ip);


            PhysicalPlan op = new PhysicalPlan();
            op.add(filter);
            op.add(read);
            op.connect(read, filter);


            DataBag outbag = BagFactory.getInstance().newDefaultBag();
            Result res;
            Tuple t = tf.newTuple();
            do {
                res = filter.getNextTuple();
                if (res.returnStatus == POStatus.STATUS_OK) {
                    outbag.add((Tuple)res.result);
                }
            } while (res.returnStatus == POStatus.STATUS_OK);
            assertEquals("Running " + this.getClass().getName() + "with nullFlags set to "

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POFilter

                t.set(1, new Integer(ints[j + 1]));
                inbag.add(t);
            }


            PORead read = GenPhyOp.topReadOp(inbag);
            POFilter filter = GenPhyOp.connectedFilterOp(read);
            filter.setPlan(ip);


            PhysicalPlan op = new PhysicalPlan();
            op.add(filter);
            op.add(read);
            op.connect(read, filter);


            DataBag outbag = BagFactory.getInstance().newDefaultBag();
            Result res;
            Tuple t = tf.newTuple();
            do {
                res = filter.getNextTuple();
                if (res.returnStatus == POStatus.STATUS_OK) {
                    outbag.add((Tuple)res.result);
                }
            } while (res.returnStatus == POStatus.STATUS_OK);
            assertEquals(POStatus.STATUS_EOP, res.returnStatus);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.