Examples of LOJoin

Examples of org.apache.pig.experimental.logical.relational.LOJoin


        @Override
        public void transform(OperatorPlan matched) throws IOException {
            subPlan = new OperatorSubPlan(currentPlan);


            LOJoin join = (LOJoin)matched.getSources().get(0);
            subPlan.add(join);     
            
            Operator next = matched.getSinks().get(0);
            while(next != null && next instanceof LOFilter) {
                LOFilter filter = (LOFilter)next;

View Full Code Here

Examples of org.apache.pig.experimental.logical.relational.LOJoin

          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
        
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(0));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 0);
          fanotherx.neverUseForRealSetUid(1);
          ProjectExpression fa = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 2);
          fa.neverUseForRealSetUid(3);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, DataType.INTEGER, new Integer(1));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 3);
          fanotherb.neverUseForRealSetUid(4);
          ProjectExpression fy = new ProjectExpression(filterPlan, DataType.INTEGER, 0, 1);
          fy.neverUseForRealSetUid(2);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
        
          LOFilter D = new LOFilter(lp, filterPlan);
          D.neverUseForRealSetSchema(cschema);
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
        
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500);
        optimizer.optimize();
        
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          aschema.getField(0).uid = 1;
          aschema.getField(1).uid = 2;
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected);
          expected.add(A);
          
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, DataType.INTEGER, 0, 0);
          fx.neverUseForRealSetUid(1);
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, DataType.INTEGER, new Integer(0));
          new EqualExpression(DAfilterPlan, fx, fc0);
          
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
          
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          bschema.getField(0).uid = 3;
          bschema.getField(1).uid = 4;
          LOLoad B = new LOLoad(null, bschema, expected);
          expected.add(B);
          
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
          ProjectExpression fb = new ProjectExpression(DBfilterPlan, DataType.INTEGER, 0, 1);
          fb.neverUseForRealSetUid(4);
          ConstantExpression fc1 = new ConstantExpression(DBfilterPlan, DataType.INTEGER, new Integer(1));
          new EqualExpression(DBfilterPlan, fb, fc1);
          
          LOFilter DB = new LOFilter(expected, DBfilterPlan);
          DB.neverUseForRealSetSchema(bschema);
          expected.add(DB);
          expected.connect(B, DB);
          
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.INTEGER));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.INTEGER));
          cschema.getField(0).uid = 1;
          cschema.getField(1).uid = 2;
          cschema.getField(2).uid = 3;
          cschema.getField(3).uid = 4;
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          ProjectExpression x = new ProjectExpression(aprojplan, DataType.INTEGER, 0, 0);
          x.neverUseForRealSetUid(1);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          ProjectExpression y = new ProjectExpression(bprojplan, DataType.INTEGER, 1, 0);
          y.neverUseForRealSetUid(3);
          MultiMap<Integer, LogicalExpressionPlan> mm = 
              new MultiMap<Integer, LogicalExpressionPlan>();
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
          LOJoin C = new LOJoin(expected, mm, JOINTYPE.HASH, new boolean[] {true, true});
          C.neverUseForRealSetSchema(cschema);
          expected.add(C);
          expected.connect(DA, C);
          expected.connect(DB, C);
          
          // D = filter

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

        if (before instanceof LOCogroup) {
            LOCogroup cg = (LOCogroup) before ;
            cg.switchGroupByPlanOp(after, newNode);
        }
        if (before instanceof LOJoin) {
            LOJoin frj = (LOJoin) before ;
            frj.switchJoinColPlanOp(after, newNode);
        }
        if (before instanceof LOJoin) {
            LOJoin frj = (LOJoin) before ;
            frj.switchJoinColPlanOp(after, newNode);
        }


        // Visit all the inner plans of before and change their projects to
        // connect to newNode instead of after.
        // Find right inner plan(s) to visit

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

        if (before instanceof LOCogroup) {
            LOCogroup cg = (LOCogroup) before ;
            cg.switchGroupByPlanOp(after, newNode);
        }
        if (before instanceof LOJoin) {
            LOJoin frj = (LOJoin) before ;
            frj.switchJoinColPlanOp(after, newNode);
        }
        if (before instanceof LOJoin) {
            LOJoin frj = (LOJoin) before ;
            frj.switchJoinColPlanOp(after, newNode);
        }


        // Visit all the inner plans of before and change their projects to
        // connect to newNode instead of after.
        // Find right inner plan(s) to visit

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin


    public void testQueryFRJoinNoSchema() throws FrontendException {
        String query = "c = join (load 'a') by $0, (load 'b') by $0 using \"replicated\";";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin frjoin = (LOJoin)lp.getLeaves().get(0);
        assertTrue(frjoin.getRelevantInputs(0, 0) == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

    @Test
    public void testQueryJoinNoSchema() throws FrontendException {
        String query = "c = join (load 'a') by $0, (load 'b') by $0;";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin join = (LOJoin)lp.getLeaves().get(0);
        assertTrue(join.getRelevantInputs(0, 0) == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

    @Test
    public void testQueryFRJoinSchema() throws FrontendException {
        String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b' as (url, rank)) by $0 using \"replicated\";";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin frjoin = (LOJoin)lp.getLeaves().get(0);
        RequiredFields frjoinRelevantFields0 = frjoin.getRelevantInputs(0, 0).get(0);
        assertTrue(frjoinRelevantFields0.getNeedAllFields() == false);
        assertTrue(frjoinRelevantFields0.getNeedNoFields() == false);
        assertTrue(frjoinRelevantFields0.getFields().size() == 1);
        assertTrue(frjoinRelevantFields0.getFields().get(0).first == 0);
        assertTrue(frjoinRelevantFields0.getFields().get(0).second == 0);
        
        assertTrue(frjoin.getRelevantInputs(0, 0).get(1)==null);
        
        RequiredFields frjoinRelevantFields1 = frjoin.getRelevantInputs(0, 1).get(0);
        assertTrue(frjoinRelevantFields1.getNeedAllFields() == false);
        assertTrue(frjoinRelevantFields1.getNeedNoFields() == false);
        assertTrue(frjoinRelevantFields1.getFields().size() == 1);
        assertTrue(frjoinRelevantFields1.getFields().get(0).first == 0);
        assertTrue(frjoinRelevantFields1.getFields().get(0).second == 1);
        
        assertTrue(frjoin.getRelevantInputs(0, 1).get(1)==null);
        
        RequiredFields frjoinRelevantFields2 = frjoin.getRelevantInputs(0, 2).get(1);
        assertTrue(frjoinRelevantFields2.getNeedAllFields() == false);
        assertTrue(frjoinRelevantFields2.getNeedNoFields() == false);
        assertTrue(frjoinRelevantFields2.getFields().size() == 1);
        assertTrue(frjoinRelevantFields2.getFields().get(0).first == 1);
        assertTrue(frjoinRelevantFields2.getFields().get(0).second == 0);
        
        assertTrue(frjoin.getRelevantInputs(0, 2).get(0)==null);
        
        RequiredFields frjoinRelevantFields3 = frjoin.getRelevantInputs(0, 3).get(1);
        assertTrue(frjoinRelevantFields3.getNeedAllFields() == false);
        assertTrue(frjoinRelevantFields3.getNeedNoFields() == false);
        assertTrue(frjoinRelevantFields3.getFields().size() == 1);
        assertTrue(frjoinRelevantFields3.getFields().get(0).first == 1);
        assertTrue(frjoinRelevantFields3.getFields().get(0).second == 1);
        
        assertTrue(frjoin.getRelevantInputs(0, 3).get(0)==null);
        
        assertTrue(frjoin.getRelevantInputs(0, 4)==null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

    @Test
    public void testQueryJoinWithSchema() throws FrontendException {
        String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b' as (url, rank)) by $0;";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin join = (LOJoin)lp.getLeaves().get(0);
        RequiredFields joinRelevantFields0 = join.getRelevantInputs(0, 0).get(0);
        assertTrue(joinRelevantFields0.getNeedAllFields() == false);
        assertTrue(joinRelevantFields0.getNeedNoFields() == false);
        assertTrue(joinRelevantFields0.getFields().size() == 1);
        assertTrue(joinRelevantFields0.getFields().get(0).first == 0);
        assertTrue(joinRelevantFields0.getFields().get(0).second == 0);
        
        assertTrue(join.getRelevantInputs(0, 0).get(1)==null);
        
        RequiredFields joinRelevantFields1 = join.getRelevantInputs(0, 1).get(0);
        assertTrue(joinRelevantFields1.getNeedAllFields() == false);
        assertTrue(joinRelevantFields1.getNeedNoFields() == false);
        assertTrue(joinRelevantFields1.getFields().size() == 1);
        assertTrue(joinRelevantFields1.getFields().get(0).first == 0);
        assertTrue(joinRelevantFields1.getFields().get(0).second == 1);
        
        assertTrue(join.getRelevantInputs(0, 1).get(1)==null);
        
        RequiredFields joinRelevantFields2 = join.getRelevantInputs(0, 2).get(1);
        assertTrue(joinRelevantFields2.getNeedAllFields() == false);
        assertTrue(joinRelevantFields2.getNeedNoFields() == false);
        assertTrue(joinRelevantFields2.getFields().size() == 1);
        assertTrue(joinRelevantFields2.getFields().get(0).first == 1);
        assertTrue(joinRelevantFields2.getFields().get(0).second == 0);
        
        assertTrue(join.getRelevantInputs(0, 2).get(0)==null);
        
        RequiredFields joinRelevantFields3 = join.getRelevantInputs(0, 3).get(1);
        assertTrue(joinRelevantFields3.getNeedAllFields() == false);
        assertTrue(joinRelevantFields3.getNeedNoFields() == false);
        assertTrue(joinRelevantFields3.getFields().size() == 1);
        assertTrue(joinRelevantFields3.getFields().get(0).first == 1);
        assertTrue(joinRelevantFields3.getFields().get(0).second == 1);
        
        assertTrue(join.getRelevantInputs(0, 3).get(0)==null);
        
        assertTrue(join.getRelevantInputs(0, 4)==null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

    @Test
    public void testQueryFRJoinWithMixedSchema() throws FrontendException {
        String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b') by $0 using \"replicated\";";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin frjoin = (LOJoin)lp.getLeaves().get(0);
        assertTrue(frjoin.getRelevantInputs(0, 0)==null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOJoin

    @Test
    public void testQueryJoinWithMixedSchema() throws FrontendException {
        String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b') by $0;";
        LogicalPlan lp = planTester.buildPlan(query);


        LOJoin join = (LOJoin)lp.getLeaves().get(0);
        assertTrue(join.getRelevantInputs(0, 0)==null);
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.