Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit

1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
            POForEach fe_c1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),
                -1, eps_c1, flat_c1);
            fe_c1.setResultType(DataType.TUPLE);
            mro.combinePlan.addAsLeaf(fe_c1);
           
            POLimit pLimit = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
          pLimit.setLimit(limit);
          mro.combinePlan.addAsLeaf(pLimit);
           
            List<PhysicalPlan> eps_c2 = new ArrayList<PhysicalPlan>();
            eps_c2.addAll(sort.getSortPlans());
       
          POLocalRearrange lr_c2 = new POLocalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)));
          try {
                lr_c2.setIndex(0);
            } catch (ExecException e) {
              int errCode = 2058;
              String msg = "Unable to set index on newly created POLocalRearrange.";             
                throw new PlanException(msg, errCode, PigException.BUG, e);
            }
          lr_c2.setKeyType((fields.length>1) ? DataType.TUPLE : keyType);
          lr_c2.setPlans(eps_c2);
          lr_c2.setResultType(DataType.TUPLE);
          mro.combinePlan.addAsLeaf(lr_c2);
        }
       
        POPackageLite pkg = new POPackageLite(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType((fields == null || fields.length>1) ? DataType.TUPLE :
            keyType);
        pkg.setNumInps(1);      
        mro.reducePlan.add(pkg);
       
        PhysicalPlan ep = new PhysicalPlan();
        POProject prj = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
        prj.setColumn(1);
        prj.setOverloaded(false);
        prj.setResultType(DataType.BAG);
        ep.add(prj);
        List<PhysicalPlan> eps2 = new ArrayList<PhysicalPlan>();
        eps2.add(ep);
        List<Boolean> flattened = new ArrayList<Boolean>();
        flattened.add(true);
        POForEach nfe1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),-1,eps2,flattened);
        mro.reducePlan.add(nfe1);
        mro.reducePlan.connect(pkg, nfe1);
       
        if (limit!=-1)
        {
          POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pLimit2.setLimit(limit);
        mro.reducePlan.addAsLeaf(pLimit2);
        }

//        ep1.add(innGen);
        return mro;
View Full Code Here
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
    }
   
    @Override
    public void visit(LOLimit loLimit) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POLimit poLimit = new POLimit(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),
                loLimit.getRequestedParallelisam());
        poLimit.setLimit(loLimit.getLimit());
        poLimit.setAlias(loLimit.getAlias());
        poLimit.setResultType(DataType.BAG);
        currentPlan.add(poLimit);
        logToPhyMap.put(loLimit, poLimit);

        if (loLimit.getLimitPlan() != null) {
            // add expression plan to POLimit
            currentPlans.push(currentPlan);
            currentPlan = new PhysicalPlan();
            PlanWalker childWalker = new ReverseDependencyOrderWalkerWOSeenChk(loLimit.getLimitPlan());
            pushWalker(childWalker);
            currentWalker.walk(new ExpToPhyTranslationVisitor(currentWalker.getPlan(), childWalker, loLimit,
                    currentPlan, logToPhyMap));
            poLimit.setLimitPlan(currentPlan);
            popWalker();
            currentPlan = currentPlans.pop();
        }

        Operator op = loLimit.getPlan().getPredecessors(loLimit).get(0);
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
207
                op = firstMROp.reducePlan.getSuccessors(op).get(0);
                break;
            }
        }
       
        POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pLimit2.setLimit(firstMROp.limit);
        pLimit2.setLimitPlan(firstMROp.limitPlan);
        secondMROp.reducePlan.addAsLeaf(pLimit2);

        while (true) {
            if (op instanceof POStore) break;
            PhysicalOperator opToMove = op;
View Full Code Here
969
970
971
972
973
974
975
976
977
978
979
980
981
                if (mro.reducePlan.isEmpty())
                {
                    MRUtil.simpleConnectMapToReduce(mro, scope, nig);
                    mro.requestedParallelism = 1;
                    if (!pigContext.inIllustrator) {
                        POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
                        pLimit2.setLimit(op.getLimit());
                        pLimit2.setLimitPlan(op.getLimitPlan());
                        mro.reducePlan.addAsLeaf(pLimit2);
                    } else {
                        mro.reducePlan.addAsLeaf(op);
                    }
                }
View Full Code Here
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
            POForEach fe_c1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),
                -1, eps_c1, flat_c1);
            fe_c1.setResultType(DataType.TUPLE);
            mro.combinePlan.addAsLeaf(fe_c1);
           
            POLimit pLimit = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
          pLimit.setLimit(limit);
          mro.combinePlan.addAsLeaf(pLimit);
           
            List<PhysicalPlan> eps_c2 = new ArrayList<PhysicalPlan>();
            eps_c2.addAll(sort.getSortPlans());
       
          POLocalRearrange lr_c2 = new POLocalRearrange(new OperatorKey(scope,nig.getNextNodeId(scope)));
          try {
                lr_c2.setIndex(0);
            } catch (ExecException e) {
              int errCode = 2058;
              String msg = "Unable to set index on newly created POLocalRearrange.";             
                throw new PlanException(msg, errCode, PigException.BUG, e);
            }
          lr_c2.setKeyType((fields.length>1) ? DataType.TUPLE : keyType);
          lr_c2.setPlans(eps_c2);
          lr_c2.setResultType(DataType.TUPLE);
          mro.combinePlan.addAsLeaf(lr_c2);
        }
       
        POPackageLite pkg = new POPackageLite(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType((fields == null || fields.length>1) ? DataType.TUPLE :
            keyType);
        pkg.setNumInps(1);      
        mro.reducePlan.add(pkg);
       
        PhysicalPlan ep = new PhysicalPlan();
        POProject prj = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
        prj.setColumn(1);
        prj.setOverloaded(false);
        prj.setResultType(DataType.BAG);
        ep.add(prj);
        List<PhysicalPlan> eps2 = new ArrayList<PhysicalPlan>();
        eps2.add(ep);
        List<Boolean> flattened = new ArrayList<Boolean>();
        flattened.add(true);
        POForEach nfe1 = new POForEach(new OperatorKey(scope,nig.getNextNodeId(scope)),-1,eps2,flattened);
        mro.reducePlan.add(nfe1);
        mro.reducePlan.connect(pkg, nfe1);
        mro.phyToMRMap.put(sort, nfe1);
        if (limit!=-1)
        {
          POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pLimit2.setLimit(limit);
        mro.reducePlan.addAsLeaf(pLimit2);
        mro.phyToMRMap.put(sort, pLimit2);
        }

//        ep1.add(innGen);
View Full Code Here
810
811
812
813
814
815
816
817
818
819
            if (inputConstraints.size() == 1) {
                inputConstraints.add(inputData.iterator().next());
                ((PreOrderDepthFirstWalker) currentWalker).setBranchFlag();
            }
        }
        POLimit poLimit = (POLimit) logToPhysMap.get(lm);
        oriLimitMap.put(lm, Long.valueOf(poLimit.getLimit()));
        poLimit.setLimit(inputConstraints.size()-1);
        lm.setLimit(poLimit.getLimit());
    }
View Full Code Here
808
809
810
811
812
813
814
815
816
817
            if (inputConstraints.size() == 1) {
                inputConstraints.add(inputData.iterator().next());
                ((PreOrderDepthFirstWalker) currentWalker).setBranchFlag();
            }
        }
        POLimit poLimit = (POLimit) logToPhysMap.get(lm);
        oriLimitMap.put(lm, Long.valueOf(poLimit.getLimit()));
        poLimit.setLimit(inputConstraints.size()-1);
        lm.setLimit(poLimit.getLimit());
    }
View Full Code Here
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
    }
   
    @Override
    public void visit(LOLimit loLimit) throws FrontendException {
        String scope = DEFAULT_SCOPE;
        POLimit physOp = new POLimit(new OperatorKey(scope,nodeGen.getNextNodeId(scope)), loLimit.getRequestedParallelisam());
        physOp.setLimit(loLimit.getLimit());
        physOp.setAlias(loLimit.getAlias());
        currentPlan.add(physOp);
        physOp.setResultType(DataType.BAG);
        logToPhyMap.put(loLimit, physOp);
        Operator op = loLimit.getPlan().getPredecessors(loLimit).get(0);

        PhysicalOperator from = logToPhyMap.get(op);
        try {
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
206
                op = firstMROp.reducePlan.getSuccessors(op).get(0);
                break;
            }
        }
       
        POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pLimit2.setLimit(firstMROp.limit);
        secondMROp.reducePlan.addAsLeaf(pLimit2);

        while (true) {
            if (op instanceof POStore) break;
            PhysicalOperator opToMove = op;
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
205
206
                op = firstMROp.reducePlan.getSuccessors(op).get(0);
                break;
            }
        }
       
        POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pLimit2.setLimit(firstMROp.limit);
        pLimit2.setLimitPlan(firstMROp.limitPlan);
        secondMROp.reducePlan.addAsLeaf(pLimit2);

        while (true) {
            if (op instanceof POStore) break;
            PhysicalOperator opToMove = op;
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLimit

  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.impl.plan.OperatorKey
  • org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor
  • org.apache.pig.pen.AugmentBaseDataVisitor
  • org.apache.pig.pen.util.ExampleTuple
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.