Examples of LOLimit


Examples of org.apache.pig.impl.logicalLayer.LOLimit

                int errCode = 2005;
                String msg = "Expected " + LOLimit.class.getSimpleName() + ", got " + (lo == null ? lo : lo.getClass().getSimpleName());
                throw new OptimizerException(msg, errCode, PigException.BUG);
            }

            LOLimit limit = (LOLimit)lo;
           
            processNode(limit);
        } catch (OptimizerException oe) {
            throw oe;
        } catch (Exception e) {
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLimit

            }
            // Limit can be duplicated, and the new instance pushed in front of an operator for the following operators
            // (that is, if you have X->limit, you can transform that to limit->X->limit):
            else if (predecessor instanceof LOCross || predecessor instanceof LOUnion)
            {
              LOLimit newLimit = null;
              List<LogicalOperator> nodesToProcess = new ArrayList<LogicalOperator>();
              for (LogicalOperator prepredecessor:mPlan.getPredecessors(predecessor))
                nodesToProcess.add(prepredecessor);
              for (LogicalOperator prepredecessor:nodesToProcess)
              {
                try {
                  newLimit = limit.duplicate();
                  insertBetween(prepredecessor, newLimit, predecessor, null);
                } catch (Exception e) {
                    int errCode = 2011;
                    String msg = "Can not insert LOLimit clone";
                  throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
                // we can move the new LOLimit even further, recursively optimize LOLimit
                processNode(newLimit);
              }
            }
            // Limit can be merged into LOSort, result a "limited sort"
            else if (predecessor instanceof LOSort)
            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        mPlan.removeAndReconnect(limit);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                mPlan.removeAndReconnect(limit);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLimit

    public void testQueryLimitNoSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a';");
        planTester.buildPlan("b = order a by $1;");
        LogicalPlan lp = planTester.buildPlan("c = limit b 10;");
       
        LOLimit limit = (LOLimit)lp.getLeaves().get(0);
        List<RequiredFields> limitRelevantFields0 = limit.getRelevantInputs(0, 0);
        assertTrue(limitRelevantFields0.get(0).getFields().size()==1);
        assertTrue(limitRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(limitRelevantFields0.get(0).getNeedNoFields()==false);
        assertTrue(limitRelevantFields0.get(0).getFields().get(0).first == 0);
        assertTrue(limitRelevantFields0.get(0).getFields().get(0).second == 0);
       
        List<RequiredFields> limitRelevantFields1 = limit.getRelevantInputs(0, 1);
        assertTrue(limitRelevantFields1.get(0).getFields().size()==1);
        assertTrue(limitRelevantFields1.get(0).getNeedAllFields()==false);
        assertTrue(limitRelevantFields1.get(0).getNeedNoFields()==false);
        assertTrue(limitRelevantFields1.get(0).getFields().get(0).first == 0);
        assertTrue(limitRelevantFields1.get(0).getFields().get(0).second == 1);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLimit

    public void testQueryLimitWithSchema() throws FrontendException {
        planTester.buildPlan("a = load 'a' as (url,hitCount);");
        planTester.buildPlan("b = order a by $1;");
        LogicalPlan lp = planTester.buildPlan("c = limit b 10;");
       
        LOLimit limit = (LOLimit)lp.getLeaves().get(0);
        List<RequiredFields> limitRelevantFields0 = limit.getRelevantInputs(0, 0);
        assertTrue(limitRelevantFields0.get(0).getFields().size()==1);
        assertTrue(limitRelevantFields0.get(0).getNeedAllFields()==false);
        assertTrue(limitRelevantFields0.get(0).getNeedNoFields()==false);
        assertTrue(limitRelevantFields0.get(0).getFields().get(0).first == 0);
        assertTrue(limitRelevantFields0.get(0).getFields().get(0).second == 0);
       
        List<RequiredFields> limitRelevantFields1 = limit.getRelevantInputs(0, 1);
        assertTrue(limitRelevantFields1.get(0).getFields().size()==1);
        assertTrue(limitRelevantFields1.get(0).getNeedAllFields()==false);
        assertTrue(limitRelevantFields1.get(0).getNeedNoFields()==false);
        assertTrue(limitRelevantFields1.get(0).getFields().get(0).first == 0);
        assertTrue(limitRelevantFields1.get(0).getFields().get(0).second == 1);
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLimit

                int errCode = 2005;
                String msg = "Expected " + LOLimit.class.getSimpleName() + ", got " + lo.getClass().getSimpleName();
                throw new OptimizerException(msg, errCode, PigException.BUG);
            }

            LOLimit limit = (LOLimit)lo;
           
            processNode(limit);
        } catch (OptimizerException oe) {
            throw oe;
        } catch (Exception e) {
View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOLimit

            }
            // Limit can be duplicated, and the new instance pushed in front of an operator for the following operators
            // (that is, if you have X->limit, you can transform that to limit->X->limit):
            else if (predecessor instanceof LOCross || predecessor instanceof LOUnion)
            {
              LOLimit newLimit = null;
              List<LogicalOperator> nodesToProcess = new ArrayList<LogicalOperator>();
              for (LogicalOperator prepredecessor:mPlan.getPredecessors(predecessor))
                nodesToProcess.add(prepredecessor);
              for (LogicalOperator prepredecessor:nodesToProcess)
              {
                try {
                  newLimit = (LOLimit)limit.duplicate();
                  insertBetween(prepredecessor, newLimit, predecessor, null);
                } catch (Exception e) {
                    int errCode = 2011;
                    String msg = "Can not insert LOLimit clone";
                  throw new OptimizerException(msg, errCode, PigException.BUG, e);
                }
                // we can move the new LOLimit even further, recursively optimize LOLimit
                processNode(newLimit);
              }
            }
            // Limit can be merged into LOSort, result a "limited sort"
            else if (predecessor instanceof LOSort)
            {
                if(mode == ExecType.LOCAL) {
                    //We don't need this optimisation to happen in the local mode.
                    //so we do nothing here.
                } else {
                    LOSort sort = (LOSort)predecessor;
                    if (sort.getLimit()==-1)
                        sort.setLimit(limit.getLimit());
                    else
                        sort.setLimit(sort.getLimit()<limit.getLimit()?sort.getLimit():limit.getLimit());
                    try {
                        removeFromChain(limit, null);
                    } catch (Exception e) {
                        int errCode = 2012;
                        String msg = "Can not remove LOLimit after LOSort";
                        throw new OptimizerException(msg, errCode, PigException.BUG, e);
                    }
                }
            }
            // Limit is merged into another LOLimit
            else if (predecessor instanceof LOLimit)
            {
              LOLimit beforeLimit = (LOLimit)predecessor;
              beforeLimit.setLimit(beforeLimit.getLimit()<limit.getLimit()?beforeLimit.getLimit():limit.getLimit());
              try {
                removeFromChain(limit, null);
              } catch (Exception e) {
                  int errCode = 2012;
                  String msg = "Can not remove LOLimit after LOLimit";
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLimit

    }

    @Override
    protected OperatorPlan buildPattern() {
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator limit = new LOLimit(plan, 0);
        plan.add(limit);
        return plan;
    }
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLimit

    public class OptimizeLimitTransformer extends Transformer {

        @Override
        public boolean check(OperatorPlan matched) {
            LOLimit limit = (LOLimit) matched.getSources().get(0);
            // Match each foreach.
            List<Operator> preds = currentPlan.getPredecessors(limit);
            if (preds == null || preds.size() == 0)
                return false;
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLimit

        }

        @Override
        public void transform(OperatorPlan matched) throws FrontendException {

            LOLimit limit = (LOLimit) matched.getSources().get(0);

            // Find the next foreach operator.
            List<Operator> preds = currentPlan.getPredecessors(limit);
            Operator pred = preds.get(0);

            if (pred instanceof LOForEach) {
                // We can safely move LOLimit up
                // Get operator before LOForEach
                Operator prepredecessor = currentPlan.getPredecessors(pred)
                    .get(0);
                currentPlan.removeAndReconnect(limit);
                currentPlan.insertBetween(prepredecessor, limit, pred);
            } else if (pred instanceof LOCross || pred instanceof LOUnion) {
                // Limit can be duplicated, and the new instance pushed in front
                // of an operator for the following operators
                // (that is, if you have X->limit, you can transform that to
                // limit->X->limit):
                LOLimit newLimit = null;
                List<Operator> nodesToProcess = new ArrayList<Operator>();
                for (Operator prepredecessor : currentPlan
                        .getPredecessors(pred))
                    nodesToProcess.add(prepredecessor);
                for (Operator prepredecessor : nodesToProcess) {
                    if (prepredecessor instanceof LOLimit) {
                        LOLimit l = (LOLimit) prepredecessor;
                        l.setLimit(l.getLimit() < limit.getLimit() ? l
                                .getLimit() : limit.getLimit());
                    } else {
                        newLimit = new LOLimit((LogicalPlan) currentPlan, limit
                                .getLimit());
                        currentPlan.insertBetween(prepredecessor, newLimit, pred);
                    }
                }
            } else if (pred instanceof LOSort) {
                LOSort sort = (LOSort) pred;
                if (sort.getLimit() == -1)
                    sort.setLimit(limit.getLimit());
                else
                    sort.setLimit(sort.getLimit() < limit.getLimit() ? sort
                            .getLimit() : limit.getLimit());

                // remove the limit
                currentPlan.removeAndReconnect(limit);
            } else if (pred instanceof LOLimit) {
                // Limit is merged into another LOLimit
                LOLimit beforeLimit = (LOLimit) pred;
                beforeLimit
                        .setLimit(beforeLimit.getLimit() < limit.getLimit() ? beforeLimit
                                .getLimit()
                                : limit.getLimit());
                // remove the limit
                currentPlan.removeAndReconnect(limit);
            } else if (pred instanceof LOSplitOutput) {
View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOLimit

        LODistinct op = new LODistinct( plan );
        return buildOp( loc, op, alias, inputAlias, partitioner );
    }

    String buildLimitOp(SourceLocation loc, String alias, String inputAlias, long limit) {
        LOLimit op = new LOLimit( plan, limit );
        return buildOp( loc, op, alias, inputAlias, null );
    }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.