Examples of LOForEach

Examples of org.apache.pig.impl.logicalLayer.LOForEach


    @Override
    public void transform(List<LogicalOperator> nodes)
            throws OptimizerException {
        try {
            LOForEach foreach = (LOForEach) getOperator(nodes);
            LogicalOperator successor = mPlan.getSuccessors(foreach).get(0);
            if (mSwap) {
                mPlan.swap(successor, foreach);
            } else if (mInsertBetween) {
                // mark the flattened columns as not flattened in the foreach
                // create a new foreach operator that projects each column of the
                // successor. Mark the remapped flattened columns as flattened
                // in the new foreach operator
                
                if(mFlattenedColumnReMap == null) {
                    int errCode = 2153;
                    String msg = "Internal error. The mapping for the flattened columns is empty";
                    throw new OptimizerException(msg, errCode, PigException.BUG);
                }
                
                // set flatten to false for all columns in the mapping
                
                ArrayList<Boolean> flattenList = (ArrayList<Boolean>)foreach.getFlatten();                
                for(Integer key: mFlattenedColumnReMap.keySet()) {
                    flattenList.set(key, false);
                }
                
                // rebuild schemas of the foreach and the successor after the foreach modification
                foreach.regenerateSchema();
                successor.regenerateSchema();
                
                Schema successorSchema = successor.getSchema();
                
                if(successorSchema == null) {
                    int errCode = 2154;
                    String msg = "Internal error. Schema of successor cannot be null for pushing down foreach with flatten.";
                    throw new OptimizerException(msg, errCode, PigException.BUG);
                }
                
                flattenList = new ArrayList<Boolean>();
                
                ArrayList<LogicalPlan> foreachInnerPlans = new ArrayList<LogicalPlan>();
                
                for(int i = 0; i < successorSchema.size(); ++i) {
                    LogicalPlan innerPlan = new LogicalPlan();
                    LOProject project = new LOProject(innerPlan, OperatorKey
                            .genOpKey(foreach.getOperatorKey().scope),
                            successor, i);
                    innerPlan.add(project);
                    foreachInnerPlans.add(innerPlan);
                    flattenList.add(false);
                }
                
                // set the flattened remapped column to true
                for(Integer key: mFlattenedColumnReMap.keySet()) {
                    Integer value = mFlattenedColumnReMap.get(key);
                    flattenList.set(value, true);
                }            
                
                
                LOForEach newForeach = new LOForEach(mPlan, OperatorKey
                        .genOpKey(foreach.getOperatorKey().scope), foreachInnerPlans,
                        flattenList);
                
                // add the new foreach to the plan
                mPlan.add(newForeach);

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOForEach

                mPushBefore = true;
                return true;


            } else if (predecessor instanceof LOForEach) {


                LOForEach loForEach = (LOForEach) predecessor;
                List<Boolean> mFlatten = loForEach.getFlatten();
                boolean hasFlatten = false;
                for (Boolean b : mFlatten) {
                    if (b.equals(true)) {
                        hasFlatten = true;
                    }
                }


                // TODO
                // A better check is to examine each column in the filter's
                // required fields. If the column is the result of a flatten
                // then
                // return false else return true


                // for now if the foreach has a flatten then return false
                if (hasFlatten) {
                    return false;
                }


                Pair<Boolean, Set<Integer>> mappingResult = isRequiredFieldMapped(requiredField, predecessor.getProjectionMap());
                boolean mapped = mappingResult.first;
                
                // Check if it is a direct mapping, that is, project optionally followed by cast, so if project->project, it is not
                // considered as a mapping
                for (Pair<Integer, Integer> pair : requiredField.getFields())
                {
                    if (!isFieldSimple(loForEach.getForEachPlans().get(pair.second)))
                    {
                        mapped = false;
                        break;
                    }
                }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

                // add another LOForEach
                if (next instanceof LOForEach) {
                    return;
                }
                
                LOForEach foreach = new LOForEach(load.getPlan());
                
                // add foreach to the base plan                       
                p.add(foreach);
                               
                p.insertBetween(load, foreach, next);
                
                LogicalPlan innerPlan = new LogicalPlan();
                foreach.setInnerPlan(innerPlan);
                
                // build foreach inner plan
                List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();              
                LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[requiredFields.getFields().size()]);
                innerPlan.add(gen);

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

                    if (!branchOutputUids.contains(split.getSchema().getField(j).uid))
                        columnsToDrop.add(j);
                }
                
                if (!columnsToDrop.isEmpty()) {
                    LOForEach foreach = Util.addForEachAfter((LogicalPlan)split.getPlan(), split, i, columnsToDrop);
                    foreach.getSchema();
                }
            }
        }
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

                if (!outputUids.contains(schema.getField(i).uid))
                    columnsToDrop.add(i);
            }
            
            if (!columnsToDrop.isEmpty()) {
                LOForEach foreach = Util.addForEachAfter((LogicalPlan)op.getPlan(), op, 0, columnsToDrop);
                foreach.getSchema();
            }
        }
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

            // For every field, build a logical plan.  If the field has a type
            // other than byte array, then the plan will be cast(project).  Else
            // it will just be project.
            LogicalPlan innerPlan = new LogicalPlan();
            
            LOForEach foreach = new LOForEach(currentPlan);
            foreach.setInnerPlan(innerPlan);
            foreach.setAlias(op.getAlias());
            
            // Insert the foreach into the plan and patch up the plan.
            Operator next = currentPlan.getSuccessors(op).get(0);
            currentPlan.insertBetween(op, foreach, next);

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach


    @Override
    protected OperatorPlan buildPattern() {
        // match each foreach.
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator foreach1 = new LOForEach(plan);
        plan.add( foreach1 );
        return plan;
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

    }
    
    public class MergeForEachTransformer extends Transformer {
        @Override
        public boolean check(OperatorPlan matched) throws FrontendException {
            LOForEach foreach1 = (LOForEach)matched.getSources().get(0);
            List<Operator> succs = currentPlan.getSuccessors( foreach1 );
            if( succs == null || succs.size() != 1 || !( succs.get(0) instanceof LOForEach) )
                return false;
            
            LOForEach foreach2 = (LOForEach)succs.get(0);
            
            // Check if the second foreach has only LOGenerate and LOInnerLoad
            Iterator<Operator> it = foreach2.getInnerPlan().getOperators();
            while( it.hasNext() ) {
                Operator op = it.next();
                if(!(op instanceof LOGenerate) && !(op instanceof LOInnerLoad))
                    return false;
            }
            
            // Check if the first foreach has flatten in its generate statement.
            LOGenerate gen1 = (LOGenerate)foreach1.getInnerPlan().getSinks().get(0);
            for (boolean flatten : gen1.getFlattenFlags()) {
                if( flatten )
                    return false;
            }
            if (gen1.getUserDefinedSchema()!=null) {
                for (LogicalSchema s : gen1.getUserDefinedSchema()) {
                    if (s!=null) {
                        return false;
                    }
                }
            }
            
            // Check if non of the 1st foreach output is referred more than once in second foreach.
            // Otherwise, we may do expression calculation more than once, defeat the benefit of this
            // optimization
            Set<Integer> inputs = new HashSet<Integer>();
            boolean duplicateInputs = false;
            for (Operator op : foreach2.getInnerPlan().getSources()) {
                // If the source is not LOInnerLoad, then it must be LOGenerate. This happens when 
                // the 1st ForEach does not rely on any input of 2nd ForEach
                if (op instanceof LOInnerLoad) {
                    LOInnerLoad innerLoad = (LOInnerLoad)op;
                    int input = innerLoad.getProjection().getColNum();

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

        
        @Override
        public void transform(OperatorPlan matched) throws FrontendException {
            subPlan = new OperatorSubPlan(currentPlan);
            
            LOForEach foreach1 = (LOForEach)matched.getSources().get(0);
            LOGenerate gen1 = (LOGenerate)foreach1.getInnerPlan().getSinks().get(0);
            
            LOForEach foreach2 = (LOForEach)currentPlan.getSuccessors(foreach1).get(0);
            LOGenerate gen2 = (LOGenerate)foreach2.getInnerPlan().getSinks().get(0);
            
            LOForEach newForEach = new LOForEach(currentPlan);
            LogicalPlan newForEachInnerPlan = new LogicalPlan();
            newForEach.setInnerPlan(newForEachInnerPlan);
            newForEach.setAlias(foreach2.getAlias());
            newForEach.setRequestedParallelism(foreach1.getRequestedParallelisam());
            List<LogicalExpressionPlan> newExpList = new ArrayList<LogicalExpressionPlan>();
            LOGenerate newGen = new LOGenerate(newForEachInnerPlan, newExpList, gen2.getFlattenFlags());
            newGen.setUserDefinedSchema(gen2.getUserDefinedSchema());
            newForEachInnerPlan.add(newGen);
            
            for (LogicalExpressionPlan exp2 : gen2.getOutputPlans()) {
                LogicalExpressionPlan newExpPlan = new LogicalExpressionPlan();
                LogicalExpressionPlan exp2Copy = exp2.deepCopy();
                newExpPlan.merge(exp2Copy);
                
                // Add expression plan in 2nd ForEach
                List<Operator> exp2Sinks = new ArrayList<Operator>();
                exp2Sinks.addAll(newExpPlan.getSinks());
                for (Operator exp2Sink : exp2Sinks) {
                    if (exp2Sink instanceof ProjectExpression) {
                        // Find referred expression plan in 1st ForEach
                        ProjectExpression proj = (ProjectExpression)exp2Sink;
                        LOInnerLoad innerLoad = (LOInnerLoad)foreach2.getInnerPlan().getPredecessors(gen2).get(proj.getInputNum());
                        int exp1Pos = innerLoad.getProjection().getColNum();
                        LogicalExpressionPlan exp1 = gen1.getOutputPlans().get(exp1Pos);
                        LogicalExpressionPlan exp1Copy = exp1.deepCopy();
                        List<Operator> exp1Sources = newExpPlan.merge(exp1Copy);
                        
                        // Copy expression plan to the new ForEach, connect to the expression plan of 2nd ForEach
                        Operator exp1Source = exp1Sources.get(0);
                        if (newExpPlan.getPredecessors(exp2Sink)!=null) {
                            Operator exp2NextToSink = newExpPlan.getPredecessors(exp2Sink).get(0);
                            Pair<Integer, Integer> pos = newExpPlan.disconnect(exp2NextToSink, exp2Sink);
                            newExpPlan.remove(exp2Sink);
                            newExpPlan.connect(exp2NextToSink, pos.first, exp1Source, 0);
                        }
                        else {
                            newExpPlan.remove(exp2Sink);
                        }
                    }
                }
                
                // Copy referred ForEach1 inner plan to new ForEach
                List<Operator> exp1Sinks = newExpPlan.getSinks();
                for (Operator exp1Sink : exp1Sinks) {
                    if (exp1Sink instanceof ProjectExpression) {
                        Operator opNextToGen = addBranchToPlan(gen1, ((ProjectExpression)exp1Sink).getInputNum(), newForEachInnerPlan, newForEach);
                        newForEachInnerPlan.connect(opNextToGen, newGen);
                        int input = newForEachInnerPlan.getPredecessors(newGen).indexOf(opNextToGen);
                        ((ProjectExpression)exp1Sink).setInputNum(input);
                    }
                }
                
                newExpList.add(newExpPlan);
            }
            
            // Adjust attachedOp
            for (LogicalExpressionPlan p : newGen.getOutputPlans()) {
                Iterator<Operator> iter = p.getOperators();
                while (iter.hasNext()) {
                    Operator op = iter.next();
                    if (op instanceof ProjectExpression) {
                        ((ProjectExpression)op).setAttachedRelationalOp(newGen);
                    }
                }
            }
            
            Iterator<Operator> iter = newForEach.getInnerPlan().getOperators();
            while (iter.hasNext()) {
                Operator op = iter.next();
                if (op instanceof LOInnerLoad) {
                    ((LOInnerLoad)op).getProjection().setAttachedRelationalOp(newForEach);
                }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOForEach

                return false;
            }


            // Limit cannot be pushed in front of ForEach if it has a flatten
            if (pred instanceof LOForEach) {
                LOForEach foreach = (LOForEach) pred;
                LogicalPlan innerPlan = foreach.getInnerPlan();
                Iterator<Operator> it = innerPlan.getOperators();
                while (it.hasNext()) {
                    Operator op = it.next();
                    if (op instanceof LOGenerate) {
                        LOGenerate gen = (LOGenerate) op;

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.