Examples of POPackage


Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        List<Operator> preds = plan.getPredecessors(cg);
       
        POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), cg.getRequestedParallelisam() );
        poGlobal.setAlias(cg.getAlias());
        POPackage poPackage = new POPackage(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)), cg.getRequestedParallelisam());
        poPackage.setAlias(cg.getAlias());
        currentPlan.add(poGlobal);
        currentPlan.add(poPackage);

        try {
            currentPlan.connect(poGlobal, poPackage);
        } catch (PlanException e1) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
        }

        Byte type = null;
        for( int i = 0 ; i < preds.size(); i++ ) {
            ArrayList<LogicalExpressionPlan> exprPlans =
                (ArrayList<LogicalExpressionPlan>) cg.getExpressionPlans().get(i);
           
            POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                    DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)), cg.getRequestedParallelisam() );
            physOp.setAlias(cg.getAlias());
           
            List<PhysicalPlan> pExprPlans = translateExpressionPlans( cg, exprPlans );
           
            try {
                physOp.setPlans(pExprPlans);
            } catch (PlanException pe) {
                int errCode = 2071;
                String msg = "Problem with setting up local rearrange's plans.";
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, pe);
            }
            try {
                physOp.setIndex(i);
            } catch (ExecException e1) {
                // int errCode = 2058;
                String msg = "Unable to set index on newly create POLocalRearrange.";
                throw new IOException(msg);
            }
            if (exprPlans.size() > 1) {
                type = DataType.TUPLE;
                physOp.setKeyType(type);
            } else {
                type = pExprPlans.get(0).getLeaves().get(0).getResultType();
                physOp.setKeyType(type);
            }
            physOp.setResultType(DataType.TUPLE);

            currentPlan.add(physOp);

            try {
                currentPlan.connect(logToPhyMap.get(preds.get(i)), physOp);
                currentPlan.connect(physOp, poGlobal);
            } catch (PlanException e) {
                int errCode = 2015;
                String msg = "Invalid physical operators in the physical plan" ;
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
            }
        }
       
        poPackage.setKeyType(type);
        poPackage.setResultType(DataType.TUPLE);
        poPackage.setNumInps(preds.size());
        poPackage.setInner(cg.getInner());
        logToPhyMap.put(cg, poPackage);
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        else if (loj.getJoinType() == LOJoin.JOINTYPE.HASH){
            POGlobalRearrange poGlobal = new POGlobalRearrange(new OperatorKey(
                    scope, nodeGen.getNextNodeId(scope)), loj
                    .getRequestedParallelisam());
            poGlobal.setAlias(loj.getAlias());
            POPackage poPackage = new POPackage(new OperatorKey(scope, nodeGen
                    .getNextNodeId(scope)), loj.getRequestedParallelisam());
            poPackage.setAlias(loj.getAlias());
            currentPlan.add(poGlobal);
            currentPlan.add(poPackage);
           
            int count = 0;
            Byte type = null;
           
            try {
                currentPlan.connect(poGlobal, poPackage);
                for (int i=0; i<inputs.size(); i++) {      
                    Operator op = inputs.get(i);
                    List<LogicalExpressionPlan> plans =
                        (List<LogicalExpressionPlan>) loj.getJoinPlan(i);
                    POLocalRearrange physOp = new POLocalRearrange(new OperatorKey(
                            scope, nodeGen.getNextNodeId(scope)), loj
                            .getRequestedParallelisam());
                    List<PhysicalPlan> exprPlans = translateExpressionPlans(loj, plans);
//                    currentPlans.push(currentPlan);
//                    for (LogicalExpressionPlan lp : plans) {
//                        currentPlan = new PhysicalPlan();
//                        PlanWalker childWalker = currentWalker
//                                .spawnChildWalker(lp);
//                        pushWalker(childWalker);
//                        //currentWalker.walk(this);
//                        currentWalker.walk(
//                                new ExpToPhyTranslationVisitor(currentWalker.getPlan(),
//                                        childWalker) );
//                        exprPlans.add(currentPlan);
//                        popWalker();
//
//                    }
//                    currentPlan = currentPlans.pop();
                    try {
                        physOp.setPlans(exprPlans);
                    } catch (PlanException pe) {
                        int errCode = 2071;
                        String msg = "Problem with setting up local rearrange's plans.";
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, pe);
                    }
                    try {
                        physOp.setIndex(count++);
                    } catch (ExecException e1) {
                        int errCode = 2058;
                        String msg = "Unable to set index on newly create POLocalRearrange.";
                        throw new VisitorException(msg, errCode, PigException.BUG, e1);
                    }
                    if (plans.size() > 1) {
                        type = DataType.TUPLE;
                        physOp.setKeyType(type);
                    } else {
                        type = exprPlans.get(0).getLeaves().get(0).getResultType();
                        physOp.setKeyType(type);
                    }
                    physOp.setResultType(DataType.TUPLE);

                    currentPlan.add(physOp);

                    try {
                        currentPlan.connect(logToPhyMap.get(op), physOp);
                        currentPlan.connect(physOp, poGlobal);
                    } catch (PlanException e) {
                        int errCode = 2015;
                        String msg = "Invalid physical operators in the physical plan" ;
                        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
                    }

                }
               
            } catch (PlanException e1) {
                int errCode = 2015;
                String msg = "Invalid physical operators in the physical plan" ;
                throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e1);
            }

            poPackage.setKeyType(type);
            poPackage.setResultType(DataType.TUPLE);
            poPackage.setNumInps(count);
           
            boolean[] innerFlags = loj.getInnerFlags();
            poPackage.setInner(innerFlags);
           
            List<PhysicalPlan> fePlans = new ArrayList<PhysicalPlan>();
            List<Boolean> flattenLst = new ArrayList<Boolean>();
           
            try{
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

                mro.reducePlan.addAsLeaf(lr);
            }
           
            blocking(op);
           
            POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
            pkg.setKeyType(DataType.TUPLE);
            pkg.setDistinct(true);
            pkg.setNumInps(1);
            boolean[] inner = {false};
            pkg.setInner(inner);
            curMROp.reducePlan.add(pkg);
           
            List<PhysicalPlan> eps1 = new ArrayList<PhysicalPlan>();
            List<Boolean> flat1 = new ArrayList<Boolean>();
            PhysicalPlan ep1 = new PhysicalPlan();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

      if(gr.getRequestedParallelism() > curMROp.requestedParallelism)
        curMROp.requestedParallelism = gr.getRequestedParallelism();
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POPakcage
      POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)), rp);
      pkg.setKeyType(type);
      pkg.setResultType(DataType.TUPLE);
      pkg.setNumInps(2);
      boolean [] inner = op.getInnerFlags();
      pkg.setInner(inner);           
      pkg.visit(this);      
      compiledInputs = new MapReduceOper[] {curMROp};
     
      // create POForEach
      List<PhysicalPlan> eps = new ArrayList<PhysicalPlan>();
      List<Boolean> flat = new ArrayList<Boolean>();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        mro.mapPlan.add(lr);
        mro.mapPlan.connect(nfe1, lr);
       
        mro.setMapDone(true);
       
        POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType(DataType.CHARARRAY);
        pkg.setNumInps(1);
        boolean[] inner = {false};
        pkg.setInner(inner);
        mro.reducePlan.add(pkg);
       
        // Lets start building the plan which will have the sort
        // for the foreach
        PhysicalPlan fe2Plan = new PhysicalPlan();
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

            throw new MRCompilerException(msg, errCode, PigException.BUG, e);
        }
       
        mro.mapPlan.addAsLeaf(lr);
       
        POPackage spkg = (POPackage)sortMROp.reducePlan.getRoots().get(0);

        POPackage pkg = null;
        try {
            pkg = spkg.clone();
        } catch (Exception e) {
            int errCode = 2148;
            String msg = "Error cloning POPackageLite for limit after sort";
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

        lr.setPlans(eps);
        lr.setResultType(DataType.TUPLE);
       
        mro.mapPlan.addAsLeaf(lr);
       
        POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
        pkg.setKeyType(DataType.TUPLE);
        pkg.setNumInps(1);
        boolean[] inner = {false};
        pkg.setInner(inner);
        mro.reducePlan.add(pkg);
       
        mro.reducePlan.addAsLeaf(getPlainForEachOP());
    }
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

   
                // On the reduce side of this indexing job, there will be a global rearrange followed by POSort.
                // Output of POSort will be index file dumped on the DFS.
   
                // First add POPackage.
                POPackage pkg = new POPackage(new OperatorKey(scope,nig.getNextNodeId(scope)));
                pkg.setKeyType(DataType.CHARARRAY);
                pkg.setNumInps(1);
                pkg.setInner(new boolean[]{false});
                rightMROpr.reducePlan.add(pkg);
   
                // Next project tuples from the bag created by POPackage.
                POProject topPrj = new POProject(new OperatorKey(scope,nig.getNextNodeId(scope)));
                topPrj.setColumn(1);
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

           
            op = mrRoots.get(0);
            if (!(op instanceof POPackage)) {
                return;
            }
            POPackage pack = (POPackage)op;
           
            List<PhysicalOperator> sucs = mr.reducePlan.getSuccessors(pack);
            if (sucs.size()!=1) {
                return;
            }
           
            op = sucs.get(0);
            boolean lastInputFlattened = true;
            boolean allSimple = true;
            if (op instanceof POForEach)
            {
                POForEach forEach = (POForEach)op;
                List<PhysicalPlan> planList = forEach.getInputPlans();
                List<Boolean> flatten = forEach.getToBeFlattened();
                POProject projOfLastInput = null;
                int i = 0;
                // check all nested foreach plans
                // 1. If it is simple projection
                // 2. If last input is all flattened
                for (PhysicalPlan p:planList)
                {
                    PhysicalOperator opProj = p.getRoots().get(0);
                    if (!(opProj instanceof POProject))
                    {
                        allSimple = false;
                        break;
                    }
                    POProject proj = (POProject)opProj;
                    // the project should just be for one column
                    // from the input
                    if(proj.getColumns().size() != 1) {
                        allSimple = false;
                        break;
                    }
                   
                    try {
                        // if input to project is the last input
                        if (proj.getColumn() == pack.getNumInps())
                        {
                            // if we had already seen another project
                            // which was also for the last input, then
                            // we might be trying to flatten twice on the
                            // last input in which case we can't optimize by
View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage

            // this call modifies the ReplFiles names of POFRJoin operators
            // within the MR plans, must be called before the plans are
            // serialized
            setupDistributedCacheForJoin(mro, pigContext, conf);

            POPackage pack = null;
            if(mro.reducePlan.isEmpty()){
                //MapOnly Job
                nwJob.setMapperClass(PigMapOnly.Map.class);
                nwJob.setNumReduceTasks(0);
                conf.set("pig.mapPlan", ObjectSerializer.serialize(mro.mapPlan));
                if(mro.isEndOfAllInputSetInMap()) {
                    // this is used in Map.close() to decide whether the
                    // pipeline needs to be rerun one more time in the close()
                    // The pipeline is rerun if there either was a stream or POMergeJoin
                    conf.set(END_OF_INP_IN_MAP, "true");
                }
            }
            else{
                //Map Reduce Job
                //Process the POPackage operator and remove it from the reduce plan
                if(!mro.combinePlan.isEmpty()){
                    POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);
                    mro.combinePlan.remove(combPack);
                    nwJob.setCombinerClass(PigCombiner.Combine.class);
                    conf.set("pig.combinePlan", ObjectSerializer.serialize(mro.combinePlan));
                    conf.set("pig.combine.package", ObjectSerializer.serialize(combPack));
                } else if (mro.needsDistinctCombiner()) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.