Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOStore

161
162
163
164
165
166
167
168
169
170
171
            "B = group A by *;" +
            "store B into '111';";
       
        LogicalPlan lp = Util.parseAndPreprocess(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        LOCogroup loCoGroup = (LOCogroup)lp.getPredecessors(loStore).get(0);
        LogicalFieldSchema groupFieldSchema = loCoGroup.getSchema().getField(0);
        Assert.assertTrue(groupFieldSchema.type==DataType.TUPLE);
        Assert.assertTrue(groupFieldSchema.schema==null);
    }
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
187
            "B = filter A by a0 > 1;" +
            "store B into 'output';";
       
        LogicalPlan lp = Util.parse(query, pc);
        Util.optimizeNewLP(lp);
        LOStore loStore = (LOStore)lp.getSinks().get(0);
        assert(loStore.getAlias().equals("B"));
       
        PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
        POStore poStore = (POStore)pp.getLeaves().get(0);
        assert(poStore.getAlias().equals("B"));
       
View Full Code Here
245
246
247
248
249
250
251
252
253
        LogicalPlan lp = Util.parse(query, pc);
        Util.optimizeNewLP(lp);
       
        LOLoad loLoad = (LOLoad)lp.getSources().get(0);
        LOFilter loFilter = (LOFilter)lp.getSuccessors(loLoad).get(0);
        LOStore loStore = (LOStore)lp.getSuccessors(loFilter).get(0);
       
        Assert.assertTrue(lp.getSuccessors(loStore)==null);
    }
View Full Code Here
304
305
306
307
308
309
310
311
312
           
        LogicalPlan lp = Util.parse(query, pc);
        Util.optimizeNewLP(lp);
       
        LOLoad loLoad = (LOLoad)lp.getSources().get(0);
        LOStore loStore = (LOStore)lp.getSuccessors(loLoad).get(0);
        Assert.assertTrue(((PartitionedLoader)loLoad.getLoadFunc()).getPartFilter()!=null);
        Assert.assertTrue(loStore.getAlias().equals("b"));
    }
View Full Code Here
796
797
798
799
800
801
802
803
804
805
806
807
808
        Assert.assertTrue(lp.size()>1);
        Operator op = lp.getSinks().get(0);
       
        Assert.assertTrue(op instanceof LOStore);
        LOStore store = (LOStore)op;

        String p = store.getFileSpec().getFileName();
        p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
       
        if (isTmp) {
            Assert.assertTrue(p.matches("/tmp.*"));
        } else {
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad( filespec1, null, plan,
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;      
        LOStore store = new LOStore(plan, filespec2) ;
       
        plan.add(load) ;
        plan.add(store) ;
       
        plan.connect(load, store) ;    
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
            + "D = foreach A generate owner, age/(double)C.total AS percentAge; "
            + "F = LIMIT D C.total/8;"
            + "store F into 'output';";
        LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);;
        optimizePlan(newLogicalPlan);
        LOStore store = (LOStore)newLogicalPlan.getSinks().get(0);
        LOForEach foreach1 = (LOForEach)newLogicalPlan.getPredecessors(store).get(0);
        LOForEach foreach2 = (LOForEach)newLogicalPlan.getPredecessors(foreach1).get(0);
        LOLimit limit = (LOLimit)newLogicalPlan.getPredecessors(foreach2).get(0);
        Assert.assertTrue(newLogicalPlan.getSoftLinkPredecessors(limit).get(0) instanceof LOStore);
    }
View Full Code Here
123
124
125
126
127
128
129
130
131
132
133
       
       
        f1.setAlias("D");
        plan.add(f1);
       
        LogicalRelationalOperator s1 = new LOStore(plan);
        plan.add(s1);      
       
        // load --|-join - filter - store
        // load --|  
        plan.connect(l1, j1);
View Full Code Here
432
433
434
435
436
437
438
439
        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }
       
        FileSpec fileSpec = new FileSpec( absPath, funcSpec );
        LOStore op = new LOStore( plan, fileSpec );
        return buildOp( loc, op, alias, inputAlias, null );
    }
View Full Code Here
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
            long outputUid = scalarExp.getFieldSchema().uid;
            boolean foundInput = false; // a variable to do sanity check on num of input relations

            //find the input relation, and use it to get lineage
            for(Operator softPred : softPreds){
                LOStore inputStore = (LOStore) softPred;
                if(inputStore.getFileSpec().getFileName().equals(inputFile)){
                   
                    if(foundInput == true){
                        throw new FrontendException(
                                "More than one input found for scalar expression",
                                2268,
                                PigException.BUG
                        );
                    }
                    foundInput = true;
                   
                    //found the store corresponding to this scalar expression
                    LogicalSchema sch = inputStore.getSchema();
                    if(sch == null){
                        //see if there is a load function associated with the store
                        FuncSpec funcSpec = rel2InputFuncMap.get(inputStore);
                        addUidLoadFuncToMap(outputUid, funcSpec);
                    }else{
View Full Code Here
TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOStore

  • org.apache.pig.newplan.logical.visitor.LineageFindRelVisitor$LineageFindExpVisitor
  • org.apache.pig.newplan.logical.visitor.ScalarVisitor
  • org.apache.pig.parser.LogicalPlanBuilder
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.penny.ParsedPigScript
  • org.apache.pig.test.TestBlackAndWhitelistValidator
  • org.apache.pig.test.TestCollectedGroup
  • org.apache.pig.test.TestInputOutputFileValidator
  • org.apache.pig.test.TestInputOutputMiniClusterFileValidator
  • org.apache.pig.test.TestLogicalPlanMigrationVisitor
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.