Package org.apache.pig

Examples of org.apache.pig.FuncSpec

170
171
172
173
174
175
176
177
178
179
180
        return funcSpec;
    }

    @Override
    public POUserComparisonFunc clone() throws CloneNotSupportedException {
        FuncSpec cloneFs = null;
        if (funcSpec != null) {
            cloneFs = funcSpec.clone();
        }
        POUserComparisonFunc clone =
            new POUserComparisonFunc(new OperatorKey(mKey.scope,
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
        String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
        if ((System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")))
            inpDir="/"+FileLocalizer.parseCygPath(inpDir, FileLocalizer.STYLE_WINDOWS);
        // copy passwd file to cluster and set that as the input location for the load
        Util.copyFromLocalToCluster(cluster, inpDir + "passwd", "passwd");
        FileSpec inpFSpec = new FileSpec("passwd", new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
        POLoad ld = GenPhyOp.topLoadOp();
        ld.setLFile(inpFSpec);
        ld.setPc(pc);
       
        DataBag inpDB = DefaultBagFactory.getInstance().newDefaultBag();
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
       
        File tempF = File.createTempFile("tmp", "");
        String fileName = tempF.getCanonicalPath();
        tempF.delete(); // we only needed the temp file name, so delete the file
        PigFile f = new PigFile(fileName);
        f.store(b, new FuncSpec(BinStorage.class.getCanonicalName()), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
613
614
615
616
617
618
619
620
621
622
623
    inputs.add(read);
    String funcName = WeirdComparator.class.getName() + "()";
    /*POUserFunc comparator = new POUserFunc(
        new OperatorKey("", r.nextLong()), -1, inputs, funcName);*/
    POUserComparisonFunc comparator = new POUserComparisonFunc(
        new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
    POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
        null, null, comparator);
    Tuple t = null;
    Result res1 = sort.getNext(t);
    // System.out.println(res1.result);
View Full Code Here
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        // You wouldn't be able to make two calls to this function on the same
        // algebraic EvalFunc as
        // func is being changed.
        switch (Function) {
        case INITIAL:
            funcSpec = new FuncSpec(getInitial());
            break;
        case INTERMEDIATE:
            funcSpec = new FuncSpec(getIntermed());
            break;
        case FINAL:
            funcSpec = new FuncSpec(getFinal());
            break;
        }
        funcSpec.setCtorArgs(origFSpec.getCtorArgs());
        instantiateFunc(funcSpec);
        setResultType(DataType.findType(((EvalFunc<?>) func).getReturnType()));
View Full Code Here
434
435
436
437
438
439
440
441
442
443
444
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  ConfigurationUtil.toConfiguration(new Properties())) ;

        // set schemas
        load1.setEnforcedSchema(null) ;
View Full Code Here
498
499
500
501
502
503
504
505
506
507
508
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  ConfigurationUtil.toConfiguration(new Properties())) ;

        // set schemas
        load1.setEnforcedSchema(null) ;
View Full Code Here
728
729
730
731
732
733
734
735
736
737
738
        php.merge(ldFil1);
       
        // set up order by *
        String funcName = WeirdComparator.class.getName();
        POUserComparisonFunc comparator = new POUserComparisonFunc(
                new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
        POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, ldFil1.getLeaves(),
                null, new ArrayList<Boolean>(), comparator);
        sort.setRequestedParallelism(20);
        PhysicalPlan nesSortPlan = new PhysicalPlan();
        POProject topPrj = new POProject(new OperatorKey("", r.nextLong()));
View Full Code Here
988
989
990
991
992
993
994
        planTester.typeCheckPlan(lp);
       
        PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
        MROperPlan mp = Util.buildMRPlan(pp, pc);
        MapReduceOper op = mp.getLeaves().get(0);
        assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName())+"('\t')"));
    }
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
            new org.apache.pig.newplan.logical.relational.LogicalPlan();
       
        LogicalSchema aschema = new LogicalSchema();     
        aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
        aschema.addField(new LogicalSchema.LogicalFieldSchema("d", null, DataType.BYTEARRAY));
        LOLoad load = new LOLoad(new FileSpec("/test/d.txt", new FuncSpec("org.apache.pig.builtin.PigStorage")), aschema, expected, null);
        expected.add(load);
       
        LOForEach foreach = new LOForEach(expected);
        org.apache.pig.newplan.logical.relational.LogicalPlan innerPlan = new org.apache.pig.newplan.logical.relational.LogicalPlan();
        LOInnerLoad l1 = new LOInnerLoad(innerPlan, foreach, 0);
        innerPlan.add(l1);
        LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach, 1);
        innerPlan.add(l2);
       
        List<LogicalExpressionPlan> eps = new ArrayList<LogicalExpressionPlan>();
        LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
        LogicalExpressionPlan p1 = new LogicalExpressionPlan();
        p1.add(new ProjectExpression(p1, 0, -1, gen));
        LogicalExpressionPlan p2 = new LogicalExpressionPlan();
        p2.add(new ProjectExpression(p2, 1, -1, gen));
        eps.add(p1);
        eps.add(p2);
       
       
        innerPlan.add(gen);
        innerPlan.connect(l1, gen);
        innerPlan.connect(l2, gen);
       
        foreach.setInnerPlan(innerPlan);     
        expected.add(foreach);
       
        LOStore s = new LOStore(expected, new FileSpec("/test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
     
        expected.add(s);
       
        expected.connect(load, foreach);
        expected.connect(foreach, s);
View Full Code Here
TOP

Related Classes of org.apache.pig.FuncSpec

  • org.apache.oozie.action.hadoop.UDFTester
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.Launcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.