Examples of org.apache.pig.LoadFunc

org.apache.pig.LoadFunc
This interface is used to implement functions to parse records from a dataset. This also includes functions to cast raw byte data into various datatypes. These are external functions because we want loaders, whenever possible, to delay casting of datatypes until the last possible moment (i.e. don't do it on load). This means we need to expose the functionality so that other sections of the code can call back to the loader to do the cast.

      t.append(r.nextDouble());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

  }
  
  @Test
  public void testStringToOther() throws IOException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

  }
  
  @Test
  public void testByteArrayToOther() throws IOException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      assertEquals(null, res.result);
    }
  }
  
  private PhysicalPlan constructPlan(POCast op) throws IOException {
        LoadFunc load = new TestLoader();
        op.setFuncSpec(new FuncSpec(load.getClass().getName()));
        POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
        PhysicalPlan plan = new PhysicalPlan();
        plan.add(prj);
        plan.add(op);
        plan.connect(prj, op);

View Full Code Here

        String query = "a = load 'testSFPig-input.txt';" +
            "store a into 'testSFPig-output.txt';";
        pigServer.setBatchOn();
        Util.registerMultiLineQuery(pigServer, query);
        pigServer.executeBatch();
        LoadFunc lfunc = new ReadToEndLoader(new PigStorage(), ConfigurationUtil.
                toConfiguration(cluster.getProperties()), "testSFPig-output.txt", 0); 
        Tuple f2 = lfunc.getNext();
        
        assertEquals(f1, f2);
        Util.deleteFile(cluster, "testSFPig-input.txt");
        Util.deleteFile(cluster, "testSFPig-output.txt");
    }

View Full Code Here

      t.append(r.nextInt());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextLong());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextFloat());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

      t.append(r.nextDouble());
      bag.add(t);
    }
    
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

  }
  
  @Test
  public void testStringToOther() throws IOException {
    POCast op = new POCast(new OperatorKey("", r.nextLong()), -1);
    LoadFunc load = new TestLoader();
    op.setFuncSpec(new FuncSpec(load.getClass().getName()));
    POProject prj = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
    PhysicalPlan plan = new PhysicalPlan();
    plan.add(prj);
    plan.add(op);
    plan.connect(prj, op);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.LoadFunc

com.netflix.lipstick.util.OutputSampler

com.twitter.elephantbird.pig.load.MultiFormatLoader

com.twitter.elephantbird.pig.load.TestLocationAsTuple

org.apache.hadoop.fs.Path

org.apache.hadoop.mapreduce.Counter

org.apache.pig.backend.hadoop.executionengine.HJob

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.