Package org.apache.pig.impl

Examples of org.apache.pig.impl.PigContext.connect()

168
169
170
171
172
173
174
175
176
177
178
        zipArchives.add(textFile);
        final List<File> tarArchives = createFiles(".tgz", ".tar.gz", ".tar");

        final PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
        final PigContext pigContext = pigServer.getPigContext();
        pigContext.connect();
        pigContext.getProperties().put("pig.streaming.ship.files",
                StringUtils.join(zipArchives, ","));
        pigContext.getProperties().put("pig.streaming.cache.files",
                StringUtils.join(tarArchives, ","));
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
      public void testNoDuplicateJarsInDistributedCache() throws Exception {

          // JobControlCompiler setup
          final PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
          PigContext pigContext = pigServer.getPigContext();
          pigContext.connect();

          Configuration conf = new Configuration();
          DistributedCache.addFileToClassPath(new Path(new URI("/lib/udf-0.jar#udf.jar")), conf, FileSystem.get(conf));
          DistributedCache.addFileToClassPath(new Path(new URI("/lib/udf1.jar#diffname.jar")), conf, FileSystem.get(conf));
          DistributedCache.addFileToClassPath(new Path(new URI("/lib/udf2.jar")), conf, FileSystem.get(conf));
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
     */
    @Test
    public void testGroupNonConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(cluster.getExecType(), cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        PigServer pigServer = new PigServer( cluster.getExecType(), cluster.getProperties() );
        String query =  "a = load '" + INPUT_FILE + "';\n" + "b = group a by $0;" + "store b into 'output';";
       
        PhysicalPlan pp = Util.buildPp( pigServer, query );
View Full Code Here
95
96
97
98
99
100
101
102
103
104
105
     */
    @Test
    public void testGroupConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(cluster.getExecType(), cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        String query = "a = load '" + INPUT_FILE + "';\n" + "b = group a by 1;" + "store b into 'output';";
        PigServer pigServer = new PigServer( cluster.getExecType(), cluster.getProperties() );
        PhysicalPlan pp = Util.buildPp( pigServer, query );

View Full Code Here
730
731
732
733
734
735
736
737
738
739
740
        PrintWriter out = new PrintWriter(new FileWriter(propertyFile));
        out.println("pig.temp.dir=/opt/temp");
        out.close();
        Properties properties = PropertiesUtil.loadDefaultProperties();
        PigContext pigContext=new PigContext(ExecType.LOCAL, properties);
        pigContext.connect();
        FileLocalizer.setInitialized(false);
        String tempPath= FileLocalizer.getTemporaryPath(pigContext).toString();
        assertTrue(tempPath.startsWith("file:/opt/temp"));
        propertyFile.delete();
        FileLocalizer.setInitialized(false);
View Full Code Here
443
444
445
446
447
448
449
450
451
452
453
    }
   
    @Test
    public void testIsTempFile() throws Exception {
        PigContext context = new PigContext(ExecType.LOCAL, new Properties());
        context.connect();
        for (int i=0; i<100; i++) {
            String file = FileLocalizer.getTemporaryPath(context).toString();
            assertTrue("not a temp file: " + file, PigStatsUtil.isTempFile(file));
        }
    }
View Full Code Here
34
35
36
37
38
39
40
41
42
43
44
        Configuration conf = new Configuration(false);
        conf.set("foo", "bar");
        PigContext pigContext = new PigContext(ExecType.MAPREDUCE, conf);
        // This should fail as pig expects Hadoop configs are present in
        // classpath.
        pigContext.connect();
    }
   
    @Test
    public void testJobConfGenerationWithUserConfigs() throws ExecException {
        Configuration conf = new Configuration(false);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
        // and not the classpath
        conf.set("pig.use.overriden.hadoop.configs", "true");
        conf.set("mapred.job.tracker", "host:12345");
        conf.set("apache", "pig");
        PigContext pigContext = new PigContext(ExecType.MAPREDUCE, conf);
        pigContext.connect();
        JobConf jc = pigContext.getExecutionEngine().getJobConf();
        Assert.assertEquals(jc.get("mapred.job.tracker"), "host:12345");
        Assert.assertEquals(jc.get("apache"), "pig");
    }
}
View Full Code Here
637
638
639
640
641
642
643
644
645
646
647
    public LogicalPlan buildPlan(String query, ClassLoader cldr) {
        LogicalPlanBuilder.classloader = cldr;
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
        try {

            pigContext.connect();
           
            LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //


            LogicalPlan lp = builder.parse("Test-Plan-Builder",
View Full Code Here
195
196
197
198
199
200
201
202
203
204
205
    private LogicalPlan buildPlan(String query, ClassLoader cldr) {

        LogicalPlanBuilder.classloader = LogicalPlanTester.class.getClassLoader() ;
        PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
        try {
            pigContext.connect();
        } catch (ExecException e1) {
            fail(e1.getClass().getName() + ": " + e1.getMessage() + " -- " + query);
        }
        LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.