Examples of org.apache.pig.PigServer.store()

org.apache.pig.PigServer.store()
Executes a Pig Latin script up to and including indicated alias and stores the resulting records into a file. That is, if a user does:
```
 PigServer server = new PigServer(); server.registerQuery("A = load 'foo';"); server.registerQuery("B = filter A by $0 > 0;"); server.registerQuery("C = order B by $1;"); 
```
Then
```
 server.store("B", "bar"); 
```
filtered but unsorted data will be stored to the file bar. If instead a user does
```
 server.store("C", "bar"); 
```
filtered and sorted data will be stored to the file bar. Equivalent to calling {@link #store(String,String,String)} withorg.apache.pig.PigStorage as the store function. @param id The alias to store @param filename The file to which to store to @return {@link ExecJob} containing information about this job @throws IOException

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = order a by $0;");
        pigServer.registerQuery("c = group b by $0;");
        pigServer.registerQuery("d = foreach c generate group, SUM(b.$1);");
        PigStats pigStats = pigServer.store("d", "output").getStatistics();
        
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output", pigServer.getPigContext()), ExecType.MAPREDUCE, pigServer.getPigContext().getDfs());
        long filesize = 0;
        while(is.read() != -1) filesize++;

View Full Code Here

            outputFile = File.createTempFile("JIAR_1027", ".out");
            String filePath = outputFile.getAbsolutePath();
            outputFile.delete();
            PigServer pig = new PigServer(ExecType.LOCAL);
            pig.registerQuery("A = load 'test/org/apache/pig/test/data/passwd';");
            ExecJob job = pig.store("A", filePath);
            PigStats stats = job.getStatistics();
            File dataFile = new File( outputFile.getAbsoluteFile() + File.separator + "part-00000" );
            assertEquals(dataFile.length(), stats.getBytesWritten());
        } catch (IOException e) {
            e.printStackTrace();

View Full Code Here

        for(int i = 0; i < 10; i++) {
            ps.println(i);
        }
        ps.close();
        pig.registerQuery("a = load 'file:" + Util.encodeEscape(tmpFile.toString())+"';");
        pig.store("a", "/tmp/abc/xyz");
        
        tmpFile.delete();
        tmpFile = new File("/tmp/abc/xyz");
        tmpFile.delete();
        tmpFile = new File("/tmp/abc");

View Full Code Here

        String file1 = "/tmp/testPigOutput" ;
        if (pig.existsFile(file1)) {
            pig.deleteFile(file1) ;
        }
        
        pig.store("A", file1, DummyStoreFunc.class.getName() + "()");
        
        String file2 = "/tmp/testPigOutput2" ;
        if (pig.existsFile(file2)) {
            pig.deleteFile(file2) ;
        }

View Full Code Here

        
        String file2 = "/tmp/testPigOutput2" ;
        if (pig.existsFile(file2)) {
            pig.deleteFile(file2) ;
        }
        pig.store("A", file2, DummyStoreFunc.class.getName() + "()");
        
        // for this test the plan will not be reused so:-
        // - initial temp file has to be read 10 times 
        // - DummyLoadStoreFunc has to be written 10 times

View Full Code Here

        String file1 = "/tmp/testPigOutput" ;
        if (pig.existsFile(file1)) {
            pig.deleteFile(file1) ;
        }
        
        pig.store("A", file1, DummyLoadStoreFunc.class.getName() + "()");
        
        String file2 = "/tmp/testPigOutput2" ;
        if (pig.existsFile(file2)) {
            pig.deleteFile(file2) ;
        }

View Full Code Here

        
        String file2 = "/tmp/testPigOutput2" ;
        if (pig.existsFile(file2)) {
            pig.deleteFile(file2) ;
        }
        pig.store("A", file2, DummyLoadStoreFunc.class.getName() + "()");
        
        // for this test the plan will be reused so:-
        // - initial temp file has to be read 5 times 
        // - the output of the first execution has to be read 5 times
        // - DummyLoadStoreFunc has to be written 10 times

View Full Code Here

        outputFile = File.createTempFile("testUriUriStorer-", ".nt");
        FileUtils.deleteQuietly(outputFile);
        String outputFilename = outputFile.getAbsolutePath().replace("\\", "\\\\");
        String query = "A = LOAD 'file:" + inputFilename + "' AS (s: chararray, o: chararray);";
        pig.registerQuery(query);
        pig.store("A", outputFilename,
            "pignlproc.storage.UriUriNTriplesStorer('http://mynamespace#followerOf')");
        assertTrue(outputFile.exists());
        File outPart = new File(outputFile, "part-m-00000");
        assertTrue(outPart.exists());
        List<String> lines = FileUtils.readLines(outPart);

View Full Code Here

        outputFile = File.createTempFile("testUriUriStorer-", ".nt");
        FileUtils.deleteQuietly(outputFile);
        String outputFilename = outputFile.getAbsolutePath().replace("\\", "\\\\");
        String query = "A = LOAD 'file:" + inputFilename + "' AS (s: chararray, o: chararray);";
        pig.registerQuery(query);
        pig.store("A", outputFilename, "pignlproc.storage.UriUriNTriplesStorer("
                                       + "'http://mynamespace#followerOf', 'http://example.org/source#',"
                                       + " 'http://example.org/target#')");
        assertTrue(outputFile.exists());
        File outPart = new File(outputFile, "part-m-00000");
        assertTrue(outPart.exists());

View Full Code Here

        outputFile = File.createTempFile("testUriUriStorer-", ".nt");
        FileUtils.deleteQuietly(outputFile);
        String outputFilename = outputFile.getAbsolutePath().replace("\\", "\\\\");
        String query = "A = LOAD 'file:" + inputFilename + "' AS (s: chararray, o: chararray);";
        pig.registerQuery(query);
        pig.store("A", outputFilename,
            "pignlproc.storage.UriStringLiteralNTriplesStorer('http://mynamespace#hasValue')");
        assertTrue(outputFile.exists());
        File outPart = new File(outputFile, "part-m-00000");
        assertTrue(outPart.exists());
        List<String> lines = FileUtils.readLines(outPart);

View Full Code Here

0 1 2 3 4 5 6

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.