Examples of org.apache.pig.PigServer.store()

org.apache.pig.PigServer.store()
Executes a Pig Latin script up to and including indicated alias and stores the resulting records into a file. That is, if a user does:
```
 PigServer server = new PigServer(); server.registerQuery("A = load 'foo';"); server.registerQuery("B = filter A by $0 > 0;"); server.registerQuery("C = order B by $1;"); 
```
Then
```
 server.store("B", "bar"); 
```
filtered but unsorted data will be stored to the file bar. If instead a user does
```
 server.store("C", "bar"); 
```
filtered and sorted data will be stored to the file bar. Equivalent to calling {@link #store(String,String,String)} withorg.apache.pig.PigStorage as the store function. @param id The alias to store @param filename The file to which to store to @return {@link ExecJob} containing information about this job @throws IOException

        outputFile = File.createTempFile("testUriUriStorer-", ".nt");
        FileUtils.deleteQuietly(outputFile);
        String outputFilename = outputFile.getAbsolutePath().replace("\\", "\\\\");
        String query = "A = LOAD 'file:" + inputFilename + "' AS (s: chararray, o: chararray);";
        pig.registerQuery(query);
        pig.store("A", outputFilename,
            "pignlproc.storage.UriStringLiteralNTriplesStorer(" +
            "'http://mynamespace#hasValue', 'http://example.org/source#', 'en')");
        assertTrue(outputFile.exists());
        File outPart = new File(outputFile, "part-m-00000");
        assertTrue(outPart.exists());

View Full Code Here

        try {
            outputFile = File.createTempFile("JIAR_1027", ".out");
            PigServer pig = new PigServer(ExecType.LOCAL);
            pig
                    .registerQuery("A = load 'test/org/apache/pig/test/data/passwd';");
            PigStats stats = pig.store("A", outputFile.getAbsolutePath())
                    .getStatistics();
            assertEquals(outputFile.length(), stats.getBytesWritten());
        } catch (IOException e) {
            fail("IOException happened");
        } finally {

View Full Code Here

        server.setBatchOn();
        server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
                + ";");


        //when
        server.store("a", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 string,f3 string");

View Full Code Here

        server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
                + ";");
        server.registerQuery("b = FOREACH a GENERATE f1, TOTUPLE(f2,f3);");


        //when
        server.store("b", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 array<string>");

View Full Code Here

        server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
                + ";");
        server.registerQuery("b = FOREACH a GENERATE f1, TOBAG(f2,f3);");


        //when
        server.store("b", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 array<string>");

View Full Code Here

        server.registerQuery("a = LOAD '" + Util.encodeEscape(singlePartitionedFile) + "' using " + loadString
                + ";");
        server.registerQuery("b = FOREACH a GENERATE f1, TOMAP(f2,f3);");


        //when
        server.store("b", outputFile.getAbsolutePath(), storeString);


        //then
        Path outputPath = new Path(outputFile.getAbsolutePath()+"/part-m-00000.rc");


        ColumnarStruct struct = readRow(outputFile, outputPath, "f1 string,f2 map<string,string>");

View Full Code Here

        "uniqcnt  = foreach transactionsG {"+
            "sym = transactions.product ;"+
            "dsym =  distinct sym ;"+
            "generate flatten(dsym.product) as f1, COUNT(dsym) as count ;" +
            "};");
    pigServer.store("uniqcnt", tempJsonFile.getAbsolutePath(), "JsonStorage");


    BufferedReader br = new BufferedReader(new FileReader(tempJsonFile.getAbsolutePath()+ "/part-r-00000"));
    String data = br.readLine();


    assertEquals(jsonOutput, data);

View Full Code Here

        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        ExecJob job = pigServer.store("c", "output_map_only");
        PigStats pigStats = job.getStatistics();
        
        //counting the no. of bytes in the output file
        //long filesize = cluster.getFileSystem().getFileStatus(new Path("output_map_only")).getLen();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath(

View Full Code Here

        pw.close();
        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = filter a by $0 > 50;");
        pigServer.registerQuery("c = foreach b generate $0 - 50;");
        ExecJob job = pigServer.store("c", "output_map_only", "BinStorage");
        PigStats pigStats = job.getStatistics();
        
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath(
                "output_map_only", pigServer.getPigContext()),
                pigServer.getPigContext());

View Full Code Here

        PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
        pigServer.registerQuery("a = load '" + file + "';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group;");


        ExecJob job = pigServer.store("c", "output");
        PigStats pigStats = job.getStatistics();
        InputStream is = FileLocalizer.open(FileLocalizer.fullPath("output",
                pigServer.getPigContext()), pigServer.getPigContext());


        long filesize = 0;

View Full Code Here

0 1 2 3 4 5 6

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.