Package com.backtype.hadoop.formats

Examples of com.backtype.hadoop.formats.RecordOutputStream

107
108
109
110
111
112
113
114
115
116
117
    private void writeStrings(Pail pail, String userfile, String... strs) throws IOException {
        writeStrings(pail, userfile, Arrays.asList(strs));
    }

    private void writeStrings(Pail pail, String userfile, Collection<String> strs) throws IOException {
        RecordOutputStream os = pail.openWrite(userfile);
        for(String s: strs) {
            os.writeRaw(s.getBytes());
        }
        os.close();
    }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
    }

    public void testAtomicity() throws Exception {
        String path = getTmpPath(local, "pail");
        Pail pail = Pail.create(local, path);
        RecordOutputStream os = pail.openWrite("aaa");
        assertEquals(0, pail.getUserFileNames().size());
        os.writeRaw(new byte[] {1, 2, 3});
        os.close();
        assertEquals(1, pail.getUserFileNames().size());
    }
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
        RecordStreamFactory factout;

        @Override
        protected void copyFile(FileSystem fsSource, Path source, FileSystem fsDest, Path target, Reporter reporter) throws IOException {
            RecordInputStream fin = factin.getInputStream(fsSource, source);
            RecordOutputStream fout = factout.getOutputStream(fsDest, target);

            try {
                byte[] record;
                int bytes = 0;
                while((record = fin.readRawRecord()) != null) {
                    fout.writeRaw(record);
                    bytes+=record.length;
                    if(bytes >= 1000000) { //every 1 MB of data report progress so we don't time out on large files
                        bytes = 0;
                        reporter.progress();
                    }
                }
            } finally {
                fin.close();
            }
            //don't complete files that aren't done yet. prevents partial files from being written
            fout.close();
        }
View Full Code Here
43
44
45
46
47
48
49
50
                    throw new IllegalArgumentException("Cannot write object " + obj.toString() + " to " + p.toString() +
                            ". Conflicts with the structure of the datastore.");
                }
                _workers.put(targetDir, Pail.super.openWrite(p.toString(), _overwrite));
            }
            RecordOutputStream os = _workers.get(targetDir);
            os.writeRaw(structure.serialize(obj));
        }
View Full Code Here
TOP

Related Classes of com.backtype.hadoop.formats.RecordOutputStream

  • com.backtype.hadoop.Coercer$CoercerMapper
  • com.backtype.hadoop.Consolidator$ConsolidatorMapper
  • com.backtype.hadoop.pail.AbstractPail
  • com.backtype.hadoop.pail.Pail$TypedRecordOutputStream
  • com.backtype.hadoop.pail.PailOpsTest
  • com.backtype.hadoop.pail.PailTest
  • com.backtype.support.TestUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.