Examples of CompressionCodec

Examples of org.apache.hadoop.io.compress.CompressionCodec

177
178
179
180
181
182
183
184
185
186
187
          totalBytesWritten += key.length;
          totalBytesWritten += value.length;
         }
        writer.close();
    } else if ("SequenceFile".equals(fileType)){
        CompressionCodec codec = null;
        if ("gz".equals(codecName))
          codec = new GzipCodec();
        else if (!"none".equals(codecName))
          throw new IOException("Codec not supported.");
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

40
41
42
43
44
45
46
47
48
49
50
                                      String name, Progressable progress)
    throws IOException {

    Path file = new Path(job.getOutputPath(), name);
    FileSystem fs = file.getFileSystem(job);
    CompressionCodec codec = null;
    CompressionType compressionType = CompressionType.NONE;
    if (getCompressOutput(job)) {
      // find the kind of compression to do
      compressionType = SequenceFile.getCompressionType(job);
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
                                      Progressable prog) throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();
     
      FileSystem dstFileSys = outputFile.getFileSystem(conf);
      FSDataOutputStream out;
      if (prog != null)
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
        int segments = 0;
        int currentFile = 0;
        boolean atEof = (currentFile >= inFiles.length);
        boolean isCompressed = false;
        boolean isBlockCompressed = false;
        CompressionCodec codec = null;
        segmentLengths.clear();
        if (atEof) {
          return 0;
        }
       
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

49
50
51
52
53
54
55
56
57
58
59
  }
 
  public void testLzoSequenceFile() throws Exception {
    if (LzoCodec.isNativeLzoLoaded()) {
      LOG.info("Testing SequenceFile with LzoCodec");
      CompressionCodec lzoCodec = null;
      try {
        lzoCodec = (CompressionCodec) ReflectionUtils.newInstance(
                                                                  conf.getClassByName(LzoCodec.class.getName()), conf);
      } catch (ClassNotFoundException cnfe) {
        throw new IOException("Cannot find LzoCodec!");
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

323
324
325
326
327
328
329
330
331
332
333
  public void testSequenceFileMetadata() throws Exception {
    LOG.info("Testing SequenceFile with metadata");
    int count = 1024 * 10;
    int megabytes = 1;
    int factor = 5;
    CompressionCodec codec = new DefaultCodec();
    Path file = new Path(System.getProperty("test.build.data",".")+"/test.seq.metadata");
    Path recordCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.rc.seq.metadata");
    Path blockCompressedFile =
      new Path(System.getProperty("test.build.data",".")+"/test.bc.seq.metadata");
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

483
484
485
486
487
488
489
490
491
492
493
        System.exit(-1);
      }

      CompressionType compressionType =
        CompressionType.valueOf(compressType);
      CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(
                                                                             conf.getClassByName(compressionCodec),
                                                                             conf);

      if (rwonly || (create && !merge)) {
        writeTest(fs, count, seed, file, compressionType, codec);
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    throws IOException {
    long start = split.getStart();
    long end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = FileSystem.get(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    InputStream in = fileIn;
    if (codec != null) {
      in = codec.createInputStream(fileIn);
      end = Long.MAX_VALUE;
    } else if (start != 0) {
      fileIn.seek(start - 1);
      LineRecordReader.readLine(fileIn, null);
      start = fileIn.getPos();
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

75
76
77
78
79
80
81
82
83
84
85
86
87
88
      FSDataOutputStream fileOut = fs.create(new Path(dir, name), progress);
      return new LineRecordWriter(fileOut);
    } else {
      Class codecClass = getOutputCompressorClass(job, GzipCodec.class);
      // create the named codec
      CompressionCodec codec = (CompressionCodec)
        ReflectionUtils.newInstance(codecClass, job);
      // build the filename including the extension
      Path filename = new Path(dir, name + codec.getDefaultExtension());
      FSDataOutputStream fileOut = fs.create(filename, progress);
      return new LineRecordWriter(new DataOutputStream
                                  (codec.createOutputStream(fileOut)));
    }
  }     
View Full Code Here

Examples of org.apache.hadoop.io.compress.CompressionCodec

2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
    throws IOException {
      FileSystem srcFileSys = inputFile.getFileSystem(conf);
      Reader reader = new Reader(srcFileSys, inputFile, 4096, conf, true);
      boolean compress = reader.isCompressed();
      boolean blockCompress = reader.isBlockCompressed();
      CompressionCodec codec = reader.getCompressionCodec();
      reader.close();

      Writer writer = createWriter(outputFile.getFileSystem(conf), conf,
                                   outputFile, keyClass, valClass, compress,
                                   blockCompress, codec, prog,
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.