Package parquet.hadoop.metadata

Examples of parquet.hadoop.metadata.CompressionCodecName

77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
  public static CompressionCodecName getParquetCompressionCodec(Configuration configuration) {
    return CompressionCodecName.fromConf(configuration.get(ParquetOutputFormat.COMPRESSION, UNCOMPRESSED.name()));
  }

  public CompressionCodecName getCodec() {
    CompressionCodecName codec;
    Configuration configuration = getConfiguration();
    if (isParquetCompressionSet(configuration)) { // explicit parquet config
      codec = getParquetCompressionCodec(configuration);
    } else if (isHadoopCompressionSet()) { // from hadoop config
      codec = getHadoopCompressionCodec();
    } else {
      if (INFO) LOG.info("Compression set to false");
      codec = CompressionCodecName.UNCOMPRESSED;
    }

    if (INFO) LOG.info("Compression: " + codec.name());
    return codec;
  }
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
    if (INFO) LOG.info("Compression: " + codec.name());
    return codec;
  }

  private CompressionCodecName getHadoopCompressionCodec() {
    CompressionCodecName codec;
    try {
      // find the right codec
      Class<?> codecClass = getHadoopOutputCompressorClass(CompressionCodecName.UNCOMPRESSED.getHadoopCompressionCodecClass());
      if (INFO) LOG.info("Compression set through hadoop codec: " + codecClass.getName());
      codec = CompressionCodecName.fromCompressionCodec(codecClass);
View Full Code Here
243
244
245
246
247
248
249
250
251
252
253
    }
  }

  private ColumnChunkMetaData readColumn(DataInput in)
      throws IOException {
    CompressionCodecName codec = CompressionCodecName.values()[in.readInt()];
    String[] columnPath = new String[in.readInt()];
    for (int i = 0; i < columnPath.length; i++) {
      columnPath[i] = in.readUTF().intern();
    }
    PrimitiveTypeName type = PrimitiveTypeName.values()[in.readInt()];
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
    //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
    messageSchema += "}";

    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);

    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
    w.start();
    HashMap<String, Integer> columnValuesWritten = new HashMap();
    int valsWritten;
    for (int k = 0; k < props.numberRowGroups; k++) {
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
88
    private ParquetRecordWriter<V> realWriter;

    public RecordWriterWrapper(ParquetOutputFormat<V> realOutputFormat,
        FileSystem fs, JobConf conf, String name, Progressable progress) throws IOException {

      CompressionCodecName codec = getCodec(conf);
      String extension = codec.getExtension() + ".parquet";
      Path file = getDefaultWorkFile(conf, name, extension);

      try {
        realWriter = (ParquetRecordWriter<V>) realOutputFormat.getRecordWriter(conf, file, codec);
      } catch (InterruptedException e) {
View Full Code Here
TOP

Related Classes of parquet.hadoop.metadata.CompressionCodecName

  • com.asakusafw.directio.hive.parquet.AbstractParquetFileFormat
  • com.asakusafw.dmdl.directio.hive.parquet.ParquetFileDriver
  • com.cloudera.cdk.data.filesystem.ParquetFileSystemDatasetWriter
  • org.apache.drill.exec.store.parquet.TestFileGenerator
  • org.apache.drill.exec.store.ParquetRecordReaderTest
  • org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper
  • org.kitesdk.data.spi.filesystem.ParquetAppender
  • parquet.hadoop.codec.CodecConfig
  • parquet.hadoop.mapred.DeprecatedParquetOutputFormat$RecordWriterWrapper
  • parquet.hadoop.ParquetInputSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.