Examples of parquet.schema.Type

parquet.schema.Type
Represents the declared type for a field in a schema. The Type object represents both the actual underlying type of the object (eg a primitive or group) as well as its attributes such as whether it is repeated, required, or optional.

    this.index = index;


    converters = new Converter[schema.getFieldCount()];


    for (int i = 0; i < converters.length; i++) {
      final Type type = schema.getType(i);
      if (type.isPrimitive()) {
        converters[i] = new SimplePrimitiveConverter(this, i);
      } else {
        converters[i] = new SimpleGroupConverter(this, i, type.asGroupType());
      }


    }
  }

View Full Code Here

  private void writeGroup(Group group, GroupType type) {
    int fieldCount = type.getFieldCount();
    for (int field = 0; field < fieldCount; ++field) {
      int valueCount = group.getFieldRepetitionCount(field);
      if (valueCount > 0) {
        Type fieldType = type.getType(field);
        String fieldName = fieldType.getName();
        recordConsumer.startField(fieldName, field);
        for (int index = 0; index < valueCount; ++index) {
          if (fieldType.isPrimitive()) {
            group.writeValue(field, index, recordConsumer);
          } else {
            recordConsumer.startGroup();
            writeGroup(group.getGroup(field, index), fieldType.asGroupType());
            recordConsumer.endGroup();
          }
        }
        recordConsumer.endField(fieldName, field);
      }

View Full Code Here

  private boolean contains(GroupType group, String[] path, int index) {
    if (index == path.length) {
      return false;
    }
    if (group.containsField(path[index])) {
      Type type = group.getType(path[index]);
      if (type.isPrimitive()) {
        return index + 1 == path.length;
      } else {
        return contains(type.asGroupType(), path, index + 1);
      }
    }
    return false;
  }

View Full Code Here

        }
        seg = seg.getChild();
      }
      String[] pathSegments = new String[segments.size()];
      segments.toArray(pathSegments);
      Type type = null;
      try {
        type = schema.getType(pathSegments);
      } catch (InvalidRecordException e) {
        logger.warn("Invalid record" , e);
      }
      if (type != null) {
        Type t = getType(pathSegments, 0, schema);
        if (projection == null) {
          projection = new MessageType(messageName, t);
        } else {
          projection = projection.union(new MessageType(messageName, t));
        }

View Full Code Here


      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);


      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
          pageReadStore.addColumn(schema.getColumnDescription(path), md);
        }
      }

View Full Code Here

      throw new ExecutionSetupException(e);
    }
  }


  private static Type getType(String[] pathSegments, int depth, MessageType schema) {
    Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1));
    if (depth + 1 == pathSegments.length) {
      return type;
    } else {
      Preconditions.checkState(!type.isPrimitive());
      return new GroupType(type.getRepetition(), type.getName(), getType(pathSegments, depth + 1, schema));
    }
  }

View Full Code Here

        }
        seg = seg.getChild();
      }
      String[] pathSegments = new String[segments.size()];
      segments.toArray(pathSegments);
      Type type = null;
      try {
        type = schema.getType(pathSegments);
      } catch (InvalidRecordException e) {
        logger.warn("Invalid record" , e);
      }
      if (type != null) {
        Type t = getType(pathSegments, 0, schema);
        if (projection == null) {
          projection = new MessageType(messageName, t);
        } else {
          projection = projection.union(new MessageType(messageName, t));
        }

View Full Code Here


      pageReadStore = new ColumnChunkIncReadStore(recordCount,
              codecFactoryExposer.getCodecFactory(), operatorContext.getAllocator(), fs, filePath);


      for (String[] path : schema.getPaths()) {
        Type type = schema.getType(path);
        if (type.isPrimitive()) {
          ColumnChunkMetaData md = paths.get(ColumnPath.get(path));
          pageReadStore.addColumn(schema.getColumnDescription(path), md);
        }
      }

View Full Code Here

      throw new ExecutionSetupException(e);
    }
  }


  private static Type getType(String[] pathSegments, int depth, MessageType schema) {
    Type type = schema.getType(Arrays.copyOfRange(pathSegments, 0, depth + 1));
    if (depth + 1 == pathSegments.length) {
      return type;
    } else {
      Preconditions.checkState(!type.isPrimitive());
      return new GroupType(type.getRepetition(), type.getName(), getType(pathSegments, depth + 1, schema));
    }
  }

View Full Code Here

0 1 2

TOP

Related Classes of parquet.schema.Type

com.asakusafw.directio.hive.parquet.DataModelMaterializer

com.asakusafw.directio.hive.parquet.DataModelWriteSupport

org.apache.drill.exec.store.parquet2.DrillParquetReader

org.apache.hadoop.hive.ql.io.parquet.convert.DataWritableGroupConverter

org.apache.hadoop.hive.ql.io.parquet.convert.HiveSchemaConverter

org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriter

org.apache.tajo.storage.parquet.TajoRecordConverter

org.apache.tajo.storage.parquet.TajoSchemaConverter

org.apache.tajo.storage.parquet.TajoWriteSupport

parquet.column.impl.ColumnReadStoreImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.