Package parquet.schema

Examples of parquet.schema.Type

97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
      return;
    }
    final Writable[] subValues = array.get();
    final int fieldCount = type.getFieldCount();
    for (int field = 0; field < fieldCount; ++field) {
      final Type subType = type.getType(field);
      recordConsumer.startField(subType.getName(), field);
      for (int i = 0; i < subValues.length; ++i) {
        final Writable subValue = subValues[i];
        if (subValue != null) {
          if (subType.isPrimitive()) {
            if (subValue instanceof ArrayWritable) {
              writePrimitive(((ArrayWritable) subValue).get()[field]);// 0 ?
            } else {
              writePrimitive(subValue);
            }
          } else {
            if (!(subValue instanceof ArrayWritable)) {
              throw new RuntimeException("This should be a ArrayWritable: " + subValue);
            } else {
              recordConsumer.startGroup();
              writeData((ArrayWritable) subValue, subType.asGroupType());
              recordConsumer.endGroup();
            }
          }
        }
      }
      recordConsumer.endField(subType.getName(), field);
    }
  }
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
  }

  // An optional group containing a repeated anonymous group "map", containing
  // 2 elements: "key", "value"
  private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) {
    final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(),
        typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
    final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(),
        typeInfo.getMapValueTypeInfo());
    return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
  }
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    currentArr = new Object[totalFieldCount];
    converters = new Converter[selectedFieldCount];

    List<Type> selectedFields = selectedGroupType.getFields();
    for (int i = 0; i < selectedFieldCount; i++) {
      Type subtype = selectedFields.get(i);
      if (containingGroupType.getFields().contains(subtype)) {
        converters[i] = getConverterFromDescription(subtype,
            containingGroupType.getFieldIndex(subtype.getName()), this);
      } else {
        throw new IllegalStateException("Group type [" + containingGroupType +
            "] does not contain requested field: " + subtype);
      }
    }
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
  }

  // An optional group containing a repeated anonymous group "map", containing
  // 2 elements: "key", "value"
  private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) {
    final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(),
        typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
    final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(),
        typeInfo.getMapValueTypeInfo());
    return ConversionPatterns.mapType(Repetition.OPTIONAL, name, keyType, valueType);
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
      return;
    }
    final int fieldCount = type.getFieldCount();
    Writable[] values = arr.get();
    for (int field = 0; field < fieldCount; ++field) {
      final Type fieldType = type.getType(field);
      final String fieldName = fieldType.getName();
      final Writable value = values[field];
      if (value == null) {
        continue;
      }

      recordConsumer.startField(fieldName, field);

      if (fieldType.isPrimitive()) {
        writePrimitive(value);
      } else {
        recordConsumer.startGroup();
        if (value instanceof ArrayWritable) {
          if (fieldType.asGroupType().getRepetition().equals(Type.Repetition.REPEATED)) {
            writeArray((ArrayWritable) value, fieldType.asGroupType());
          } else {
            writeData((ArrayWritable) value, fieldType.asGroupType());
          }
        } else if (value != null) {
          throw new ParquetEncodingException("This should be an ArrayWritable or MapWritable: " + value);
        }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
      return;
    }
    final Writable[] subValues = array.get();
    final int fieldCount = type.getFieldCount();
    for (int field = 0; field < fieldCount; ++field) {
      final Type subType = type.getType(field);
      recordConsumer.startField(subType.getName(), field);
      for (int i = 0; i < subValues.length; ++i) {
        final Writable subValue = subValues[i];
        if (subValue != null) {
          if (subType.isPrimitive()) {
            if (subValue instanceof ArrayWritable) {
              writePrimitive(((ArrayWritable) subValue).get()[field]);// 0 ?
            } else {
              writePrimitive(subValue);
            }
          } else {
            if (!(subValue instanceof ArrayWritable)) {
              throw new RuntimeException("This should be a ArrayWritable: " + subValue);
            } else {
              recordConsumer.startGroup();
              writeData((ArrayWritable) subValue, subType.asGroupType());
              recordConsumer.endGroup();
            }
          }
        }
      }
      recordConsumer.endField(subType.getName(), field);
    }
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    PrimitiveConverter converter = getPrimitiveConverter(path);
    return new ColumnReaderImpl(path, pageReader, converter);
  }

  private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) {
    Type currentType = schema;
    Converter currentConverter = recordConverter;
    for (String fieldName : path.getPath()) {
      final GroupType groupType = currentType.asGroupType();
      int fieldIndex = groupType.getFieldIndex(fieldName);
      currentType = groupType.getType(fieldName);
      currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex);
    }
    PrimitiveConverter converter = currentConverter.asPrimitiveConverter();
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    delegate.startField(field, index);
  }

  private void validateMissingFields(int index) {
    for (int i = previousField.peek() + 1; i < index; i++) {
      Type type = types.peek().asGroupType().getType(i);
      if (type.isRepetition(Repetition.REQUIRED)) {
        throw new InvalidRecordException("required field is missing " + type);
      }
    }
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
    types.pop();
    previousField.pop();
  }

  private void validate(PrimitiveTypeName p) {
    Type currentType = types.peek().asGroupType().getType(fields.peek());
    int c = fieldValueCount.pop() + 1;
    fieldValueCount.push(c);
    if (DEBUG) LOG.debug("validate " + p + " for " + currentType.getName());
    switch (currentType.getRepetition()) {
      case OPTIONAL:
      case REQUIRED:
        if (c > 1) {
          throw new InvalidRecordException("repeated value when the type is not repeated in " + currentType);
        }
        break;
      case REPEATED:
        break;
      default:
        throw new InvalidRecordException("unknown repetition " + currentType.getRepetition() + " in " + currentType);
    }
    if (!currentType.isPrimitive() || currentType.asPrimitiveType().getPrimitiveTypeName() != p) {
      throw new InvalidRecordException("expected type " + currentType + " but got "+ p);
    }
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
105
106
      throw new RuntimeException("not found " + fieldIndex + "(" + schema.getFieldName(fieldIndex) + ") element number " + index + " in group:\n" + this);
    }
  }

  private void add(int fieldIndex, Primitive value) {
    Type type = schema.getType(fieldIndex);
    List<Object> list = data[fieldIndex];
    if (!type.isRepetition(Type.Repetition.REPEATED)
        && !list.isEmpty()) {
      throw new IllegalStateException("field "+fieldIndex+" (" + type.getName() + ") can not have more than one value: " + list);
    }
    list.add(value);
  }
View Full Code Here
TOP

Related Classes of parquet.schema.Type

  • com.asakusafw.directio.hive.parquet.DataModelMaterializer
  • com.asakusafw.directio.hive.parquet.DataModelWriteSupport
  • org.apache.drill.exec.store.parquet2.DrillParquetReader
  • org.apache.hadoop.hive.ql.io.parquet.convert.DataWritableGroupConverter
  • org.apache.hadoop.hive.ql.io.parquet.convert.HiveSchemaConverter
  • org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriter
  • org.apache.tajo.storage.parquet.TajoRecordConverter
  • org.apache.tajo.storage.parquet.TajoSchemaConverter
  • org.apache.tajo.storage.parquet.TajoWriteSupport
  • parquet.column.impl.ColumnReadStoreImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.