Package org.apache.hadoop.hive.ql.io.orc

Examples of org.apache.hadoop.hive.ql.io.orc.OrcStruct$OrcStructInspector


    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    PType<OrcStruct> ptype = Orcs.orcs(typeInfo);
   
    HiveVarchar varchar = new HiveVarchar("Hello World", 32);
    Map<Text, OrcStruct> map = new HashMap<Text, OrcStruct>();
    OrcStruct value = OrcUtils.createOrcStruct(mapValueTypeInfo, new Text("age"), new IntWritable(24));
    map.put(new Text("Bob"), value);
    OrcStruct s = OrcUtils.createOrcStruct(typeInfo, new IntWritable(1024), new Text("Alice"),
        null, new HiveVarcharWritable(varchar), map);
    OrcWritable w = new OrcWritable();
    w.set(s);
   
    testInputOutputFn(ptype, s, w);
View Full Code Here


    map.put(new Text("Alice"), OrcUtils.createOrcStruct(Person.TYPE_INFO, new Text("Alice"), new IntWritable(23),
        Arrays.asList(new Text("666-677-9999"))));
    map.put(new Text("Bob"), OrcUtils.createOrcStruct(Person.TYPE_INFO, new Text("Bob"), new IntWritable(26),
        Arrays.asList(new Text("999-888-1132"), new Text("000-222-9934"))));
    map.put(new Text("David"), null);
    OrcStruct s = OrcUtils.createOrcStruct(AddressBook.TYPE_INFO, new Text("John Smith"),
        Arrays.asList(new Text("919-333-4452"), new Text("650-777-4329")), map, new TimestampWritable(now),
        new BytesWritable(signature));
    OrcWritable w = new OrcWritable();
    w.set(s);
   
View Full Code Here

    String typeStr = "struct<a:int,b:string,c:" + Person.TYPE_STR + ",d:struct<d1:string,d2:" + Person.TYPE_STR + ">>";
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    String tableTypeStr = "struct<a:string,b:" + Person.TYPE_STR + ">";
    TypeInfo tableTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(tableTypeStr);
   
    OrcStruct s = OrcUtils.createOrcStruct(typeInfo, new IntWritable(1), new Text("John Smith"),
        OrcUtils.createOrcStruct(Person.TYPE_INFO, new Text("Alice"), new IntWritable(23),
            Arrays.asList(new Text("666-677-9999"))
        ),
        OrcUtils.createOrcStruct(tableTypeInfo, new Text("Bob"),
            OrcUtils.createOrcStruct(Person.TYPE_INFO, new Text("Bob"), new IntWritable(26),
View Full Code Here

   */
  public static OrcStruct createOrcStruct(TypeInfo typeInfo, Object... objs) {
    SettableStructObjectInspector oi = (SettableStructObjectInspector) OrcStruct
        .createObjectInspector(typeInfo);
    List<StructField> fields = (List<StructField>) oi.getAllStructFieldRefs();
    OrcStruct result = (OrcStruct) oi.create();
    result.setNumFields(fields.size());
    for (int i = 0; i < fields.size(); i++) {
      oi.setStructFieldData(result, fields.get(i), objs[i]);
    }
    return result;
  }
View Full Code Here

    String typeStr = "struct<a:int,b:string,c:float>";
    TypeInfo info = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    StructObjectInspector oi = (StructObjectInspector) OrcStruct.createObjectInspector(info);
    BinarySortableSerDe serde = OrcUtils.createBinarySerde(info);
   
    OrcStruct struct = OrcUtils.createOrcStruct(info,
        new IntWritable(1), new Text("Alice"), new FloatWritable(165.3f));
    OrcWritable writable = new OrcWritable();
    writable.set(struct);
    assertTrue(struct == writable.get());
   
    writable.setObjectInspector(oi);
    writable.setSerde(serde);
   
    WritableDeepCopier<OrcWritable> deepCopier = new WritableDeepCopier<OrcWritable>(OrcWritable.class);
    OrcWritable copied = deepCopier.deepCopy(writable);
    assertTrue(writable != copied);
    assertEquals(writable, copied);
   
    copied.setObjectInspector(oi);
    copied.setSerde(serde);
    OrcStruct copiedStruct = copied.get();
    assertTrue(struct != copiedStruct);
    assertEquals(struct, copiedStruct);
   
    List<Object> items = oi.getStructFieldsDataAsList(struct);
    List<Object> copiedItems = oi.getStructFieldsDataAsList(copiedStruct);
    for (int i = 0; i < items.size(); i++) {
      assertTrue(items.get(i) != copiedItems.get(i));
      assertEquals(items.get(i), copiedItems.get(i));
    }
   
    OrcWritable copied2 = deepCopier.deepCopy(copied);
    assertTrue(copied2 != copied);
    assertEquals(copied2, copied);
   
    copied2.setObjectInspector(oi);
    copied2.setSerde(serde);
    OrcStruct copiedStruct2 = copied2.get();
    assertTrue(copiedStruct2 != copiedStruct);
    assertEquals(copiedStruct2, copiedStruct);
   
    List<Object> copiedItems2 = oi.getStructFieldsDataAsList(copiedStruct2);
    for (int i = 0; i < items.size(); i++) {
View Full Code Here

    String typeStr = "struct<a:int,b:string,c:float>";
    TypeInfo info = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    StructObjectInspector oi = (StructObjectInspector) OrcStruct.createObjectInspector(info);
    BinarySortableSerDe serde = OrcUtils.createBinarySerde(info);
   
    OrcStruct struct1 = OrcUtils.createOrcStruct(info, new IntWritable(1), new Text("AAA"), new FloatWritable(3.2f));
    OrcStruct struct2 = OrcUtils.createOrcStruct(info, new IntWritable(1), new Text("AAB"), null);
    OrcStruct struct3 = OrcUtils.createOrcStruct(info, new IntWritable(2), new Text("AAA"), null);
    OrcStruct struct4 = OrcUtils.createOrcStruct(info, new IntWritable(2), new Text("AAA"), new FloatWritable(3.2f));
   
    OrcWritable writable1 = new OrcWritable();
    writable1.set(struct1);
    OrcWritable writable2 = new OrcWritable();
    writable2.set(struct2);
View Full Code Here

public class OrcFileSourceTargetIT extends OrcFileTest implements Serializable {
 
  private void generateInputData() throws IOException {
    String typeStr = "struct<name:string,age:int,numbers:array<string>>";
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    OrcStruct s = OrcUtils.createOrcStruct(typeInfo, new Text("Alice"), new IntWritable(23),
        Arrays.asList(new Text("919-342-5555"), new Text("650-333-2913")));
   
    OrcFileWriter<OrcStruct> writer = new OrcFileWriter<OrcStruct>(conf, new Path(tempPath, "input.orc"), Orcs.orcs(typeInfo));
    writer.write(s);
    writer.close();
View Full Code Here

  public void testOrcs() throws IOException {
    generateInputData();
   
    String typeStr = "struct<name:string,age:int,numbers:array<string>>";
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    OrcStruct expected = OrcUtils.createOrcStruct(typeInfo, new Text("Alice"), new IntWritable(23),
        Arrays.asList(new Text("919-342-5555"), new Text("650-333-2913")));
   
    testSourceTarget(Orcs.orcs(typeInfo), expected);
  }
View Full Code Here

 
  @Test
  public void testGrouping() throws IOException {
    String typeStr = "struct<name:string,age:int,numbers:array<string>>";
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    OrcStruct s1 = OrcUtils.createOrcStruct(typeInfo, new Text("Bob"), new IntWritable(28), null);
    OrcStruct s2 = OrcUtils.createOrcStruct(typeInfo, new Text("Bob"), new IntWritable(28), null);
    OrcStruct s3 = OrcUtils.createOrcStruct(typeInfo, new Text("Alice"), new IntWritable(23),
        Arrays.asList(new Text("444-333-9999")));
    OrcStruct s4 = OrcUtils.createOrcStruct(typeInfo, new Text("Alice"), new IntWritable(36),
        Arrays.asList(new Text("919-342-5555"), new Text("650-333-2913")));

    Path inputPath = new Path(tempPath, "input.orc");
    OrcFileWriter<OrcStruct> writer = new OrcFileWriter<OrcStruct>(conf, inputPath, Orcs.orcs(typeInfo));
    writer.write(s1);
View Full Code Here

    Assert.assertEquals(1, splits.length);
    org.apache.hadoop.mapred.RecordReader<NullWritable, OrcStruct> rr =
            inf.getRecordReader(splits[0], job, Reporter.NULL);

    NullWritable key = rr.createKey();
    OrcStruct value = rr.createValue();
    for (int i = 0; i < records.length; i++) {
      Assert.assertEquals(true, rr.next(key, value));
      Assert.assertEquals(records[i], value.toString());
    }
    Assert.assertEquals(false, rr.next(key, value));
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.ql.io.orc.OrcStruct$OrcStructInspector

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.