Package org.apache.avro.generic.GenericData

Examples of org.apache.avro.generic.GenericData.Record

65
66
67
68
69
70
71
72
73
74
75
76
77
78
    writer.close();
  }

  @Test
  public void testSpecific() throws IOException {
    GenericRecord savedRecord = new Record(Person.SCHEMA$);
    savedRecord.put("name", "John Doe");
    savedRecord.put("age", 42);
    savedRecord.put("siblingnames", Lists.newArrayList("Jimmy", "Jane"));
    populateGenericFile(Lists.newArrayList(savedRecord), Person.SCHEMA$);

    Pipeline pipeline = new MRPipeline(AvroParquetFileSourceTargetIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Person> genericCollection = pipeline.read(new AvroParquetFileSource<Person>(new Path(avroFile.getAbsolutePath()),
        Avros.records(Person.class)));
View Full Code Here
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  @Test
  public void testGeneric() throws IOException {
    String genericSchemaJson = Person.SCHEMA$.toString().replace("Person", "GenericPerson");
    Schema genericPersonSchema = new Schema.Parser().parse(genericSchemaJson);
    GenericRecord savedRecord = new Record(genericPersonSchema);
    savedRecord.put("name", "John Doe");
    savedRecord.put("age", 42);
    savedRecord.put("siblingnames", Lists.newArrayList("Jimmy", "Jane"));
    populateGenericFile(Lists.newArrayList(savedRecord), genericPersonSchema);

    Pipeline pipeline = new MRPipeline(AvroParquetFileSourceTargetIT.class, tmpDir.getDefaultConfiguration());
    PCollection<Record> genericCollection = pipeline.read(new AvroParquetFileSource<Record>(new Path
        (avroFile.getAbsolutePath()),
View Full Code Here
164
165
166
167
168
169
170
171
172
    GenericData.Record record = new GenericData.Record(Person.SCHEMA$);
    record.put("name", "name value");
    record.put("age", 42);
    record.put("siblingnames", Lists.newArrayList());

    Record detachedRecord = genericType.getDetachedValue(record);
    assertEquals(record, detachedRecord);
    assertNotSame(record, detachedRecord);
  }
View Full Code Here
68
69
70
71
72
73
74
  }

  @Test(expected=AvroRuntimeException.class)
  public void testRecordCreateEmptySchema() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    Record r = new GenericData.Record(s);
  }
View Full Code Here
89
90
91
92
93
94
95
96
  public void testRecordPutInvalidField() throws Exception {
    Schema s = Schema.createRecord("schemaName", "schemaDoc", "namespace", false);
    List<Schema.Field> fields = new ArrayList<Schema.Field>();
    fields.add(new Schema.Field("someFieldName", s, "docs", null));
    s.setFields(fields);
    Record r = new GenericData.Record(s);
    r.put("invalidFieldName", "someValue");
  }
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
340
341
342
    ByteArrayOutputStream b1 = new ByteArrayOutputStream(5);
    ByteArrayOutputStream b2 = new ByteArrayOutputStream(5);
    BinaryEncoder b1Enc = EncoderFactory.get().binaryEncoder(b1, null);
    BinaryEncoder b2Enc = EncoderFactory.get().binaryEncoder(b2, null);
    // Prepare two different datums
    Record testDatum1 = new Record(record);
    testDatum1.put(0, 1);
    Record testDatum2 = new Record(record);
    testDatum2.put(0, 2);
    GenericDatumWriter<Record> gWriter = new GenericDatumWriter<Record>(record);
    Integer start1 = 0, start2 = 0;
    try {
      // Write two datums in each stream
      // and get the offset length after the first write in each.
View Full Code Here
87
88
89
90
91
92
93
94
95
    writeCollection.write(To.avroFile(avroFile.getAbsolutePath()));

    PCollection<Record> readCollection = MemPipeline.getInstance().read(
        At.avroFile(avroFile.getAbsolutePath(), Avros.generics(writeRecord.getSchema())));

    Record readRecord = readCollection.materialize().iterator().next();

    assertEquals(writeRecord, readRecord);
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    private final HadoopSerialization ser;
    private final DataOutputBuffer tmpOutputBuffer = new DataOutputBuffer();

    public TupleRecordWriter(org.apache.avro.Schema schema, Schema pangoolSchema,
        DataFileWriter<Record> writer, HadoopSerialization ser) {
      record = new Record(schema);
      this.ser = ser;
      this.avroSchema = schema;
      this.writer = writer;
      this.pangoolSchema = pangoolSchema;
    }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
    assertNotSame(person, deepCopyPerson);
  }

  @Test
  public void testDeepCopyGeneric() {
    Record record = new Record(Person.SCHEMA$);
    record.put("name", "John Doe");
    record.put("age", 42);
    record.put("siblingnames", Lists.newArrayList());

    Record deepCopyRecord = new AvroDeepCopier.AvroGenericDeepCopier(Person.SCHEMA$)
        .deepCopy(record);

    assertEquals(record, deepCopyRecord);
    assertNotSame(record, deepCopyRecord);
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
      tuple = new Tuple(context.getTupleMRConfig().getIntermediateSchema("tweet"));
    };
   
    public void map(AvroWrapper<Record> key, NullWritable value, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException {
      Record tweet = key.datum();
      tuple.set("tweet_id",tweet.get("id"));
      tuple.set("tweet_hashtags",tweet.get("hashtags"));
      collector.write(tuple);
    }
View Full Code Here
TOP

Related Classes of org.apache.avro.generic.GenericData.Record

  • com.cloudera.cdk.data.filesystem.TestPartitionedDatasetWriter
  • com.datasalt.pangool.BaseTest
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$CountReducer
  • com.datasalt.pangool.examples.avro.AvroCustomSerializationJob$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$CountReducer
  • com.datasalt.pangool.examples.avro.AvroTopicalWordCount$TokenizeMapper
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$Red
  • com.datasalt.pangool.examples.avro.AvroTweetsJoin$TweetsMapper
  • com.datasalt.pangool.examples.avro.TestAvroTopicalWordCount
  • com.datasalt.pangool.examples.avro.TestAvroTweetsJoin
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.