Package com.facebook.hive.orc.OrcTestUtils

Examples of com.facebook.hive.orc.OrcTestUtils.IntStruct


              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    ReaderWriterProfiler.setProfilerOptions(conf);
    WriterImplWithForceFlush writer = new WriterImplWithForceFlush(fs, testFilePath, conf,
        inspector, 1000000, CompressionKind.NONE, 100, 1000, new MemoryManager(conf));
    writer.addRow(new IntStruct(1));
    writer.addRow(new IntStruct(2));
    writer.addRow(new IntStruct(3));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new IntStruct(123));
    }
    writer.forceFlushStripe();
    for (int i = 0; i < 1000; i++) {
      writer.addRow(new IntStruct(123));
    }
    writer.forceFlushStripe();
    writer.addRow(new IntStruct(1));
    writer.addRow(new IntStruct(2));
    writer.addRow(new IntStruct(3));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new IntStruct(123));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
View Full Code Here


    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 10000, memory);

    // Write 500 rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Force the writer to enter low memory mode, note since the stride length was set to 10000
    // we're still in the first stride
    memory.forceEnterLowMemoryMode();

    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i + 500));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);

    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Write 250 more rows (a portion of the second stride)
    for (int i = 0; i < 250; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're still in the second stride
    memory.forceEnterLowMemoryMode();

    // Write 250 more rows (which still gets written to the second stride, but not enough to fill
    // it)
    for (int i = 0; i < 250; i ++) {
      writer.addRow(new IntStruct(i + 250));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);

    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're just starting the second stride
    memory.forceEnterLowMemoryMode();

    // Write 500 more rows (a portion of the second stride)
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);

    // Write 1000 rows (the first stride)
    for (int i = 0; i < 1001; i ++) {
      writer.addRow(new IntStruct(i % 1000));
    }

    // Force the writer to enter low memory mode, note since the stride length was set to 1000
    // we're just after starting the second stride
    memory.forceEnterLowMemoryMode();

    // Write 499 more rows (a portion of the second stride)
    for (int i = 1; i < 500; i ++) {
      writer.addRow(new IntStruct(i));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    WriterImplWithForceFlush writer = new WriterImplWithForceFlush(fs, testFilePath, conf,
        inspector, 1000000, CompressionKind.NONE, 100, 10000, memory);

    // Write 500 rows
    for (int i = 0; i < 1000; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Flush the first stripe
    writer.forceFlushStripe();

    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Force the writer to enter low memory mode
    memory.forceEnterLowMemoryMode();

    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i + 500));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    WriterImplWithForceFlush writer = new WriterImplWithForceFlush(fs, testFilePath, conf,
        inspector, 1000000, CompressionKind.NONE, 100, 10000, new MemoryManager(conf));

    // Write 100 rows
    for (int i = 0; i < 100; i++) {
      writer.addRow(new IntStruct(i));
    }

    // Flush the first stripe
    writer.forceFlushStripe();

    // Write 100 more rows
    for (int i = 0; i < 100; i++) {
      writer.addRow(new IntStruct(i + 100));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

    WriterImplWithForceFlush writer = new WriterImplWithForceFlush(fs, testFilePath, conf,
        inspector, 1000000, CompressionKind.NONE, 100, 10000, new MemoryManager(conf));

    // Write 100 rows
    for (int i = 0; i < 100; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Flush the first stripe
    writer.forceFlushStripe();

    // Write 100 more rows
    for (int i = 0; i < 100; i ++) {
      writer.addRow(new IntStruct(i + 100));
    }

    writer.close();

    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
View Full Code Here

    }
    MemoryManagerWithForce memory = new MemoryManagerWithForce(conf);
    ReaderWriterProfiler.setProfilerOptions(conf);
    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 1000, memory);
    writer.addRow(new IntStruct(1));
    writer.addRow(new IntStruct(2));
    writer.addRow(new IntStruct(3));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new IntStruct(123));
    }
    memory.forceFlushStripe();
    for (int i = 0; i < 1000; i++) {
      writer.addRow(new IntStruct(123));
    }
    memory.forceFlushStripe();
    writer.addRow(new IntStruct(1));
    writer.addRow(new IntStruct(2));
    writer.addRow(new IntStruct(3));
    for (int i = 0; i < 997; i++) {
      writer.addRow(new IntStruct(123));
    }
    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
    OrcLazyStruct lazyRow = null;
View Full Code Here

    Writer writer = new WriterImpl(fs, testFilePath, conf, inspector,
        1000000, CompressionKind.NONE, 100, 10000, memory);

    // Write 500 rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i));
    }

    // Force the writer to enter low memory mode, note since the stride length was set to 10000
    // we're still in the first stride
    memory.forceEnterLowMemoryMode();

    // Write 500 more rows
    for (int i = 0; i < 500; i ++) {
      writer.addRow(new IntStruct(i + 500));
    }

    writer.close();
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);
    RecordReader rows = reader.rows(null);
View Full Code Here

TOP

Related Classes of com.facebook.hive.orc.OrcTestUtils.IntStruct

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.