Package org.springframework.data.hadoop.store.split

Examples of org.springframework.data.hadoop.store.split.StaticLengthSplitter

69
70
71
72
73
74
75
76
77
78
79
80
  @Test
  public void testWithSplit() throws Exception {
    Path path = new Path("/syarn-tmp/DataStoreItemReaderTests-testWithSplit/data.txt");
    createTestData(path);

    StaticLengthSplitter splitter = new StaticLengthSplitter(getConfiguration(), 1500);
    List<Split> splits = splitter.getSplits(path);
    assertThat(splits.size(), is(2));

    DataStoreReader<String> reader = new TextFileReader(configuration, path, null, splits.get(0), null);
    DataStoreItemReader<String> itemReader = new DataStoreItemReader<String>();
    itemReader.setDataStoreReader(reader);
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, null);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, true);

    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 110l);
    List<Split> inputSplits = splitter.getSplits(testDefaultPath);
    assertNotNull(inputSplits);
    assertThat(inputSplits.size(), is(3));

    TextFileReader reader1 = new TextFileReader(getConfiguration(), testDefaultPath, null, inputSplits.get(0), null);
    List<String> readData1 = TestUtils.readData(reader1);
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
91
    TextFileWriter writer = new TextFileWriter(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo());
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, false);
    TestUtils.writeData(writer, DATA09ARRAY, true);

    Splitter splitter = new StaticLengthSplitter(getConfiguration(), 30l);
    List<Split> inputSplits = splitter.getSplits(testDefaultPath);
    assertNotNull(inputSplits);
    assertThat(inputSplits.size(), is(3));

    TextFileReader reader1 = new TextFileReader(getConfiguration(), testDefaultPath, Codecs.BZIP2.getCodecInfo(), inputSplits.get(0), null);
    List<String> readData1 = TestUtils.readData(reader1);
View Full Code Here
TOP

Related Classes of org.springframework.data.hadoop.store.split.StaticLengthSplitter

  • org.springframework.data.hadoop.store.TextFileStoreSplitTests
  • org.springframework.yarn.batch.item.DataStoreItemReaderTests
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.