Examples of org.apache.hama.bsp.message.io.SpilledByteBuffer

org.apache.hama.bsp.message.io.SpilledByteBuffer
SpilledByteBuffer encapsulates a ByteBuffer. It lets the user to mark the end of last record written into the buffer.


  }


  public void testSpillingByteBuffer() throws Exception {
    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    SpilledByteBuffer spillBuffer = new SpilledByteBuffer(buffer);
    for (int i = 0; i < 100; ++i) {
      spillBuffer.putInt(i);
      spillBuffer.markEndOfRecord();
    }
    spillBuffer.putInt(100);
    assertEquals(spillBuffer.getMarkofLastRecord(), 400);
    assertEquals(spillBuffer.remaining(), (512 - 404));
    spillBuffer.flip();
    assertEquals(spillBuffer.remaining(), 404);
    assertEquals(spillBuffer.getMarkofLastRecord(), 400);


  }

View Full Code Here


    buffer.flip();


    DirectByteBufferInputStream inStream = new DirectByteBufferInputStream();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));
    for (int i = 0; i < 100; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());
    }


    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();


    // push in another buffer and check if the unmarked data could be read.


    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    buffer = ByteBuffer.allocateDirect(2048);
    stream.setBuffer(buffer);


    for (int i = 0; i < 400; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));


    // Read previous data
    intWritable.readFields(inStream);
    assertEquals(99, intWritable.get());


    for (int i = 0; i < 100; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());
    }


    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();


    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);


    for (int i = 0; i < 100; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));


    // Read previous data with resized intermediate buffer
    for (int i = 100; i < 400; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());

View Full Code Here

      intWritable.write(stream);
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();
    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 400));


    Iterator<IntWritable> iter = reuseByteBuffer.iterator();
    int j = 0;
    while (iter.hasNext()) {
      assertEquals(iter.next().get(), j++);
    }
    assertEquals(j, 100);
    reuseByteBuffer.prepareForNext();


    buffer.clear();


    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);


    for (int i = 0; i < 101; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 404));
    iter = reuseByteBuffer.iterator();
    assertEquals(iter.next().get(), 99);


    j = 0;
    while (iter.hasNext()) {

View Full Code Here

    CombineSpilledDataProcessor<IntWritable> processor = new CombineSpilledDataProcessor<IntWritable>(
        fileName);
    assertTrue(processor.init(conf));
    File f = new File(fileName);
    try {
      assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer,
          400)));
      buffer.flip();
      assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer,
          400)));
      assertTrue(processor.close());


      assertTrue(f.exists());
      assertEquals(f.length(), 8);

View Full Code Here


  }


  public void testSpillingByteBuffer() throws Exception {
    ByteBuffer buffer = ByteBuffer.allocateDirect(512);
    SpilledByteBuffer spillBuffer = new SpilledByteBuffer(buffer);
    for (int i = 0; i < 100; ++i) {
      spillBuffer.putInt(i);
      spillBuffer.markEndOfRecord();
    }
    spillBuffer.putInt(100);
    assertEquals(spillBuffer.getMarkofLastRecord(), 400);
    assertEquals(spillBuffer.remaining(), (512 - 404));
    spillBuffer.flip();
    assertEquals(spillBuffer.remaining(), 404);
    assertEquals(spillBuffer.getMarkofLastRecord(), 400);


  }

View Full Code Here


    buffer.flip();


    DirectByteBufferInputStream inStream = new DirectByteBufferInputStream();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));
    for (int i = 0; i < 100; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());
    }


    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();


    // push in another buffer and check if the unmarked data could be read.


    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    buffer = ByteBuffer.allocateDirect(2048);
    stream.setBuffer(buffer);


    for (int i = 0; i < 400; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));


    // Read previous data
    intWritable.readFields(inStream);
    assertEquals(99, intWritable.get());


    for (int i = 0; i < 100; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());
    }


    assertFalse(inStream.hasDataToRead());
    assertTrue(inStream.hasUnmarkData());
    inStream.prepareForNext();


    buffer.clear();
    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);


    for (int i = 0; i < 100; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    inStream.setBuffer(new SpilledByteBuffer(buffer, 400));


    // Read previous data with resized intermediate buffer
    for (int i = 100; i < 400; ++i) {
      intWritable.readFields(inStream);
      assertEquals(i, intWritable.get());

View Full Code Here

      intWritable.write(stream);
    }
    intWritable.write(stream);
    stream.close();
    buffer.flip();
    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 400));


    Iterator<IntWritable> iter = reuseByteBuffer.iterator();
    int j = 0;
    while (iter.hasNext()) {
      assertEquals(iter.next().get(), j++);
    }
    assertEquals(j, 100);
    reuseByteBuffer.prepareForNext();


    buffer.clear();


    stream = new DirectByteBufferOutputStream();
    stream.setBuffer(buffer);


    for (int i = 0; i < 101; ++i) {
      intWritable.set(i);
      intWritable.write(stream);
    }
    stream.close();
    buffer.flip();


    reuseByteBuffer.set(new SpilledByteBuffer(buffer, 404));
    iter = reuseByteBuffer.iterator();
    assertEquals(iter.next().get(), 99);


    j = 0;
    while (iter.hasNext()) {

View Full Code Here

    CombineSpilledDataProcessor<IntWritable> processor = new CombineSpilledDataProcessor<IntWritable>(
        fileName);
    assertTrue(processor.init(conf));
    File f = new File(fileName);
    try {
      assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer,
          400)));
      buffer.flip();
      assertTrue(processor.handleSpilledBuffer(new SpilledByteBuffer(buffer,
          400)));
      assertTrue(processor.close());


      assertTrue(f.exists());
      assertEquals(f.length(), 8);

View Full Code Here

TOP

Related Classes of org.apache.hama.bsp.message.io.SpilledByteBuffer

org.apache.hama.bsp.message.TestMessageIO

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.