Package org.apache.flink.runtime.io.disk.iomanager

Examples of org.apache.flink.runtime.io.disk.iomanager.BlockChannelWriter


    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
View Full Code Here


    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_LONG_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_LONG; i++) {
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);

    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, 1);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
View Full Code Here

    final TestData.Generator generator = new TestData.Generator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final Channel.ID channel = this.ioManager.createChannel();
   
    // create the writer output view
    List<MemorySegment> memory = this.memoryManager.allocatePages(this.parentTask, NUM_MEMORY_SEGMENTS);
    final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channel);
    final ChannelWriterOutputView outView = new ChannelWriterOutputView(writer, memory, MEMORY_PAGE_SIZE);
   
    // write a number of pairs
    final Record rec = new Record();
    for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
View Full Code Here

  private void testChannelWithSegments(int numSegments) throws Exception
  {
    final List<MemorySegment> memory = this.memManager.allocatePages(memoryOwner, numSegments);
    final Channel.ID channel = this.ioManager.createChannel();
   
    BlockChannelWriter writer = null;
    BlockChannelReader reader = null;
   
    try
      writer = this.ioManager.createBlockChannelWriter(channel);
      final ChannelWriterOutputView out = new ChannelWriterOutputView(writer, memory, this.memManager.getPageSize());
     
      long writeStart = System.currentTimeMillis();
     
      int valsLeft = NUM_INTS_WRITTEN;
      while (valsLeft-- > 0) {
        out.writeInt(valsLeft);
      }
     
      out.close();
      final int numBlocks = out.getBlockCount();
      writer.close();
      writer = null;
     
      long writeElapsed = System.currentTimeMillis() - writeStart;
     
      // ----------------------------------------------------------------
     
      reader = ioManager.createBlockChannelReader(channel);
      final ChannelReaderInputView in = new ChannelReaderInputView(reader, memory, numBlocks, false);
     
      long readStart = System.currentTimeMillis();
     
      valsLeft = NUM_INTS_WRITTEN;
      while (valsLeft-- > 0) {
        in.readInt();
//        Assert.assertTrue(rec.getValue() == valsLeft);
      }
     
      in.close();
      reader.close();
     
      long readElapsed = System.currentTimeMillis() - readStart;
     
      reader.deleteChannel();
      reader = null;
     
      LOG.info("IOManager with " + numSegments + " mem segments: write " + writeElapsed + " msecs, read " + readElapsed + " msecs.");
     
      memManager.release(memory);
    }
    finally {
      if (reader != null) {
        reader.closeAndDelete();
      }
      if (writer != null) {
        writer.closeAndDelete();
      }
    }
  }
View Full Code Here

  {
    final int NUM_IOS = 1111;
   
    try {
      final Channel.ID channelID = this.ioManager.createChannel();
      final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channelID);
     
      MemorySegment memSeg = this.memoryManager.allocatePages(new DummyInvokable(), 1).get(0);
     
      for (int i = 0; i < NUM_IOS; i++) {
        for (int pos = 0; pos < memSeg.size(); pos += 4) {
          memSeg.putInt(pos, i);
        }
       
        writer.writeBlock(memSeg);
        memSeg = writer.getNextReturnedSegment();
      }
     
      writer.close();
     
      final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channelID);
      for (int i = 0; i < NUM_IOS; i++) {
        reader.readBlock(memSeg);
        memSeg = reader.getNextReturnedSegment();
View Full Code Here

    final int NUM_SEGS = 16;
   
    try {
      final List<MemorySegment> memSegs = this.memoryManager.allocatePages(new DummyInvokable(), NUM_SEGS);
      final Channel.ID channelID = this.ioManager.createChannel();
      final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(channelID);
     
      for (int i = 0; i < NUM_IOS; i++) {
        final MemorySegment memSeg = memSegs.isEmpty() ? writer.getNextReturnedSegment() : memSegs.remove(0);
       
        for (int pos = 0; pos < memSeg.size(); pos += 4) {
          memSeg.putInt(pos, i);
        }
       
        writer.writeBlock(memSeg);
      }
      writer.close();
     
      // get back the memory
      while (memSegs.size() < NUM_SEGS) {
        memSegs.add(writer.getNextReturnedSegment());
      }
     
      final BlockChannelReader reader = this.ioManager.createBlockChannelReader(channelID);
      while(!memSegs.isEmpty()) {
        reader.readBlock(memSegs.remove(0));
View Full Code Here

        if (LOG.isDebugEnabled()) {
          LOG.debug("Creating temp file " + channel.toString() + '.');
        }

        // create writer
        final BlockChannelWriter writer = this.ioManager.createBlockChannelWriter(
                                channel, this.numWriteBuffersToCluster);
        registerOpenChannelToBeRemovedAtShudown(writer);
        final ChannelWriterOutputView output = new ChannelWriterOutputView(writer, this.writeMemory,
                                      this.memManager.getPageSize());
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.io.disk.iomanager.BlockChannelWriter

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.