Package org.apache.tez.runtime.api

Examples of org.apache.tez.runtime.api.OutputContext


    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
   
    dist.makeInitialAllocations();
   
View Full Code Here


    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1);
   
    // Fourth request - processor
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
View Full Code Here

    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
   
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
   
    dist.makeInitialAllocations();
   
View Full Code Here

    doReturn("task").when(context).getTaskVertexName();
    return context;
  }
 
  protected OutputContext createTestOutputContext() {
    OutputContext context = mock(OutputContext.class);
    doReturn("output").when(context).getDestinationVertexName();
    doReturn("task").when(context).getTaskVertexName();
    return context;
  }
View Full Code Here

  @Test(timeout = 10000)
  public void testBufferSizing() throws IOException {
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezCounters counters = new TezCounters();
    String uniqueId = UUID.randomUUID().toString();
    OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId);

    int maxSingleBufferSizeBytes = 2047;
    Configuration conf = createConfiguration(outputContext, IntWritable.class, LongWritable.class,
        false, maxSingleBufferSizeBytes);
View Full Code Here

      InterruptedException {
    Partitioner partitioner = new HashPartitioner();
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezCounters counters = new TezCounters();
    String uniqueId = UUID.randomUUID().toString();
    OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId);
    Random random = new Random();

    Configuration conf = createConfiguration(outputContext, Text.class, Text.class, shouldCompress,
        -1, HashPartitioner.class);
    CompressionCodec codec = null;
View Full Code Here

      boolean shouldCompress) throws IOException, InterruptedException {
    PartitionerForTest partitioner = new PartitionerForTest();
    ApplicationId appId = ApplicationId.newInstance(10000, 1);
    TezCounters counters = new TezCounters();
    String uniqueId = UUID.randomUUID().toString();
    OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId);

    Configuration conf = createConfiguration(outputContext, IntWritable.class, LongWritable.class,
        shouldCompress, -1);
    CompressionCodec codec = null;
    if (shouldCompress) {
View Full Code Here

    return sb.toString();
  }

  private OutputContext createMockOutputContext(TezCounters counters, ApplicationId appId,
      String uniqueId) {
    OutputContext outputContext = mock(OutputContext.class);
    doReturn(counters).when(outputContext).getCounters();
    doReturn(appId).when(outputContext).getApplicationId();
    doReturn(1).when(outputContext).getDAGAttemptNumber();
    doReturn("dagName").when(outputContext).getDAGName();
    doReturn("destinationVertexName").when(outputContext).getDestinationVertexName();
View Full Code Here


    OutputDescriptor outputDescriptor = mock(OutputDescriptor.class);
    when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor");

    OutputContext outputContext = new TezOutputContextImpl(conf, new String[] {workDir.toString()},
        appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName,
        -1, taskAttemptID, counters, 0, userPayload, runtimeTask,
        null, auxEnv, new MemoryDistributor(1, 1, conf) , outputDescriptor, null);

    UnorderedKVOutput kvOutput = new OnFileUnorderedKVOutputForTest(outputContext, 1);

    List<Event> events = null;

    events = kvOutput.initialize();
    assertTrue(events != null && events.size() == 0);

    KeyValueWriter kvWriter = kvOutput.getWriter();
    List<KVPair> data = KVDataGen.generateTestData(true);
    for (KVPair kvp : data) {
      kvWriter.write(kvp.getKey(), kvp.getvalue());
    }

    events = kvOutput.close();
    assertTrue(events != null && events.size() == 1);
    DataMovementEvent dmEvent = (DataMovementEvent)events.get(0);

    assertEquals("Invalid source index", 0, dmEvent.getSourceIndex());

    DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto
        .parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));

    assertFalse(shufflePayload.hasEmptyPartitions());
    assertEquals(outputContext.getUniqueIdentifier(), shufflePayload.getPathComponent());
    assertEquals(shufflePort, shufflePayload.getPort());
    assertEquals("host", shufflePayload.getHost());
  }
View Full Code Here

    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);

    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);

    // Fourth request - OnFileSortedOutput
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    OutputContext e4OutputContext2 = createTestOutputContext();
    OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
    dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);

    dist.makeInitialAllocations();
View Full Code Here

TOP

Related Classes of org.apache.tez.runtime.api.OutputContext

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.