Examples of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

Package org.apache.tez.runtime.api.events

Examples of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

org.apache.tez.runtime.api.events.CompositeDataMovementEvent
A convenience class to specify multiple DataMovementEvents which share the same payload. A contiguous range of srcIndices can be specified. This event will NOT be seen by downstream Inputs - instead they will see {@link DataMovementEvent}s which are generated based on the range specified in this event. This event should be used by an output which has the same payload for all of the Physical Outputs that it generates.

    return builder.build();
  }


  public static CompositeDataMovementEvent convertCompositeDataMovementEventFromProto(
      EventProtos.CompositeEventProto proto) {
    return new CompositeDataMovementEvent(proto.getStartIndex(),
        proto.getEndIndex(),
        proto.hasUserPayload() ?
            proto.getUserPayload().toByteArray() : null);
  }

View Full Code Here

        outputContext.getDestinationVertexName(), vmBuilder.build().toByteArray());    


    List<Event> events = Lists.newArrayListWithCapacity(numOutputs+1);
    events.add(vmEvent);


    CompositeDataMovementEvent csdme = new CompositeDataMovementEvent(0, numOutputs, payloadBytes);
    events.add(csdme);


    return events;
  }

View Full Code Here

    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0
    
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = new CompositeDataMovementEvent(0, destTasks.size(), "bytes".getBytes());
    cdmEvent.setVersion(2); // AttemptNum
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.


    edge.sendTezEventToDestinationTasks(tezEvent);

View Full Code Here

    TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0);


    List<TezEvent> taskEvents = Lists.newLinkedList();
    TezEvent tezEvent1 = new TezEvent(
        new CompositeDataMovementEvent(0, 1, new byte[0]), 
        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    TezEvent tezEvent2 = new TezEvent(
        new DataMovementEvent(0, new byte[0]), 
        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    taskEvents.add(tezEvent1);

View Full Code Here

    TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0);
    TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0);


    List<TezEvent> taskEvents = Lists.newLinkedList();
    TezEvent tezEvent1 = new TezEvent(
        new CompositeDataMovementEvent(0, 1, new byte[0]), 
        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    TezEvent tezEvent2 = new TezEvent(
        new DataMovementEvent(0, new byte[0]), 
        new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
    taskEvents.add(tezEvent1);

View Full Code Here

    }
  }
  


  private void handleCompositeDataMovementEvent(TezEvent tezEvent) {
    CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
    EventMetaData srcInfo = tezEvent.getSourceInfo();
    
    for (DataMovementEvent dmEvent : compEvent.getEvents()) {
      TezEvent newEvent = new TezEvent(dmEvent, srcInfo);
      sendTezEventToDestinationTasks(newEvent);
    }
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.tez.runtime.api.events.CompositeDataMovementEvent

org.apache.tez.common.ProtoConverters

org.apache.tez.dag.app.dag.impl.Edge

org.apache.tez.dag.app.dag.impl.TestEdge

org.apache.tez.dag.app.dag.impl.TestVertexImpl

org.apache.tez.runtime.api.event.TestCompositeDataMovementEvent

org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter

org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter

org.apache.tez.runtime.library.output.OnFileSortedOutput

org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.