Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.EdgeProperty


    dag.addVertex(stage1Vertex);
    dag.addVertex(stage2Vertex);
    dag.addVertex(stage3Vertex);

    Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
    Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
        DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
        OnFileSortedOutput.class.getName()), new InputDescriptor(
                ShuffledMergedInputLegacy.class.getName())));
View Full Code Here


        sourceVertex.getName(),
        null);
  }

  public synchronized void setCustomEdgeManager(EdgeManagerDescriptor descriptor) {
    EdgeProperty modifiedEdgeProperty =
        new EdgeProperty(descriptor,
            edgeProperty.getDataSourceType(),
            edgeProperty.getSchedulingType(),
            edgeProperty.getEdgeSource(),
            edgeProperty.getEdgeDestination());
    this.edgeProperty = modifiedEdgeProperty;
View Full Code Here

    return DAGState.INITED;
  }

  private void createDAGEdges(DAGImpl dag) {
    for (EdgePlan edgePlan : dag.getJobPlan().getEdgeList()) {
      EdgeProperty edgeProperty = DagTypeConverters
          .createEdgePropertyMapFromDAGPlan(edgePlan);

      // If CUSTOM without an edge manager, setup a fake edge manager. Avoid
      // referencing the fake edge manager within the API module.
      if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM
          && edgeProperty.getEdgeManagerDescriptor() == null) {
        EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(
            NullEdgeManager.class.getName());
        EdgeProperty ep = new EdgeProperty(edgeDesc, edgeProperty.getDataSourceType(),
            edgeProperty.getSchedulingType(), edgeProperty.getEdgeSource(),
            edgeProperty.getEdgeDestination());
        edgeProperty = ep;
      }
View Full Code Here

      FlowNode edgeTargetFlowNode = nodeGraph.getEdgeTarget( processEdge );

      FlowElement flowElement = processEdge.getFlowElement();
      List<FlowNode> sourceNodes = nodeGraph.getElementSourceProcesses( flowElement );

      EdgeProperty edgeProperty = createEdgeProperty( initializedConfig, processEdge );

      Vertex targetVertex = vertexMap.get( edgeTargetFlowNode );

      if( sourceNodes.size() == 1 || flowElement instanceof CoGroup || flowElement instanceof Boundary ) // todo: create group vertices around incoming ordinal
        {
        FlowNode edgeSourceFlowNode = nodeGraph.getEdgeSource( processEdge );
        Vertex sourceVertex = vertexMap.get( edgeSourceFlowNode );

        LOG.debug( "adding edge between: {} and {}", sourceVertex, targetVertex );

        dag.addEdge( Edge.create( sourceVertex, targetVertex, edgeProperty ) );
        }
      else if( flowElement instanceof GroupBy || flowElement instanceof Merge ) // merge - source nodes > 1
        {
        List<String> sourceVerticesIDs = new ArrayList<>();
        List<Vertex> sourceVertices = new ArrayList<>();

        for( FlowNode edgeSourceFlowNode : sourceNodes )
          {
          sourceVerticesIDs.add( edgeSourceFlowNode.getID() );
          sourceVertices.add( vertexMap.get( edgeSourceFlowNode ) );
          processedEdges.add( nodeGraph.getEdge( edgeSourceFlowNode, edgeTargetFlowNode ) );
          }

        VertexGroup vertexGroup = dag.createVertexGroup( edgeTargetFlowNode.getID(), sourceVertices.toArray( new Vertex[ sourceVertices.size() ] ) );

        String inputClassName = flowElement instanceof Group ? OrderedGroupedMergedKVInput.class.getName() : ConcatenatedMergedKeyValueInput.class.getName();

        InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() );

        LOG.info( "adding grouped edge between: {} and {}", Util.join( sourceVerticesIDs, "," ), targetVertex.getName() );
        dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) );
        }
      else
View Full Code Here

TOP

Related Classes of org.apache.tez.dag.api.EdgeProperty

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.