Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.DAG.addVertex()

264
265
266
267
268
269
270
271
272
273
274
      } else {
        // Regular vertices
        JobConf wxConf = utils.initializeVertexConf(conf, w);
        Vertex wx = utils.createVertex(wxConf, w, scratchDir, appJarLr,
          additionalLr, fs, ctx, !isFinal, work);
        dag.addVertex(wx);
        utils.addCredentials(w, dag);
        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName());
        workToVertex.put(w, wx);
        workToConf.put(w, wxConf);
       
View Full Code Here
487
488
489
490
491
492
493
494
495
496
497
          i == 0 ? mapInputLocations : reduceInputLocations, i,
          stageConfs.length);
    }

    for (int i = 0; i < vertices.length; i++) {
      dag.addVertex(vertices[i]);
      if (i > 0) {
        // Set edge conf based on Input conf (compression etc properties for MapReduce are
        // w.r.t Outputs - MAP_OUTPUT_COMPRESS for example)
        Map<String, String> partitionerConf = null;
        if (stageConfs[i-1] != null) {
View Full Code Here
564
565
566
567
568
569
570
571
572
573
574
   
    verifySessionStateForSubmission();
   
    DAG dag = org.apache.tez.dag.api.DAG.create(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX + "_"
        + preWarmDAGCounter++);
    dag.addVertex(preWarmVertex);

    try {
      waitTillReady();
    } catch (InterruptedException e) {
      throw new IOException("Interrupted while waiting for AM to become available", e);
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
    }
    DAG dag = DAG.create(name);
    Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(Edge.create(v1, v3,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
      payload = TezUtils.createUserPayloadFromConf(conf);
    }
    DAG dag = DAG.create(name);
    Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    Vertex v4 = Vertex.create("v4", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v5 = Vertex.create("v5", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v6 = Vertex.create("v6", TestProcessor.getProcDesc(payload), taskCount, defaultResource);

    // add vertex not in the topological order, since we are using this dag for testing vertex topological order
    dag.addVertex(v4)
      .addVertex(v5)
      .addVertex(v6)
      .addVertex(v1)
      .addVertex(v2)
      .addVertex(v3)
View Full Code Here
339
340
341
342
343
344
345
346
347
348
349
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("2").getBytes()))));
    v3.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("3").getBytes()))));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(Edge.create(v1, v2,
        EdgeProperty.create(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
View Full Code Here
187
188
189
190
191
192
193
194
195
196
197
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
View Full Code Here
236
237
238
239
240
241
242
243
244
245
246
      for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
                SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
            Resource.newInstance(1024, 1));
        dag.addVertex(vertex);

        DAGClient dagClient = null;
        try {
          dagClient = tezSession.submitDAG(dag);
          if (dagIndex > 1) {
View Full Code Here
275
276
277
278
279
280
281
282
283
284
285
    DAG dag = DAG.create("TezSleepProcessor");
    Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
            SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path stagingDir = new Path(TEST_ROOT_DIR, "testNonDefaultFSStagingDir"
        + String.valueOf(random.nextInt(100000)));
    FileSystem localFs = FileSystem.getLocal(tezConf);
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.