Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.DAG.addVertex()

55
56
57
58
59
60
61
62
63
64
65
    }
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
View Full Code Here
61
62
63
64
65
66
67
68
69
70
71
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
            TestInput.getInputDesc(payload))));
    dag.addVertex(v3).addEdge(new Edge(v2, v3,
            new EdgeProperty(DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                TestOutput.getOutputDesc(payload),
                TestInput.getInputDesc(payload))));
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
    DAG dag = new DAG("TezSleepProcessor");
    Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
        SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(new Random()
        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
View Full Code Here
149
150
151
152
153
154
155
156
157
158
159
    DAG dag = new DAG("TezSleepProcessor");
    Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
        SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
        Resource.newInstance(1024, 1));
    dag.addVertex(vertex);

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    Path stagingDir = new Path(TEST_ROOT_DIR, "testNonDefaultFSStagingDir"
        + String.valueOf(new Random().nextInt(100000)));
    FileSystem localFs = FileSystem.getLocal(tezConf);
View Full Code Here
55
56
57
58
59
60
61
62
63
64
65
    }
    DAG dag = new DAG(name);
    Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
View Full Code Here
223
224
225
226
227
228
229
230
231
232
233
    Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
        DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
        SchedulingType.SEQUENTIAL, new OutputDescriptor(
            OnFileUnorderedKVOutput.class.getName()), new InputDescriptor(
            ShuffledUnorderedKVInput.class.getName())));
    dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);

    LOG.info("Submitting DAG to Tez Session");
    DAGClient dagClient = tezSession.submitDAG(dag);
    LOG.info("Submitted DAG to Tez Session");
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
      } else {
        // Regular vertices
        JobConf wxConf = utils.initializeVertexConf(conf, w);
        Vertex wx = utils.createVertex(wxConf, w, scratchDir, appJarLr,
          additionalLr, fs, ctx, !isFinal, work);
        dag.addVertex(wx);
        utils.addCredentials(w, dag);
        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName());
        workToVertex.put(w, wx);
        workToConf.put(w, wxConf);
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
        // Regular vertices
        JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
        Vertex wx =
            utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
                work, work.getVertexType(w));
        dag.addVertex(wx);
        utils.addCredentials(w, dag);
        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName());
        workToVertex.put(w, wx);
        workToConf.put(w, wxConf);
View Full Code Here
261
262
263
264
265
266
267
268
269
270
271
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("2").getBytes()));
    v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
        FailOnAttemptVertexManagerPlugin.class.getName())
        .setUserPayload(new String("3").getBytes()));
    dag.addVertex(v1).addVertex(v2).addVertex(v3);
    dag.addEdge(new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER,
            DataSourceType.PERSISTED,
            SchedulingType.SEQUENTIAL,
            TestOutput.getOutputDesc(payload),
View Full Code Here
349
350
351
352
353
354
355
356
357
358
359
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")),
            new InputDescriptor("merge.class"));
   
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    return dag.createDag(conf);
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.