Package org.apache.tez.dag.api

Examples of org.apache.tez.dag.api.UserPayload

165
166
167
168
169
170
171
172
173
174
175
          newEdgeManagers.clear();
          for (Entry<String, EdgeManagerPluginDescriptor> entry :
              ((Map<String, EdgeManagerPluginDescriptor>)invocation.getArguments()[2]).entrySet()) {


            final UserPayload userPayload = entry.getValue().getUserPayload();
            EdgeManagerPluginContext emContext = new EdgeManagerPluginContext() {
              @Override
              public UserPayload getUserPayload() {
                return userPayload == null ? null : userPayload;
              }
View Full Code Here
529
530
531
532
533
534
535
536
537
538
539
  private ShuffleVertexManager createManager(Configuration conf,
      VertexManagerPluginContext context, float min, float max) {
    conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min);
    conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max);
    UserPayload payload;
    try {
      payload = TezUtils.createUserPayloadFromConf(conf);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
View Full Code Here
214
215
216
217
218
219
220
221
222
223
224
    vertices.add(ivertex);

    ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
    finalReduceConf.writeXml(finalReduceOutputStream);
    String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
    UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
    Vertex finalReduceVertex = Vertex.create("finalreduce",
        ProcessorDescriptor.create(
            ReduceProcessor.class.getName())
            .setUserPayload(finalReducePayload)
            .setHistoryText(finalReduceStageHistoryText), 1);
View Full Code Here
105
106
107
108
109
110
111
112
113
114
115
    TezDAGID dagID = TezDAGID.getInstance("2000", 1, 1);
    TezVertexID vertexID = TezVertexID.getInstance(dagID, 1);
    TezTaskID taskID = TezTaskID.getInstance(vertexID, 1);
    TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
    TezCounters counters = new TezCounters();
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    RuntimeTask runtimeTask = mock(RuntimeTask.class);
   
    int shufflePort = 2112;
    Map<String, String> auxEnv = new HashMap<String, String>();
    ByteBuffer bb = ByteBuffer.allocate(4);
View Full Code Here
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
    commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);

    List<Vertex> vertices = new ArrayList<Vertex>();

   
    UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf);
    int numTasks = generateSplitsInAM ? -1 : numMapper;

    Vertex mapVertex = Vertex.create("map", ProcessorDescriptor.create(
        MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
        .addTaskLocalFiles(commonLocalResources);
    mapVertex.addDataSource("MRInput", dataSource);
    vertices.add(mapVertex);

    if (iReduceStagesCount > 0
        && numIReducer > 0) {
      for (int i = 0; i < iReduceStagesCount; ++i) {
        Configuration iconf =
            intermediateReduceStageConfs[i];
        UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
        Vertex ivertex = Vertex.create("ireduce" + (i + 1),
            ProcessorDescriptor.create(ReduceProcessor.class.getName()).
                setUserPayload(iReduceUserPayload), numIReducer);
        ivertex.addTaskLocalFiles(commonLocalResources);
        vertices.add(ivertex);
      }
    }

    Vertex finalReduceVertex = null;
    if (numReducer > 0) {
      UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
      finalReduceVertex = Vertex.create("reduce", ProcessorDescriptor.create(
          ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
      finalReduceVertex.addTaskLocalFiles(commonLocalResources);
      finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigBuilder(finalReduceConf,
          NullOutputFormat.class).build());
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
  @Test
  public void testPayloadToAndFromConf() throws IOException {
    Configuration conf = getConf();
    Assert.assertEquals(conf.size(), 6);
    UserPayload bConf = TezUtils.createUserPayloadFromConf(conf);
    conf.clear();
    Assert.assertEquals(conf.size(), 0);
    conf = TezUtils.createConfFromUserPayload(bConf);
    Assert.assertEquals(conf.size(), 6);
    checkConf(conf);
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    Configuration stage2Conf = new JobConf(conf);

    stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
    stage2Conf.setBoolean("mapred.mapper.new-api", false);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    // Setup stage1 Vertex
    Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create(
        FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
        .addTaskLocalFiles(commonLocalResources);
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
  private DAG createDAG(TezConfiguration conf, int numTasks, int maxSleepTimeMillis,
                        int payloadSize, String mode) throws IOException {

    Map<String, LocalResource> localResourceMap = new HashMap<String, LocalResource>();
    UserPayload payload =
        createUserPayload(conf, maxSleepTimeMillis, payloadSize, mode, localResourceMap);

    Vertex vertex = Vertex.create("RPCLoadVertex",
        ProcessorDescriptor.create(RPCSleepProcessor.class.getName()).setUserPayload(
            payload), numTasks).addTaskLocalFiles(localResourceMap);
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
      if (numOneToOneTasks < 1) {
        numOneToOneTasks = 1;
      }
    }
    byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
    UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte));

    System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");

    Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create(
        InputProcessor.class.getName()), numBroadcastTasks);
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    Configuration stage2Conf = new JobConf(conf);
    stage2Conf.set(FileOutputFormat.OUTDIR, outputPath);
    stage2Conf.setBoolean("mapred.mapper.new-api", false);

    UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
    // Setup stage1 Vertex
    Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create(
        FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
        .addTaskLocalFiles(commonLocalResources);
View Full Code Here
TOP

Related Classes of org.apache.tez.dag.api.UserPayload

  • cascading.flow.tez.Hadoop2TezFlowStep
  • org.apache.hadoop.hive.ql.exec.tez.CustomPartitionVertex
  • org.apache.hadoop.hive.ql.exec.tez.DagUtils
  • org.apache.hadoop.hive.ql.exec.tez.HivePreWarmProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.runtime.PigProcessor
  • org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder
  • org.apache.pig.tools.pigstats.tez.TezDAGStats
  • org.apache.tez.common.TestTezUtils
  • org.apache.tez.dag.app.dag.impl.Edge
  • org.apache.tez.dag.app.dag.impl.TestVertexImpl
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.