Examples of org.apache.tez.client.TezClient.start()

org.apache.tez.client.TezClient.start()
Start the client. This establishes a connection to the YARN cluster. In session mode, this start the App Master thats runs all the DAGs in the session. @throws TezException @throws IOException

        .nextInt(100000))));
    remoteFs.mkdirs(remoteStagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {

View Full Code Here

      Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String.valueOf(random
          .nextInt(100000))));
      remoteFs.mkdirs(remoteStagingDir);
      tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
      tezSession = TezClient.create("OrderedWordCountSession", tezConf, true);
      tezSession.start();


      SleepProcessorConfig spConf = new SleepProcessorConfig(1);
      for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {
        DAG dag = DAG.create("TezSleepProcessor");
        Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(

View Full Code Here

    stagingDir = localFs.makeQualified(stagingDir);
    localFs.mkdirs(stagingDir);
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());


    TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {

View Full Code Here

    tezConf.set(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR,
        localFs.makeQualified(historyLogDir).toString());


    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    TezClient tezSession = TezClient.create("TezSleepProcessorHistoryLogging", tezConf);
    tezSession.start();


    DAGClient dagClient = tezSession.submitDAG(dag);


    DAGStatus dagStatus = dagClient.getDAGStatus(null);
    while (!dagStatus.isCompleted()) {

View Full Code Here

    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
    TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
    TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf);
    session.start();


    // DAG should fail as it never completes on the first attempt
    DAG dag = MultiAttemptDAG.createDAG("TestSingleAttemptDAG", null);
    runDAGAndVerify(dag, State.FAILED, session);
    session.stop();

View Full Code Here

    remoteFs.mkdirs(remoteStagingDir);
    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("testrelocalizationsession", tezConf, true);
    tezSession.start();
    Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
    return tezSession;
  }


  // Submits a DAG to AM via RPC after AM has started

View Full Code Here

        mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
        remoteStagingDir.toString());


    TezClient tezSession = TezClient.create("testsession", tezConf, true);
    tezSession.start();
    Assert.assertEquals(TezAppMasterStatus.INITIALIZING,
        tezSession.getAppMasterStatus());


    State finalState = testMRRSleepJobDagSubmitCore(true, false, false,
        tezSession, false, null, null);

View Full Code Here

        tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
      } else {
        tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
      }
      tezSession = TezClient.create("testsession", tempTezconf);
      tezSession.start();
    } else {
      tezSession = reUseTezSession;
    }
    if(!dagViaRPC) {
      // TODO Use utility method post TEZ-205 to figure out AM arguments etc.

View Full Code Here

    return execute(args, tezConf, tezClient);
  }
  
  private TezClient createTezClient(TezConfiguration tezConf) throws TezException, IOException {
    TezClient tezClient = TezClient.create("JoinValidate", tezConf);
    tezClient.start();
    return tezClient;
  }


  private int execute(String[] args, TezConfiguration tezConf, TezClient tezClient)
      throws IOException, TezException, InterruptedException {

View Full Code Here

    // mode can be changed via configuration. However if the application wants to run exclusively in 
    // session mode then it can do so in code directly using the appropriate constructor
    
    // tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); // via config OR via code
    TezClient tezClient = TezClient.create("SimpleSessionExample", tezConf, true);
    tezClient.start();
    
    // Session pre-warming allows the user to hide initial startup, resource acquisition latency etc.
    // by pre-allocating execution resources in the Tez session. They can run initialization logic 
    // in these pre-allocated resources (containers) to pre-warm the containers.
    // In between DAG executions, the session can hold on to a minimum number of containers.

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.