Package org.apache.hadoop.mapreduce.v2.app.TestRecovery

Examples of org.apache.hadoop.mapreduce.v2.app.TestRecovery.MRAppWithHistory


  @Test
  public void testAMInfosWithoutRecoveryEnabled() throws Exception {
    int runCount = 0;
    MRApp app =
        new MRAppWithHistory(1, 0, false, this.getClass().getName(), true,
          ++runCount);
    Configuration conf = new Configuration();
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    Job job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);

    long am1StartTime = app.getAllAMInfos().get(0).getStartTime();

    Assert.assertEquals("No of tasks not correct", 1, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task mapTask = it.next();
    app.waitForState(mapTask, TaskState.RUNNING);
    TaskAttempt taskAttempt = mapTask.getAttempts().values().iterator().next();
    app.waitForState(taskAttempt, TaskAttemptState.RUNNING);

    // stop the app
    app.stop();

    // rerun
    app =
        new MRAppWithHistory(1, 0, false, this.getClass().getName(), false,
          ++runCount);
    conf = new Configuration();
    // in rerun the AMInfo will be recovered from previous run even if recovery
    // is not enabled.
    conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, false);
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);
    Assert.assertEquals("No of tasks not correct", 1, job.getTasks().size());
    it = job.getTasks().values().iterator();
    mapTask = it.next();
    // There should be two AMInfos
    List<AMInfo> amInfos = app.getAllAMInfos();
    Assert.assertEquals(2, amInfos.size());
    AMInfo amInfoOne = amInfos.get(0);
    Assert.assertEquals(am1StartTime, amInfoOne.getStartTime());
    app.stop();
  }
View Full Code Here


  @Test
  public void testAMInfosWithoutRecoveryEnabled() throws Exception {
    int runCount = 0;
    MRApp app =
        new MRAppWithHistory(1, 0, false, this.getClass().getName(), true,
          ++runCount);
    Configuration conf = new Configuration();
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    Job job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);

    long am1StartTime = app.getAllAMInfos().get(0).getStartTime();

    Assert.assertEquals("No of tasks not correct", 1, job.getTasks().size());
    Iterator<Task> it = job.getTasks().values().iterator();
    Task mapTask = it.next();
    app.waitForState(mapTask, TaskState.RUNNING);
    TaskAttempt taskAttempt = mapTask.getAttempts().values().iterator().next();
    app.waitForState(taskAttempt, TaskAttemptState.RUNNING);

    // stop the app
    app.stop();

    // rerun
    app =
        new MRAppWithHistory(1, 0, false, this.getClass().getName(), false,
          ++runCount);
    conf = new Configuration();
    // in rerun the AMInfo will be recovered from previous run even if recovery
    // is not enabled.
    conf.setBoolean(MRJobConfig.MR_AM_JOB_RECOVERY_ENABLE, false);
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    job = app.submit(conf);
    app.waitForState(job, JobState.RUNNING);
    Assert.assertEquals("No of tasks not correct", 1, job.getTasks().size());
    it = job.getTasks().values().iterator();
    mapTask = it.next();
    // There should be two AMInfos
    List<AMInfo> amInfos = app.getAllAMInfos();
    Assert.assertEquals(2, amInfos.size());
    AMInfo amInfoOne = amInfos.get(0);
    Assert.assertEquals(am1StartTime, amInfoOne.getStartTime());
    app.stop();
  }
View Full Code Here

          } catch (InterruptedException e) {
          }
        }
      }
    };
    CommitterEventHandler commitHandler =
        createCommitterEventHandler(dispatcher, committer);
    commitHandler.init(conf);
    commitHandler.start();

    JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
    JobId jobId = job.getID();
    job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
    assertJobState(job, JobStateInternal.INITED);
    job.handle(new JobStartEvent(jobId));
    assertJobState(job, JobStateInternal.SETUP);

    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    assertJobState(job, JobStateInternal.KILL_ABORT);

    job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
    assertJobState(job, JobStateInternal.KILLED);
    dispatcher.stop();
    commitHandler.stop();
  }
View Full Code Here

    dispatcher.start();

    OutputCommitter committer = mock(OutputCommitter.class);
    doThrow(new IOException("forcefail"))
      .when(committer).setupJob(any(JobContext.class));
    CommitterEventHandler commitHandler =
        createCommitterEventHandler(dispatcher, committer);
    commitHandler.init(conf);
    commitHandler.start();

    AppContext mockContext = mock(AppContext.class);
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
    JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
    JobId jobId = job.getID();
    job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
    assertJobState(job, JobStateInternal.INITED);
    job.handle(new JobStartEvent(jobId));
    assertJobState(job, JobStateInternal.FAILED);

    job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED));
    assertJobState(job, JobStateInternal.FAILED);
    job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
    assertJobState(job, JobStateInternal.FAILED);
    job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED));
    assertJobState(job, JobStateInternal.FAILED);
    job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE));
    assertJobState(job, JobStateInternal.FAILED);
    Assert.assertEquals(JobState.RUNNING, job.getState());
    when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
    Assert.assertEquals(JobState.FAILED, job.getState());

    dispatcher.stop();
    commitHandler.stop();
  }
View Full Code Here

    };
    ApplicationAttemptId id =
      ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(id);
    CommitterEventHandler handler =
        new CommitterEventHandler(appContext, committer, heartbeatHandler);
    dispatcher.register(CommitterEventType.class, handler);
    return handler;
  }
View Full Code Here

          }
        }

        //If all tasks are already done, we should go directly to FAIL_ABORT
        if(allDone) {
          job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
            job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED)
          );
          return JobStateInternal.FAIL_ABORT;
        }
View Full Code Here

      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      JobCommitFailedEvent jcfe = (JobCommitFailedEvent)event;
      job.addDiagnostic("Job commit failed: " + jcfe.getMessage());
      job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
          job.jobContext,
          org.apache.hadoop.mapreduce.JobStatus.State.FAILED));
    }
View Full Code Here

  private static class KilledDuringCommitTransition implements
      SingleArcTransition<JobImpl, JobEvent> {
    @Override
    public void transition(JobImpl job, JobEvent event) {
      job.setFinishTime();
      job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
          job.jobContext,
          org.apache.hadoop.mapreduce.JobStatus.State.KILLED));
    }
View Full Code Here

      TaskCompletedTransition {
    @Override
    protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) {
      if (job.completedTaskCount == job.tasks.size()) {
        job.setFinishTime();
        job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
            job.jobContext,
            org.apache.hadoop.mapreduce.JobStatus.State.KILLED));
        return JobStateInternal.KILL_ABORT;
      }
      //return the current state, Job not finished yet
View Full Code Here

 
  protected JobStateInternal checkReadyForCommit() {
    JobStateInternal currentState = getInternalState();
    if (completedTaskCount == tasks.size()
        && currentState == JobStateInternal.RUNNING) {
      eventHandler.handle(new CommitterJobCommitEvent(jobId, getJobContext()));
      return JobStateInternal.COMMITTING;
    }
    // return the current state as job not ready to commit yet
    return getInternalState();
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.app.TestRecovery.MRAppWithHistory

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.