Package org.apache.hadoop.mapreduce.test.system

Examples of org.apache.hadoop.mapreduce.test.system.TTClient


    //in both mapred-site.xml and taskcontroller.cfg.
    //This change can be done after cluster is brought up as
    //Linux task controller will access taskcontroller.cfg
    //when a job's task starts.
    for ( int i = 0;i < ttClients.size();i++ ) {
      TTClient ttClient = (TTClient)ttClients.get(i);
      String ttClientHostName = ttClient.getHostName();
      try {
        RemoteExecution rExec = new SSHRemoteExecution();
        rExec.executeCommand(ttClientHostName, userName,
          replaceTaskControllerCommand);
      } catch (Exception e) { e.printStackTrace(); };
View Full Code Here


   * @throws Exception in case of test errors
   */
  @Test
  public void testScriptTimeout() throws Exception {
    LOG.info("running testScriptTimeout");
    TTClient client = cluster.getTTClient();
    Configuration tConf= client.getProxy().getDaemonConf();
    int defaultTimeout = tConf.getInt("mapred.healthChecker.script.timeout", 0);
    tConf.set("mapred.task.tracker.report.address",
        cluster.getConf().get("mapred.task.tracker.report.address"));
    Assert.assertTrue("Health script timeout was not set",defaultTimeout != 0);    
    tConf.set("mapred.healthChecker.script.path", remotePath+File.separator+
        healthScriptTimeout);
    tConf.setInt("mapred.healthChecker.script.timeout", 100);
    tConf.setInt("mapred.healthChecker.interval",1000);   
    helper.copyFileToRemoteHost(healthScriptTimeout, client.getHostName(),
        remotePath, cluster);
    cluster.restartDaemonWithNewConfig(client, "mapred-site.xml", tConf,
        Role.TT);
    //make sure the TT is blacklisted
    helper.verifyTTBlackList(tConf, client, "Node health script timed out",
        cluster);
    //Now put back the task tracker in a health state
    cluster.restart(client, Role.TT);
    tConf = client.getProxy().getDaemonConf();
    //now do the opposite of blacklist verification
    helper.deleteFileOnRemoteHost(remotePath+File.separator+healthScriptTimeout,
        client.getHostName());
   
  }
View Full Code Here

   * @throws Exception in case of test errors
   */
  @Test
  public void testHealthScriptPathError() throws Exception {
    LOG.info("running testHealthScriptPathError");
    TTClient client = cluster.getTTClient();
    Configuration tConf= client.getProxy().getDaemonConf();   
    tConf.set("mapred.task.tracker.report.address",
        cluster.getConf().get("mapred.task.tracker.report.address"));
    String defaultHealthScript = tConf.get("mapred.healthChecker.script.path");
    Assert.assertTrue("Health script was not set", defaultHealthScript != null);   
    tConf.set("mapred.healthChecker.script.path", remotePath+File.separator+
        invalidHealthScript);
    tConf.setInt("mapred.healthChecker.interval",1000);
    cluster.restartDaemonWithNewConfig(client, "mapred-site.xml", tConf,
        Role.TT);
    //For a invalid health script the TT remains healthy
    helper.verifyTTNotBlackListed( client, tConf, cluster);
    cluster.restart(client, Role.TT);   
    tConf = client.getProxy().getDaemonConf();
  }
View Full Code Here

    LOG.info("Hostname of local machine: " + testRunningHostName);

    List<TTClient> ttClients = cluster.getTTClients();

    //One slave is got
    TTClient ttClient = (TTClient)ttClients.get(0);
    String ttClientHostName = ttClient.getHostName();

    //Hadoop Conf directory is got
    String hadoopConfDir = cluster.getConf().get(
        HadoopDaemonRemoteCluster.CONF_HADOOPCONFDIR);
View Full Code Here

        taskInfo = taskinfo;
        break;
      }
    }

    TTClient ttClient = cluster.getTTClientInstance(taskInfo);
    String pid = null;
    ttClient.kill();
    ttClient.waitForTTStop();
    ttClient.start();
    ttClient.waitForTTStart();

    count = 0;
    LOG.info("Waiting till the job is completed...");
    while (!jInfo.getStatus().isJobComplete()) {
      UtilsForTests.waitFor(1000);
View Full Code Here

  private List<TTClient> getTTClients(String[] taskTrackers)
      throws Exception {
    List<TTClient> ttClientList= new ArrayList<TTClient>();
    for (String taskTracker: taskTrackers) {
      taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
      TTClient ttClient = cluster.getTTClient(taskTracker);
      if (ttClient != null) {
        ttClientList.add(ttClient);
      }
    }
    return ttClientList;
View Full Code Here

              org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));

      // counter for job Loop
      countLoop++;

      TTClient tClient = null;
      JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
      LOG.info("jInfo is :" + jInfo);

      // Assert if jobInfo is null
      Assert.assertNotNull("jobInfo is null", jInfo);

      // Wait for the job to start running.
      count = 0;
      while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
        UtilsForTests.waitFor(10000);
        count++;
        jInfo = wovenClient.getJobInfo(rJob.getID());
        // If the count goes beyond a point, then break; This is to avoid
        // infinite loop under unforeseen circumstances. Testcase will anyway
        // fail later.
        if (count > 10) {
          Assert.fail("job has not reached running state for more than"
              + "100 seconds. Failing at this point");
        }
      }

      LOG.info("job id is :" + rJob.getID().toString());

      TaskInfo[] taskInfos =
          cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());

      boolean distCacheFileIsFound;

      for (TaskInfo taskInfo : taskInfos) {
        distCacheFileIsFound = false;
        String[] taskTrackers = taskInfo.getTaskTrackers();
        for (String taskTracker : taskTrackers) {
          // Formatting tasktracker to get just its FQDN
          taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
          LOG.info("taskTracker is :" + taskTracker);

          // This will be entered from the second job onwards
          if (countLoop > 1) {
            if (taskTracker != null) {
              continueLoop = taskTrackerCollection.contains(taskTracker);
            }
            if (!continueLoop) {
              break;
            }
          }

          // Collecting the tasktrackers
          if (taskTracker != null)
            taskTrackerCollection.add(taskTracker);

          // we have loopped through enough number of times to look for task
          // getting submitted on same tasktrackers.The same tasktracker
          // for subsequent jobs was not hit maybe because of many number
          // of tasktrackers. So, testcase has to stop here.
          if (countLoop > 2) {
            continueLoop = false;
          }

          tClient = cluster.getTTClient(taskTracker);

          // tClient maybe null because the task is already dead. Ex: setup
          if (tClient == null) {
            continue;
          }

          String[] localDirs = tClient.getMapredLocalDirs();
          int distributedFileCount = 0;
          // Go to every single path
          for (String localDir : localDirs) {
            // Public Distributed cache will always be stored under
            // mapre.local.dir/tasktracker/archive
            localDir =
                localDir
                    + Path.SEPARATOR
                    + TaskTracker.getPublicDistributedCacheDir();
            LOG.info("localDir is : " + localDir);

            // Get file status of all the directories
            // and files under that path.
            FileStatus[] fileStatuses =
                tClient.listStatus(localDir, true, true);
            for (FileStatus fileStatus : fileStatuses) {
              Path path = fileStatus.getPath();
              LOG.info("path is :" + path.toString());
              // Checking if the received path ends with
              // the distributed filename
View Full Code Here

              org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));

      // counter for job Loop
      countLoop++;

      TTClient tClient = null;
      JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
      LOG.info("jInfo is :" + jInfo);

      // Assert if jobInfo is null
      Assert.assertNotNull("jobInfo is null", jInfo);

      // Wait for the job to start running.
      count = 0;
      while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
        UtilsForTests.waitFor(10000);
        count++;
        jInfo = wovenClient.getJobInfo(rJob.getID());
        // If the count goes beyond a point, then break; This is to avoid
        // infinite loop under unforeseen circumstances. Testcase will anyway
        // fail later.
        if (count > 10) {
          Assert.fail("job has not reached running state for more than"
              + "100 seconds. Failing at this point");
        }
      }

      LOG.info("job id is :" + rJob.getID().toString());

      TaskInfo[] taskInfos =
          cluster.getJTClient().getProxy().getTaskInfo(rJob.getID());

      boolean distCacheFileIsFound;

      for (TaskInfo taskInfo : taskInfos) {
        distCacheFileIsFound = false;
        String[] taskTrackers = taskInfo.getTaskTrackers();
        for (String taskTracker : taskTrackers) {
          // Formatting tasktracker to get just its FQDN
          taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
          LOG.info("taskTracker is :" + taskTracker);

          // The tasktrackerFound variable is initialized
          taskTrackerFound = false;

          // This will be entered from the second job onwards
          if (countLoop > 1) {
            if (taskTracker != null) {
              continueLoop = taskTrackerCollection.contains(taskTracker);
            }
            if (continueLoop) {
              taskTrackerFound = true;
            }
          }
          // Collecting the tasktrackers
          if (taskTracker != null)
            taskTrackerCollection.add(taskTracker);

          // we have loopped through two times to look for task
          // getting submitted on same tasktrackers.The same tasktracker
          // for subsequent jobs was not hit maybe because of many number
          // of tasktrackers. So, testcase has to stop here.
          if (countLoop > 1) {
            continueLoop = false;
          }

          tClient = cluster.getTTClient(taskTracker);

          // tClient maybe null because the task is already dead. Ex: setup
          if (tClient == null) {
            continue;
          }

          String[] localDirs = tClient.getMapredLocalDirs();
          int distributedFileCount = 0;
          // Go to every single path
          for (String localDir : localDirs) {
            // Public Distributed cache will always be stored under
            // mapre.local.dir/tasktracker/archive
            localDir =
                localDir
                    + Path.SEPARATOR
                    + TaskTracker.getPublicDistributedCacheDir();
            LOG.info("localDir is : " + localDir);

            // Get file status of all the directories
            // and files under that path.
            FileStatus[] fileStatuses =
                tClient.listStatus(localDir, true, true);
            for (FileStatus fileStatus : fileStatuses) {
              Path path = fileStatus.getPath();
              LOG.info("path is :" + path.toString());
              // Checking if the received path ends with
              // the distributed filename
View Full Code Here

        cluster.getJTClient().getClient().getJob(org.apache.hadoop.mapred.JobID.downgrade(slpJob.getJobID()));

    JobStatus[] jobStatus = client.getAllJobs();
    String userName = jobStatus[0].getUsername();

    TTClient tClient = null;
    JobInfo jInfo = wovenClient.getJobInfo(rJob.getID());
    LOG.info("jInfo is :" + jInfo);

    //Assert if jobInfo is null
    Assert.assertNotNull("jobInfo is null", jInfo);

    //Wait for the job to start running.
    count = 0;
    while (jInfo.getStatus().getRunState() != JobStatus.RUNNING) {
      UtilsForTests.waitFor(10000);
      count++;
      jInfo = wovenClient.getJobInfo(rJob.getID());
      //If the count goes beyond a point, then Assert; This is to avoid
      //infinite loop under unforeseen circumstances.
      if (count > 10) {
        Assert.fail("job has not reached running state for more than" +
            "100 seconds. Failing at this point");
      }
    }

    LOG.info("job id is :" + rJob.getID().toString());

    TaskInfo[] taskInfos = cluster.getJTClient().getProxy()
           .getTaskInfo(rJob.getID());

    boolean distCacheFileIsFound;

    for (TaskInfo taskInfo : taskInfos) {
      distCacheFileIsFound = false;
      String[] taskTrackers = taskInfo.getTaskTrackers();

      for(String taskTracker : taskTrackers) {
        //Getting the exact FQDN of the tasktracker from
        //the tasktracker string.
        taskTracker = UtilsForTests.getFQDNofTT(taskTracker);
        tClient =  cluster.getTTClient(taskTracker);
        String[] localDirs = tClient.getMapredLocalDirs();
        int distributedFileCount = 0;
        String localDirOnly = null;

        boolean FileNotPresentForThisDirectoryPath = false;

        //Go to every single path
        for (String localDir : localDirs) {
          FileNotPresentForThisDirectoryPath = false;
          localDirOnly = localDir;

          //Public Distributed cache will always be stored under
          //mapred.local.dir/tasktracker/archive
          localDirOnly = localDir + Path.SEPARATOR + TaskTracker.SUBDIR +
              Path.SEPARATOR +  userName;

          //Private Distributed cache will always be stored under
          //mapre.local.dir/taskTracker/<username>/distcache
          //Checking for username directory to check if it has the
          //proper permissions
          localDir = localDir + Path.SEPARATOR +
                  TaskTracker.getPrivateDistributedCacheDir(userName);

          FileStatus fileStatusMapredLocalDirUserName = null;

          try {
            fileStatusMapredLocalDirUserName = tClient.
                            getFileStatus(localDirOnly, true);
          } catch (Exception e) {
            LOG.info("LocalDirOnly :" + localDirOnly + " not found");
            FileNotPresentForThisDirectoryPath = true;
          }

          //File will only be stored under one of the mapred.lcoal.dir
          //If other paths were hit, just continue 
          if (FileNotPresentForThisDirectoryPath)
            continue;

          Path pathMapredLocalDirUserName =
              fileStatusMapredLocalDirUserName.getPath();
          FsPermission fsPermMapredLocalDirUserName =
              fileStatusMapredLocalDirUserName.getPermission();
          Assert.assertTrue("Directory Permission is not 700",
            fsPermMapredLocalDirUserName.equals(new FsPermission("700")));

          //Get file status of all the directories
          //and files under that path.
          FileStatus[] fileStatuses = tClient.listStatus(localDir,
              true, true);
          for (FileStatus  fileStatus : fileStatuses) {
            Path path = fileStatus.getPath();
            LOG.info("path is :" + path.toString());
            //Checking if the received path ends with
View Full Code Here

    TaskInfo[] myTaskInfos = wovenClient.getTaskInfo(id);
    boolean isOneTaskStored = false;
    String sometaskpid = null;
    org.apache.hadoop.mapreduce.TaskAttemptID sometaskId = null;
    TTClient myCli = null;
    for (TaskInfo info : myTaskInfos) {
      if (!info.isSetupOrCleanup()) {
        String[] taskTrackers = info.getTaskTrackers();
        for (String taskTracker : taskTrackers) {
          TTInfo ttInfo = wovenClient.getTTInfo(taskTracker);
          TTClient ttCli = cluster.getTTClient(ttInfo.getStatus().getHost());
          TaskID taskId = info.getTaskID();
          TTTaskInfo ttTaskInfo = ttCli.getProxy().getTask(taskId);
          Assert.assertNotNull(ttTaskInfo);
          Assert.assertNotNull(ttTaskInfo.getConf());
          Assert.assertNotNull(ttTaskInfo.getUser());
          Assert.assertTrue(ttTaskInfo.getTaskStatus().getProgress() >= 0.0);
          Assert.assertTrue(ttTaskInfo.getTaskStatus().getProgress() <= 1.0);
          // Get the pid of the task attempt. The task need not have
          // reported the pid of the task by the time we are checking
          // the pid. So perform null check.
          String pid = ttTaskInfo.getPid();
          int i = 1;
          while (pid.isEmpty()) {
            Thread.sleep(1000);
            LOG.info("Waiting for task to report its pid back");
            ttTaskInfo = ttCli.getProxy().getTask(taskId);
            pid = ttTaskInfo.getPid();
            if (i == 40) {
              Assert.fail("The task pid not reported for 40 seconds.");
            }
            i++;
          }
          if (!isOneTaskStored) {
            sometaskpid = pid;
            sometaskId = ttTaskInfo.getTaskStatus().getTaskID();
            myCli = ttCli;
            isOneTaskStored = true;
          }
          LOG.info("verified task progress to be between 0 and 1");
          State state = ttTaskInfo.getTaskStatus().getRunState();
          if (ttTaskInfo.getTaskStatus().getProgress() < 1.0
              && ttTaskInfo.getTaskStatus().getProgress() > 0.0) {
            Assert.assertEquals(TaskStatus.State.RUNNING, state);
            LOG.info("verified run state as " + state);
          }
          FinishTaskControlAction action =
              new FinishTaskControlAction(org.apache.hadoop.mapred.TaskID
                  .downgrade(info.getTaskID()));
          ttCli.getProxy().sendAction(action);
        }
      }
    }
    rJob.killJob();
    int i = 1;
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.test.system.TTClient

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.