Package org.apache.hadoop.yarn.api

Examples of org.apache.hadoop.yarn.api.ClientRMProtocol


        attempt.getAppAttemptState());
  }

  // get new application id
  public GetNewApplicationResponse getNewAppId() throws Exception {
    ClientRMProtocol client = getClientRMService();
    return client.getNewApplication(Records
        .newRecord(GetNewApplicationRequest.class));
  }
View Full Code Here


    return submitApp(masterMemory, name, user, acls, false, queue);
 

  public RMApp submitApp(int masterMemory, String name, String user,
      Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue) throws Exception {
    ClientRMProtocol client = getClientRMService();
    GetNewApplicationResponse resp = client.getNewApplication(Records
        .newRecord(GetNewApplicationRequest.class));
    ApplicationId appId = resp.getApplicationId();

    SubmitApplicationRequest req = Records
        .newRecord(SubmitApplicationRequest.class);
    ApplicationSubmissionContext sub = Records
        .newRecord(ApplicationSubmissionContext.class);
    sub.setApplicationId(appId);
    sub.setApplicationName(name);
    sub.setUser(user);
    if(unmanaged) {
      sub.setUnmanagedAM(true);
    }
    if (queue != null) {
      sub.setQueue(queue);
    }
    ContainerLaunchContext clc = Records
        .newRecord(ContainerLaunchContext.class);
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(masterMemory);
    clc.setResource(capability);
    clc.setApplicationACLs(acls);
    sub.setAMContainerSpec(clc);
    req.setApplicationSubmissionContext(sub);

    UserGroupInformation fakeUser =
      UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"});
    PrivilegedAction<SubmitApplicationResponse> action =
      new PrivilegedAction<SubmitApplicationResponse>() {
      ClientRMProtocol client;
      SubmitApplicationRequest req;
      @Override
      public SubmitApplicationResponse run() {
        try {
          return client.submitApplication(req);
        } catch (YarnRemoteException e) {
          e.printStackTrace();
        }
        return null;
      }
View Full Code Here

    Assert.assertEquals("Node state is not correct (timedout)", finalState,
        node.getState());
  }

  public void killApp(ApplicationId appId) throws Exception {
    ClientRMProtocol client = getClientRMService();
    KillApplicationRequest req = Records
        .newRecord(KillApplicationRequest.class);
    req.setApplicationId(appId);
    client.forceKillApplication(req);
  }
View Full Code Here

    final ClientRMService clientRMService = new ClientRMServiceForTest(conf,
        scheduler, rmDtSecretManager);
    clientRMService.init(conf);
    clientRMService.start();

    ClientRMProtocol clientRMWithDT = null;
    try {

      // Create a user for the renewr and fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
      Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
      // Default realm is APACHE.ORG
      loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);

     
      DelegationToken token = getDelegationToken(loggedInUser, clientRMService,
          loggedInUser.getShortUserName());
      long tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);
      // Now try talking to RMService using the delegation token
      clientRMWithDT = getClientRMProtocolWithDT(token,
          clientRMService.getBindAddress(), "loginuser1", conf);

      GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);
     
      try {
        clientRMWithDT.getNewApplication(request);
      } catch (UndeclaredThrowableException e) {
        fail("Unexpected exception" + e);
      }
     
      // Renew after 50% of token age.
      while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
        Thread.sleep(500l);
      }
      long nextExpTime = renewDelegationToken(loggedInUser, clientRMService, token);
      long renewalTime = System.currentTimeMillis();
      LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: "
          + nextExpTime);

      // Wait for first expiry, but before renewed expiry.
      while (System.currentTimeMillis() > tokenFetchTime + initialInterval
          && System.currentTimeMillis() < nextExpTime) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
     
      // Valid token because of renewal.
      try {
        clientRMWithDT.getNewApplication(request);
      } catch (UndeclaredThrowableException e) {
        fail("Unexpected exception" + e);
      }
     
      // Wait for expiry.
      while(System.currentTimeMillis() < renewalTime + renewInterval) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
      LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
      // Token should have expired.     
      try {
        clientRMWithDT.getNewApplication(request);
        fail("Should not have succeeded with an expired token");
      } catch (UndeclaredThrowableException e) {
        assertTrue(e.getCause().getMessage().contains("is expired"));
      }

      // Test cancellation
      // Stop the existing proxy, start another.
      if (clientRMWithDT != null) {
        RPC.stopProxy(clientRMWithDT);
        clientRMWithDT = null;
      }
      token = getDelegationToken(loggedInUser, clientRMService,
          loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);
      // Now try talking to RMService using the delegation token
      clientRMWithDT = getClientRMProtocolWithDT(token,
          clientRMService.getBindAddress(), "loginuser2", conf);

      request = Records.newRecord(GetNewApplicationRequest.class);
     
      try {
        clientRMWithDT.getNewApplication(request);
      } catch (UndeclaredThrowableException e) {
        fail("Unexpected exception" + e);
      }
      cancelDelegationToken(loggedInUser, clientRMService, token);
      if (clientRMWithDT != null) {
        RPC.stopProxy(clientRMWithDT);
        clientRMWithDT = null;
      }
     
      // Creating a new connection.
      clientRMWithDT = getClientRMProtocolWithDT(token,
          clientRMService.getBindAddress(), "loginuser2", conf);
      LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
      // Verify cancellation worked.
      try {
        clientRMWithDT.getNewApplication(request);
        fail("Should not have succeeded with a cancelled delegation token");
      } catch (UndeclaredThrowableException e) {
      }

View Full Code Here

    UserGroupInformation ugi = UserGroupInformation
        .createRemoteUser(user);
    ugi.addToken(ProtoUtils.convertFromProtoFormat(token, rmAddress));

    final YarnRPC rpc = YarnRPC.create(conf);
    ClientRMProtocol clientRMWithDT = ugi
        .doAs(new PrivilegedAction<ClientRMProtocol>() {
          @Override
          public ClientRMProtocol run() {
            return (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
                rmAddress, conf);
View Full Code Here

      rmDTToken.setIdentifier(ByteBuffer.wrap(new byte[2]));
      rmDTToken.setKind("Testclusterkind");
      rmDTToken.setPassword(ByteBuffer.wrap("testcluster".getBytes()));
      rmDTToken.setService("0.0.0.0:8032");
      getDTResponse.setRMDelegationToken(rmDTToken);
      final ClientRMProtocol cRMProtocol = mock(ClientRMProtocol.class);
      when(cRMProtocol.getDelegationToken(any(
          GetDelegationTokenRequest.class))).thenReturn(getDTResponse);
      ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate(
          new YarnConfiguration(conf)) {
        @Override
        public synchronized void start() {
View Full Code Here

  /**
   * Tests that getRootQueues makes a request for the (recursive) child queues
   */
  @Test
  public void testGetRootQueues() throws IOException, InterruptedException {
    final ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class);
    GetQueueInfoResponse response = Mockito.mock(GetQueueInfoResponse.class);
    org.apache.hadoop.yarn.api.records.QueueInfo queueInfo =
      Mockito.mock(org.apache.hadoop.yarn.api.records.QueueInfo.class);
    Mockito.when(response.getQueueInfo()).thenReturn(queueInfo);
    Mockito.when(applicationsManager.getQueueInfo(Mockito.any(
      GetQueueInfoRequest.class))).thenReturn(response);

    ResourceMgrDelegate delegate = new ResourceMgrDelegate(
      new YarnConfiguration()) {
      @Override
View Full Code Here

      argument.getValue().getRecursive());
  }

  @Test
  public void tesAllJobs() throws Exception {
    final ClientRMProtocol applicationsManager = Mockito.mock(ClientRMProtocol.class);
    GetAllApplicationsResponse allApplicationsResponse = Records
        .newRecord(GetAllApplicationsResponse.class);
    List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
    applications.add(getApplicationReport(YarnApplicationState.FINISHED,
        FinalApplicationStatus.FAILED));
    applications.add(getApplicationReport(YarnApplicationState.FINISHED,
        FinalApplicationStatus.SUCCEEDED));
    applications.add(getApplicationReport(YarnApplicationState.FINISHED,
        FinalApplicationStatus.KILLED));
    applications.add(getApplicationReport(YarnApplicationState.FAILED,
        FinalApplicationStatus.FAILED));
    allApplicationsResponse.setApplicationList(applications);
    Mockito.when(
        applicationsManager.getAllApplications(Mockito
            .any(GetAllApplicationsRequest.class))).thenReturn(
        allApplicationsResponse);
    ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
      new YarnConfiguration()) {
      @Override
View Full Code Here

  }

  @Test(timeout=20000)
  public void testResourceMgrDelegate() throws Exception {
    /* we not want a mock of resource mgr delegate */
    final ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class);
    ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
      @Override
      public synchronized void start() {
        this.rmClient = clientRMProtocol;
      }
    };
    /* make sure kill calls finish application master */
    when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
    .thenReturn(null);
    delegate.killApplication(appId);
    verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));

    /* make sure getalljobs calls get all applications */
    when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))).
    thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class));
    delegate.getAllJobs();
    verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class));

    /* make sure getapplication report is called */
    when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
    .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
    delegate.getApplicationReport(appId);
    verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));

    /* make sure metrics is called */
    GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
        (GetClusterMetricsResponse.class);
    clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
        YarnClusterMetrics.class));
    when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
    .thenReturn(clusterMetricsResponse);
    delegate.getClusterMetrics();
    verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));

    when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
    thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
    delegate.getActiveTrackers();
    verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
   
    GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
        GetNewApplicationResponse.class);
    newAppResponse.setApplicationId(appId);
    when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
    thenReturn(newAppResponse);
    delegate.getNewJobID();
    verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
   
    GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
        GetQueueInfoResponse.class);
    queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
    when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
    thenReturn(queueInfoResponse);
    delegate.getQueues();
    verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));

    GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
        GetQueueUserAclsInfoResponse.class);
    when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
    .thenReturn(aclResponse);
    delegate.getQueueAclsForCurrentUser();
    verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
  }
View Full Code Here

  }

  @Test
  public void testResourceMgrDelegate() throws Exception {
    /* we not want a mock of resourcemgr delegate */
    ClientRMProtocol clientRMProtocol = mock(ClientRMProtocol.class);
    ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf, clientRMProtocol);
    /* make sure kill calls finish application master */
    when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
    .thenReturn(null);
    delegate.killApplication(appId);
    verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));

    /* make sure getalljobs calls get all applications */
    when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))).
    thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class));
    delegate.getAllJobs();
    verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class));

    /* make sure getapplication report is called */
    when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
    .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
    delegate.getApplicationReport(appId);
    verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));

    /* make sure metrics is called */
    GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
        (GetClusterMetricsResponse.class);
    clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
        YarnClusterMetrics.class));
    when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
    .thenReturn(clusterMetricsResponse);
    delegate.getClusterMetrics();
    verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));

    when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
    thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
    delegate.getActiveTrackers();
    verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
   
    GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
        GetNewApplicationResponse.class);
    newAppResponse.setApplicationId(appId);
    when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
    thenReturn(newAppResponse);
    delegate.getNewJobID();
    verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
   
    GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
        GetQueueInfoResponse.class);
    queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
    when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
    thenReturn(queueInfoResponse);
    delegate.getQueues();
    verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));

    GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
        GetQueueUserAclsInfoResponse.class);
    when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
    .thenReturn(aclResponse);
    delegate.getQueueAclsForCurrentUser();
    verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.yarn.api.ClientRMProtocol

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.