Package org.apache.hadoop.mapreduce.v2.hs

Examples of org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer


      try {
        getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
                        MiniYARNCluster.getHostname() + ":0");
        getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
                        MiniYARNCluster.getHostname() + ":0");
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here


          MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
              + ":0");
          getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
            hostname + ":0");
        }
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

   
    final long initialInterval = 10000l;
    final long maxLifetime= 20000l;
    final long renewInterval = 10000l;

    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
      jobHistoryServer = new JobHistoryServer() {
        protected void doSecureLogin(Configuration conf) throws IOException {
          // no keytab based login
        };

        protected JHSDelegationTokenSecretManager createJHSSecretManager(
            Configuration conf) {
          return new JHSDelegationTokenSecretManager(initialInterval,
              maxLifetime, renewInterval, 3600000);
        }
      };
//      final JobHistoryServer jobHistoryServer = jhServer;
      jobHistoryServer.init(conf);
      jobHistoryServer.start();
      final MRClientProtocol hsService = jobHistoryServer.getClientService()
          .getClientHandler();

      // Fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
      Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
      // Default realm is APACHE.ORG
      loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);


      DelegationToken token =
          getDelegationToken(loggedInUser, hsService,
              loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);

      // Now try talking to JHS using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "TheDarkLord", conf);

      GetJobReportRequest jobReportRequest =
          Records.newRecord(GetJobReportRequest.class);
      jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (YarnRemoteException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
   // Renew after 50% of token age.
      while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
        Thread.sleep(500l);
      }
      long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
      long renewalTime = System.currentTimeMillis();
      LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: "
          + nextExpTime);
     
      // Wait for first expiry, but before renewed expiry.
      while (System.currentTimeMillis() > tokenFetchTime + initialInterval
          && System.currentTimeMillis() < nextExpTime) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
     
      // Valid token because of renewal.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (YarnRemoteException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
      // Wait for expiry.
      while(System.currentTimeMillis() < renewalTime + renewInterval) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
      LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
      // Token should have expired.     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with an expired token");
      } catch (YarnRemoteException e) {
        assertTrue(e.getMessage().contains("is expired"));
      } catch (UndeclaredThrowableException ute) {
        assertTrue(ute.getCause().getMessage().contains("is expired"));
      }
     
      // Test cancellation
      // Stop the existing proxy, start another.
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
      token =
          getDelegationToken(loggedInUser, hsService,
              loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);
      // Now try talking to HSService using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);

     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (YarnRemoteException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
      cancelDelegationToken(loggedInUser, hsService, token);
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
     
      // Creating a new connection.
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);
      LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
      // Verify cancellation worked.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with a cancelled delegation token");
      } catch (YarnRemoteException e) {
      } catch (UndeclaredThrowableException ute) {
      }
    } finally {
      jobHistoryServer.stop();
    }
  }
View Full Code Here

          getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
              MiniYARNCluster.getHostname() + ":0");
          getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
              MiniYARNCluster.getHostname() + ":0");
        }
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

   
    final long initialInterval = 10000l;
    final long maxLifetime= 20000l;
    final long renewInterval = 10000l;

    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
      jobHistoryServer = new JobHistoryServer() {
        protected void doSecureLogin(Configuration conf) throws IOException {
          // no keytab based login
        };

        @Override
        protected JHSDelegationTokenSecretManager createJHSSecretManager(
            Configuration conf, HistoryServerStateStoreService store) {
          return new JHSDelegationTokenSecretManager(initialInterval,
              maxLifetime, renewInterval, 3600000, store);
        }

        @Override
        protected HistoryClientService createHistoryClientService() {
          return new HistoryClientService(historyContext,
            this.jhsDTSecretManager) {
            @Override
            protected void initializeWebApp(Configuration conf) {
              // Don't need it, skip.;
              }
          };
        }
      };
//      final JobHistoryServer jobHistoryServer = jhServer;
      jobHistoryServer.init(conf);
      jobHistoryServer.start();
      final MRClientProtocol hsService = jobHistoryServer.getClientService()
          .getClientHandler();

      // Fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
      Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
   // Default realm is APACHE.ORG
      loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);


      Token token = getDelegationToken(loggedInUser, hsService,
          loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);

      // Now try talking to JHS using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "TheDarkLord", conf);

      GetJobReportRequest jobReportRequest =
          Records.newRecord(GetJobReportRequest.class);
      jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (IOException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
   // Renew after 50% of token age.
      while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
        Thread.sleep(500l);
      }
      long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
      long renewalTime = System.currentTimeMillis();
      LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: "
          + nextExpTime);
     
      // Wait for first expiry, but before renewed expiry.
      while (System.currentTimeMillis() > tokenFetchTime + initialInterval
          && System.currentTimeMillis() < nextExpTime) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
     
      // Valid token because of renewal.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (IOException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
      // Wait for expiry.
      while(System.currentTimeMillis() < renewalTime + renewInterval) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
      LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
      // Token should have expired.     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with an expired token");
      } catch (IOException e) {
        assertTrue(e.getCause().getMessage().contains("is expired"));
      }
     
      // Test cancellation
      // Stop the existing proxy, start another.
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
      token = getDelegationToken(loggedInUser, hsService,
          loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);
      // Now try talking to HSService using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);

     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (IOException e) {
        fail("Unexpected exception" + e);
      }
      cancelDelegationToken(loggedInUser, hsService, token);
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
     
      // Creating a new connection.
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);
      LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
      // Verify cancellation worked.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with a cancelled delegation token");
      } catch (IOException e) {
      }


     
    } finally {
      jobHistoryServer.stop();
    }
  }
View Full Code Here

          MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
              + ":0");
          getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
            hostname + ":0");
        }
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

          getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
              MiniYARNCluster.getHostname() + ":0");
          getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
              MiniYARNCluster.getHostname() + ":0");
        }
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

   
    final long initialInterval = 10000l;
    final long maxLifetime= 20000l;
    final long renewInterval = 10000l;

    JobHistoryServer jobHistoryServer = null;
    MRClientProtocol clientUsingDT = null;
    long tokenFetchTime;
    try {
      jobHistoryServer = new JobHistoryServer() {
        protected void doSecureLogin(Configuration conf) throws IOException {
          // no keytab based login
        };

        protected JHSDelegationTokenSecretManager createJHSSecretManager(
            Configuration conf) {
          return new JHSDelegationTokenSecretManager(initialInterval,
              maxLifetime, renewInterval, 3600000);
        }
      };
//      final JobHistoryServer jobHistoryServer = jhServer;
      jobHistoryServer.init(conf);
      jobHistoryServer.start();
      final MRClientProtocol hsService = jobHistoryServer.getClientService()
          .getClientHandler();

      // Fake the authentication-method
      UserGroupInformation loggedInUser = UserGroupInformation
          .createRemoteUser("testrenewer@APACHE.ORG");
      Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
   // Default realm is APACHE.ORG
      loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);


      DelegationToken token = getDelegationToken(loggedInUser, hsService,
          loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);

      // Now try talking to JHS using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "TheDarkLord", conf);

      GetJobReportRequest jobReportRequest =
          Records.newRecord(GetJobReportRequest.class);
      jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (YarnRemoteException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
   // Renew after 50% of token age.
      while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
        Thread.sleep(500l);
      }
      long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
      long renewalTime = System.currentTimeMillis();
      LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: "
          + nextExpTime);
     
      // Wait for first expiry, but before renewed expiry.
      while (System.currentTimeMillis() > tokenFetchTime + initialInterval
          && System.currentTimeMillis() < nextExpTime) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
     
      // Valid token because of renewal.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (UndeclaredThrowableException e) {
        Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
      }
     
      // Wait for expiry.
      while(System.currentTimeMillis() < renewalTime + renewInterval) {
        Thread.sleep(500l);
      }
      Thread.sleep(50l);
      LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
      // Token should have expired.     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with an expired token");
      } catch (UndeclaredThrowableException e) {
        assertTrue(e.getCause().getMessage().contains("is expired"));
      }
     
      // Test cancellation
      // Stop the existing proxy, start another.
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
      token = getDelegationToken(loggedInUser, hsService,
          loggedInUser.getShortUserName());
      tokenFetchTime = System.currentTimeMillis();
      LOG.info("Got delegation token at: " + tokenFetchTime);
      // Now try talking to HSService using the delegation token
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);

     
      try {
        clientUsingDT.getJobReport(jobReportRequest);
      } catch (UndeclaredThrowableException e) {
        fail("Unexpected exception" + e);
      }
      cancelDelegationToken(loggedInUser, hsService, token);
      if (clientUsingDT != null) {
//        RPC.stopProxy(clientUsingDT);
        clientUsingDT = null;
      }
     
      // Creating a new connection.
      clientUsingDT = getMRClientProtocol(token, jobHistoryServer
          .getClientService().getBindAddress(), "loginuser2", conf);
      LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
      // Verify cancellation worked.
      try {
        clientUsingDT.getJobReport(jobReportRequest);
        fail("Should not have succeeded with a cancelled delegation token");
      } catch (UndeclaredThrowableException e) {
      }


     
    } finally {
      jobHistoryServer.stop();
    }
  }
View Full Code Here

    }

    @Override
    public synchronized void start() {
      try {
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

    }

    @Override
    public synchronized void start() {
      try {
        historyServer = new JobHistoryServer();
        historyServer.init(getConfig());
        new Thread() {
          public void run() {
            historyServer.start();
          };
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.