Package org.apache.hadoop.hdfs.server.namenode.metrics

Examples of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics


      ServletContext context = getServletContext();
      final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context);
      final GetImageParams parsedParams = new GetImageParams(request, response);
      final Configuration conf = (Configuration) context
          .getAttribute(JspHelper.CURRENT_CONF);
      final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
     
      if (UserGroupInformation.isSecurityEnabled() &&
          !isValidRequestor(context, request.getUserPrincipal().getName(), conf)) {
        response.sendError(HttpServletResponse.SC_FORBIDDEN,
            "Only Namenode, Secondary Namenode, and administrators may access " +
            "this servlet");
        LOG.warn("Received non-NN/SNN/administrator request for image or edits from "
            + request.getUserPrincipal().getName() + " at " + request.getRemoteHost());
        return;
      }
     
      String myStorageInfoString = nnImage.getStorage().toColonSeparatedString();
      String theirStorageInfoString = parsedParams.getStorageInfoString();
      if (theirStorageInfoString != null &&
          !myStorageInfoString.equals(theirStorageInfoString)) {
        response.sendError(HttpServletResponse.SC_FORBIDDEN,
            "This namenode has storage info " + myStorageInfoString +
            " but the secondary expected " + theirStorageInfoString);
        LOG.warn("Received an invalid request file transfer request " +
            "from a secondary with storage info " + theirStorageInfoString);
        return;
      }
     
      UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          if (parsedParams.isGetImage()) {
            long txid = parsedParams.getTxId();
            File imageFile = null;
            String errorMessage = "Could not find image";
            if (parsedParams.shouldFetchLatest()) {
              imageFile = nnImage.getStorage().getHighestFsImageName();
            } else {
              errorMessage += " with txid " + txid;
              imageFile = nnImage.getStorage().getFsImageName(txid);
            }
            if (imageFile == null) {
              throw new IOException(errorMessage);
            }
            CheckpointFaultInjector.getInstance().beforeGetImageSetsHeaders();
            long start = now();
            serveFile(imageFile);

            if (metrics != null) { // Metrics non-null only when used inside name node
              long elapsed = now() - start;
              metrics.addGetImage(elapsed);
            }
          } else if (parsedParams.isGetEdit()) {
            long startTxId = parsedParams.getStartTxId();
            long endTxId = parsedParams.getEndTxId();
           
            File editFile = nnImage.getStorage()
                .findFinalizedEditsFile(startTxId, endTxId);
            long start = now();
            serveFile(editFile);

            if (metrics != null) { // Metrics non-null only when used inside name node
              long elapsed = now() - start;
              metrics.addGetEdit(elapsed);
            }
          } else if (parsedParams.isPutImage()) {
            final long txid = parsedParams.getTxId();

            if (! currentlyDownloadingCheckpoints.add(txid)) {
              response.sendError(HttpServletResponse.SC_CONFLICT,
                  "Another checkpointer is already in the process of uploading a" +
                  " checkpoint made at transaction ID " + txid);
              return null;
            }

            try {
              if (nnImage.getStorage().findImageFile(txid) != null) {
                response.sendError(HttpServletResponse.SC_CONFLICT,
                    "Another checkpointer already uploaded an checkpoint " +
                    "for txid " + txid);
                return null;
              }
             
              // We may have lost our ticket since last checkpoint, log in again, just in case
              if (UserGroupInformation.isSecurityEnabled()) {
                UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
              }
             
              long start = now();
              // issue a HTTP get request to download the new fsimage
              MD5Hash downloadImageDigest =
                TransferFsImage.downloadImageToStorage(
                        parsedParams.getInfoServer(conf), txid,
                        nnImage.getStorage(), true);
              nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);

              if (metrics != null) { // Metrics non-null only when used inside name node
                long elapsed = now() - start;
                metrics.addPutImage(elapsed);
              }
             
              // Now that we have a new checkpoint, we might be able to
              // remove some old ones.
              nnImage.purgeOldStorage();
View Full Code Here


    String oneKB = StringUtils.byteToHexString(
        new byte[500]);
   
    try {
      log.openForWrite();
      NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
      log.setMetricsForTests(mockMetrics);

      for (int i = 0; i < 400; i++) {
        log.logDelete(oneKB, 1L, false);
      }
View Full Code Here

      endTime = Time.now();
      namesystem.writeUnlock();
    }

    // Log the block report processing stats from Namenode perspective
    final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
    if (metrics != null) {
      metrics.addBlockReport((int) (endTime - startTime));
    }
    blockLog.info("BLOCK* processReport: from "
        + nodeID + ", blocks: " + newReport.getNumberOfBlocks()
        + ", processing time: " + (endTime - startTime) + " msecs");
  }
View Full Code Here

    String oneKB = StringUtils.byteToHexString(
        new byte[500]);
   
    try {
      log.openForWrite();
      NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
      log.setMetricsForTests(mockMetrics);

      for (int i = 0; i < 400; i++) {
        log.logDelete(oneKB, 1L);
      }
View Full Code Here

    String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
    namesystem.loadFSImage(startOpt, fsImage,
      HAUtil.isHAEnabled(conf, nameserviceId));
    long timeTakenToLoadFSImage = now() - loadStart;
    LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
    NameNodeMetrics nnMetrics = NameNode.getNameNodeMetrics();
    if (nnMetrics != null) {
      nnMetrics.setFsImageLoadTime((int) timeTakenToLoadFSImage);
    }
    return namesystem;
  }
View Full Code Here

      endTime = Time.now();
      namesystem.writeUnlock();
    }

    // Log the block report processing stats from Namenode perspective
    final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
    if (metrics != null) {
      metrics.addBlockReport((int) (endTime - startTime));
    }
    blockLog.info("BLOCK* processReport: from storage " + storage.getStorageID()
        + " node " + nodeID + ", blocks: " + newReport.getNumberOfBlocks()
        + ", processing time: " + (endTime - startTime) + " msecs");
    return !node.hasStaleStorages();
View Full Code Here

      final ServletContext context = getServletContext();
      final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context);
      final GetImageParams parsedParams = new GetImageParams(request, response);
      final Configuration conf = (Configuration) context
          .getAttribute(JspHelper.CURRENT_CONF);
      final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();

      validateRequest(context, conf, request, response, nnImage,
          parsedParams.getStorageInfoString());

      UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          if (parsedParams.isGetImage()) {
            long txid = parsedParams.getTxId();
            File imageFile = null;
            String errorMessage = "Could not find image";
            if (parsedParams.shouldFetchLatest()) {
              imageFile = nnImage.getStorage().getHighestFsImageName();
            } else {
              errorMessage += " with txid " + txid;
              imageFile = nnImage.getStorage().getFsImage(txid,
                  EnumSet.of(NameNodeFile.IMAGE, NameNodeFile.IMAGE_ROLLBACK));
            }
            if (imageFile == null) {
              throw new IOException(errorMessage);
            }
            CheckpointFaultInjector.getInstance().beforeGetImageSetsHeaders();
            long start = monotonicNow();
            serveFile(imageFile);

            if (metrics != null) { // Metrics non-null only when used inside name node
              long elapsed = monotonicNow() - start;
              metrics.addGetImage(elapsed);
            }
          } else if (parsedParams.isGetEdit()) {
            long startTxId = parsedParams.getStartTxId();
            long endTxId = parsedParams.getEndTxId();
           
            File editFile = nnImage.getStorage()
                .findFinalizedEditsFile(startTxId, endTxId);
            long start = monotonicNow();
            serveFile(editFile);

            if (metrics != null) { // Metrics non-null only when used inside name node
              long elapsed = monotonicNow() - start;
              metrics.addGetEdit(elapsed);
            }
          }
          return null;
        }
View Full Code Here

      final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context);
      final Configuration conf = (Configuration) getServletContext()
          .getAttribute(JspHelper.CURRENT_CONF);
      final PutImageParams parsedParams = new PutImageParams(request, response,
          conf);
      final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();

      validateRequest(context, conf, request, response, nnImage,
          parsedParams.getStorageInfoString());

      UserGroupInformation.getCurrentUser().doAs(
          new PrivilegedExceptionAction<Void>() {

            @Override
            public Void run() throws Exception {

              final long txid = parsedParams.getTxId();

              final NameNodeFile nnf = parsedParams.getNameNodeFile();

              if (!currentlyDownloadingCheckpoints.add(txid)) {
                response.sendError(HttpServletResponse.SC_CONFLICT,
                    "Another checkpointer is already in the process of uploading a"
                        + " checkpoint made at transaction ID " + txid);
                return null;
              }
              try {
                if (nnImage.getStorage().findImageFile(nnf, txid) != null) {
                  response.sendError(HttpServletResponse.SC_CONFLICT,
                      "Another checkpointer already uploaded an checkpoint "
                          + "for txid " + txid);
                  return null;
                }

                InputStream stream = request.getInputStream();
                try {
                  long start = monotonicNow();
                  MD5Hash downloadImageDigest = TransferFsImage
                      .handleUploadImageRequest(request, txid,
                          nnImage.getStorage(), stream,
                          parsedParams.getFileSize(), getThrottler(conf));
                  nnImage.saveDigestAndRenameCheckpointImage(nnf, txid,
                      downloadImageDigest);
                  // Metrics non-null only when used inside name node
                  if (metrics != null) {
                    long elapsed = monotonicNow() - start;
                    metrics.addPutImage(elapsed);
                  }
                  // Now that we have a new checkpoint, we might be able to
                  // remove some old ones.
                  nnImage.purgeOldStorage(nnf);
                } finally {
View Full Code Here

      fsImage.close();
      throw ioe;
    }
    long timeTakenToLoadFSImage = now() - loadStart;
    LOG.info("Finished loading FSImage in " + timeTakenToLoadFSImage + " msecs");
    NameNodeMetrics nnMetrics = NameNode.getNameNodeMetrics();
    if (nnMetrics != null) {
      nnMetrics.setFsImageLoadTime((int) timeTakenToLoadFSImage);
    }
    return namesystem;
  }
View Full Code Here

      endTime = Util.now();
      namesystem.writeUnlock();
    }

    // Log the block report processing stats from Namenode perspective
    final NameNodeMetrics metrics = NameNode.getNameNodeMetrics();
    if (metrics != null) {
      metrics.addBlockReport((int) (endTime - startTime));
    }
    blockLog.info("BLOCK* processReport: from "
        + nodeID.getName() + ", blocks: " + newReport.getNumberOfBlocks()
        + ", processing time: " + (endTime - startTime) + " msecs");
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.