Package org.apache.hadoop.hdfs.server.common.Storage

Examples of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory


    for (URI u : dirs) {
      boolean required = FSNamesystem.getRequiredNamespaceEditsDirs(conf)
          .contains(u);
      if (u.getScheme().equals(NNStorage.LOCAL_URI_SCHEME)) {
        StorageDirectory sd = storage.getStorageDirectory(u);
        if (sd != null) {
          journalSet.add(new FileJournalManager(sd, storage), required);
        }
      } else {
        journalSet.add(createJournal(u), required);
View Full Code Here


    boolean canRollback = false;
    FSImage prevState = new FSImage(conf);
    try {
      prevState.getStorage().layoutVersion = HdfsConstants.LAYOUT_VERSION;
      for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File prevDir = sd.getPreviousDir();
        if (!prevDir.exists()) {  // use current directory then
          LOG.info("Storage directory " + sd.getRoot()
            + " does not contain previous fs state.");
          // read and verify consistency with other directories
          storage.readProperties(sd);
          continue;
        }

        // read and verify consistency of the prev dir
        prevState.getStorage().readPreviousVersionProperties(sd);

        if (prevState.getLayoutVersion() != HdfsConstants.LAYOUT_VERSION) {
          throw new IOException(
            "Cannot rollback to storage version " +
                prevState.getLayoutVersion() +
                " using this version of the NameNode, which uses storage version " +
                HdfsConstants.LAYOUT_VERSION + ". " +
              "Please use the previous version of HDFS to perform the rollback.");
        }
        canRollback = true;
      }
      if (!canRollback)
        throw new IOException("Cannot rollback. None of the storage "
            + "directories contain previous fs state.");

      // Now that we know all directories are going to be consistent
      // Do rollback for each directory containing previous state
      for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
        StorageDirectory sd = it.next();
        File prevDir = sd.getPreviousDir();
        if (!prevDir.exists())
          continue;

        LOG.info("Rolling back storage directory " + sd.getRoot()
          + ".\n   new LV = " + prevState.getStorage().getLayoutVersion()
          + "; new CTime = " + prevState.getStorage().getCTime());
        File tmpDir = sd.getRemovedTmp();
        assert !tmpDir.exists() : "removed.tmp directory must not exist.";
        // rename current to tmp
        File curDir = sd.getCurrentDir();
        assert curDir.exists() : "Current directory must exist.";
        NNStorage.rename(curDir, tmpDir);
        // rename previous to current
        NNStorage.rename(prevDir, curDir);

        // delete tmp dir
        NNStorage.deleteDir(tmpDir);
        LOG.info("Rollback of " + sd.getRoot()+ " is complete.");
      }
      isUpgradeFinalized = true;
    } finally {
      prevState.close();
    }
View Full Code Here

    getStorage().writeAll();
  }

  void finalizeUpgrade() throws IOException {
    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
      StorageDirectory sd = it.next();
      doFinalize(sd);
    }
  }
View Full Code Here

  }

  void loadFSImageFile(FSNamesystem target, MetaRecoveryContext recovery,
      FSImageFile imageFile) throws IOException {
    LOG.debug("Planning to load image :\n" + imageFile);
    StorageDirectory sdForProperties = imageFile.sd;
    storage.readProperties(sdForProperties);

    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
                               getLayoutVersion())) {
      // For txid-based layout, we should have a .md5 file
      // next to the image file
      loadFSImage(imageFile.getFile(), target, recovery);
    } else if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM,
                                      getLayoutVersion())) {
      // In 0.22, we have the checksum stored in the VERSION file.
      String md5 = storage.getDeprecatedProperty(
          NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY);
      if (md5 == null) {
        throw new InconsistentFSStateException(sdForProperties.getRoot(),
            "Message digest property " +
            NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY +
            " not set for storage directory " + sdForProperties.getRoot());
      }
      loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery);
    } else {
      // We don't have any record of the md5sum
      loadFSImage(imageFile.getFile(), null, target, recovery);
View Full Code Here

    try {
      List<Thread> saveThreads = new ArrayList<Thread>();
      // save images into current
      for (Iterator<StorageDirectory> it
             = storage.dirIterator(NameNodeDirType.IMAGE); it.hasNext();) {
        StorageDirectory sd = it.next();
        FSImageSaver saver = new FSImageSaver(ctx, sd);
        Thread saveThread = new Thread(saver, saver.toString());
        saveThreads.add(saveThread);
        saveThread.start();
      }
View Full Code Here

    numTransactions = totalTimeTransactions = 0;
    if (editStreams == null)
      editStreams = new ArrayList<EditLogOutputStream>();
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      File eFile = getEditFile(sd);
      try {
        EditLogOutputStream eStream = new EditLogFileOutputStream(eFile);
        editStreams.add(eStream);
      } catch (IOException e) {
View Full Code Here

    //
    // Open edits.new
    //
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      try {
        EditLogFileOutputStream eStream =
             new EditLogFileOutputStream(getEditNewFile(sd));
        eStream.create();
        editStreams.add(eStream);
View Full Code Here

    //
    // Delete edits and rename edits.new to edits.
    //
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      if (!getEditNewFile(sd).renameTo(getEditFile(sd))) {
        //
        // renameTo() fails on Windows if the destination
        // file exists.
        //
View Full Code Here

  /**
   * Return the name of the edit file
   */
  synchronized File getFsEditName() throws IOException {
    StorageDirectory sd = null;
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();)
      sd = it.next();
    return getEditFile(sd);
  }
View Full Code Here

    numTransactions = totalTimeTransactions = numTransactionsBatchedInSync = 0;
    if (editStreams == null)
      editStreams = new ArrayList<EditLogOutputStream>();
    for (Iterator<StorageDirectory> it =
           fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      StorageDirectory sd = it.next();
      File eFile = getEditFile(sd);
      try {
        EditLogOutputStream eStream = new EditLogFileOutputStream(eFile);
        editStreams.add(eStream);
      } catch (IOException e) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.