Examples of EditLogFileInputStream

com.alimama.mdrill.editlog.read.EditLogFileInputStream
org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream
An implementation of the abstract class {@link EditLogInputStream}, which reads edits from a file. That file may be either on the local disk or accessible via a URL.
org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

    int numEdits = 0;
    File jSpoolFile = getJSpoolFile(sdEdits);
    long startTime = now();
    if(jSpoolFile.exists()) {
      // load edits.new
      EditLogFileInputStream edits = new EditLogFileInputStream(jSpoolFile);
      DataInputStream in = edits.getDataInputStream();
      FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
      numEdits += logLoader.loadFSEdits(in, false);


      // first time reached the end of spool
      jsState = JSpoolState.WAIT;
      numEdits += logLoader.loadEditRecords(getLayoutVersion(), in, true);
      getFSNamesystem().dir.updateCountForINodeWithQuota();
      edits.close();
    }


    FSImage.LOG.info("Edits file " + jSpoolFile.getCanonicalPath() 
        + " of size " + jSpoolFile.length() + " edits # " + numEdits 
        + " loaded in " + (now()-startTime)/1000 + " seconds.");

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS);
      // Start from 0 when loading edit logs.
      editLog.setStartTransactionId(0);
      System.out.println("Verifying file: " + editFile);
      EditLogInputStream is = new EditLogFileInputStream(editFile);
      FSEditLogLoader loader = new FSEditLogLoader(namesystem);
      int numEdits = loader.loadFSEdits(is, namesystem.getEditLog().getCurrentTxId());
      int numLeases = namesystem.leaseManager.countLease();
      System.out.println("Number of outstanding leases " + numLeases);
      assertEquals(0, numLeases);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

          continue;
        
        cluster.getNameNode().getFSImage().getEditLog().setStartTransactionId(0);
        
        System.out.println("Verifying file: " + editFile);
        EditLogInputStream is = new EditLogFileInputStream(editFile);
        FSEditLogLoader loader = new FSEditLogLoader(namesystem);
        int numEdits = loader.loadFSEdits(is, namesystem.getEditLog().getCurrentTxId());
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

        File file = null;
        EditLogInputStream elis = null;
        OfflineEditsLoader loader = null;
        try {
          file = new File(inputFileName);
          elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID,
              HdfsConstants.INVALID_TXID, false);
          loader = new OfflineEditsBinaryLoader(visitor, elis, flags);
        } finally {
          if ((loader == null) && (elis != null)) {
            elis.close();

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
        
        System.out.println("Verifying file: " + editFile);
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
        
        System.out.println("Verifying file: " + editFile);
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

    
    LOG.info("Trying to load the edit log from " + txFile.getAbsolutePath());
    
    do {
      try {
        inputStream = new EditLogFileInputStream(txFile);
        editLogFilePosition = inputStream.getPosition();
        curStreamConsumed = false;
        curStreamFinished = false;
        readNullAfterStreamFinished = false;
        LOG.info("Successfully loaded the edits log from " +

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            ledgerId,
            0,
            1,
            numEdits,
            false);
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);


    assertEquals("Length in bytes must be equal!",
        bkEditsIn.length(), fileEditsIn.length());


    FSEditLogTestUtil.assertStreamsAreEquivalent(numEdits,
        ImmutableMap.of("BookKeeper", bkEditsIn, "File", fileEditsIn));
    assertNull("BookKeeper edit log must end at txid 100", bkEditsIn.readOp());
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            0,
            1,
            numEdits,
            false);


    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);


    assertEquals("Length in bytes must be equal!",
        bkEditsIn.length(), fileEditsIn.length());


    long lastBkPos = bkEditsIn.getPosition();
    long lastFilePos = fileEditsIn.getPosition();
    for (int i = 1; i <= numEdits; i++) {
      assertEquals("Position in file must be equal position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
          "Operation read from file and BookKeeper must be same after refresh",
          opFromBk, opFromFile);
      lastBkPos = bkEditsIn.getPosition();
      lastFilePos = fileEditsIn.getPosition();
    }
    assertNull("BookKeeper edit log must end at last txId", bkEditsIn.readOp());
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

            ledgerId,
            0,
            1,
            numEdits,
            false);
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);
    long lastBkPos = bkEditsIn.getPosition();
    long lastFilePos = fileEditsIn.getPosition();
    for (int i = 0; i <= numEdits; i++) {
      assertEquals("Position in file must be equal to position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
          "Operation read from file and BookKeeper must be same after refresh",
          opFromBk, opFromFile);
      lastBkPos = bkEditsIn.getPosition();
      lastFilePos = fileEditsIn.getPosition();
      bkEditsIn =
          new BookKeeperEditLogInputStream(ledgerProvider,
              ledgerId,
              0,
              1,
              numEdits,
              false);
      fileEditsIn =
          new EditLogFileInputStream(tempEditsFile);
    }
    bkEditsIn.refresh(lastBkPos, -1);
    assertNull("BookKeeper edit log must end at last txId", bkEditsIn.readOp());
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.