Examples of org.apache.hadoop.hdfs.DistributedRaidFileSystem

org.apache.hadoop.hdfs.DistributedRaidFileSystem
This is an implementation of the Hadoop RAID Filesystem. This FileSystem wraps an instance of the DistributedFileSystem. If a file is corrupted, this FileSystem uses the parity blocks to regenerate the bad block.

      Path srcPath2 = new Path("/ignore2/test/rename/f1");
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath, 
          1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2, 
          1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));


      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
      
      FileStatus srcStat = fileSys.getFileStatus(srcPath);
      FileStatus srcStat2 = fileSys.getFileStatus(srcPath2);
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat, 
          conf);
      Path srcParityPath = parity.getPath();
      ParityFilePair parity2 = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat2, 
          conf);
      Path srcParityPath2 = parity2.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));
      
      // test rename files
      raidFs.rename(srcPath, new Path("/ignore1/test/rename/f2"));
      raidFs.rename(srcPath2, new Path("/ignore2/test/rename/f2"));
      // verify we did not rename the parityPath
      assertFalse(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcParityPath));
      assertFalse(raidFs.exists(srcPath2));
      assertTrue(raidFs.exists(srcParityPath2));
    } finally {
      stopCluster();
    }
  }

View Full Code Here


      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath, 
          1, 8, 8192L);
      TestRaidDfs.createTestFilePartialLastBlock(fileSys, srcPath2, 
          1, 8, 8192L);
      DistributedRaidFileSystem raidFs = getRaidFS();
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));


      // generate the parity files.
      doRaid(srcPath, Codec.getCodec("xor"));
      doRaid(srcPath2, Codec.getCodec("xor"));
      
      FileStatus srcStat = fileSys.getFileStatus(srcPath);
      FileStatus srcStat2 = fileSys.getFileStatus(srcPath2);
      ParityFilePair parity = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat, 
          conf);
      Path srcParityPath = parity.getPath();
      ParityFilePair parity2 = ParityFilePair.getParityFile(
          Codec.getCodec("xor"),
          srcStat2, 
          conf);
      Path srcParityPath2 = parity2.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));


      // do the delete file
      assertTrue(raidFs.delete(srcPath));


      // verify the results.
      assertFalse(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));


      // do the undelete using non-exist userName
      String nonExistedUser = UUID.randomUUID().toString();
      assertFalse(raidFs.undelete(srcPath, nonExistedUser));


      // verify the results
      assertFalse(raidFs.exists(srcPath));
      assertFalse(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));


      // do the undelete file using current userName
      assertTrue(raidFs.undelete(srcPath, null));
      //verify the results.
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));


      // delete the dir
      assertTrue(raidFs.delete(srcPath2.getParent()));
      // verify the results.
      assertFalse(raidFs.exists(srcPath2.getParent()));
      assertFalse(raidFs.exists(srcParityPath));
      assertFalse(raidFs.exists(srcParityPath2));
      assertFalse(raidFs.exists(srcParityPath.getParent()));


      // undelete the dir
      assertTrue(raidFs.undelete(srcPath2.getParent(), null));
      // verify the results.
      assertTrue(raidFs.exists(srcPath));
      assertTrue(raidFs.exists(srcPath2));
      assertTrue(raidFs.exists(srcParityPath));
      assertTrue(raidFs.exists(srcParityPath2));
      
      // try to delete not existed file.
      Path notExistedPath = new Path("/user/dhruba/raidtest/raidnotexist");
      assertFalse(raidFs.delete(notExistedPath));
      
    } finally {
      stopCluster();
    }
  }

View Full Code Here

      int[] seeds = new int[3];
      short repl = 1;
      Path dirPath = new Path("/user/dikang/raidtest");
      
      mySetup();
      DistributedRaidFileSystem raidFs = getRaidFS();
      Path[] files = TestRaidDfs.createTestFiles(dirPath,
          fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
      FileStatus stat = raidFs.getFileStatus(dirPath); 
      Codec codec = Codec.getCodec("dir-rs");
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
          new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
        false, repl, repl);
      
      Path destPath = new Path("/user/dikang/raidtest_new");
      
      assertTrue(raidFs.exists(dirPath));
      assertFalse(raidFs.exists(destPath));
      
      ParityFilePair parity = ParityFilePair.getParityFile(
          codec, stat, conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      
      raidFs.mkdirs(destPath);
      // do the rename file
      assertTrue(raidFs.rename(files[0], new Path(destPath, "file0")));
      // verify the results.
      assertFalse(raidFs.exists(files[0]));
      assertTrue(raidFs.exists(new Path(destPath, "file0")));
      assertTrue(raidFs.exists(srcParityPath));
      
      // rename the left files
      assertTrue(raidFs.rename(files[1], new Path(destPath, "file1")));
      assertTrue(raidFs.rename(files[2], new Path(destPath, "file2")));
      assertFalse(raidFs.exists(srcParityPath));
      
      Path newParityPath = new Path(codec.parityDirectory, 
          "user/dikang/raidtest_new");
      assertTrue(raidFs.exists(newParityPath));
    } finally {
      stopCluster();
    }
  }

View Full Code Here

      int[] seeds = new int[3];
      short repl = 1;
      Path dirPath = new Path("/user/dikang/raidtest");
      
      mySetup();
      DistributedRaidFileSystem raidFs = getRaidFS();
      Path[] files = TestRaidDfs.createTestFiles(dirPath,
          fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
      FileStatus stat = raidFs.getFileStatus(dirPath); 
      Codec codec = Codec.getCodec("dir-rs");
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
          new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
        false, repl, repl);
      
      Path destPath = new Path("/user/dikang/raidtest_new");
      
      assertTrue(raidFs.exists(dirPath));
      assertFalse(raidFs.exists(destPath));
      
      ParityFilePair parity = ParityFilePair.getParityFile(
          codec, stat, conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      // do the rename file
      assertTrue(raidFs.rename(dirPath, destPath));
      // verify the results.
      assertFalse(raidFs.exists(dirPath));
      assertTrue(raidFs.exists(destPath));
      assertFalse(raidFs.exists(srcParityPath));
      FileStatus srcDest = raidFs.getFileStatus(destPath);
      parity = ParityFilePair.getParityFile(codec, srcDest, conf);
      assertTrue(raidFs.exists(parity.getPath()));
    } finally {
      stopCluster();
    }
  }

View Full Code Here

      int[] seeds = new int[3];
      short repl = 1;
      Path dirPath = new Path("/user/dikang/raidtest");
      
      mySetup();
      DistributedRaidFileSystem raidFs = getRaidFS();
      Path[] files = TestRaidDfs.createTestFiles(dirPath,
          fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
      FileStatus stat = raidFs.getFileStatus(dirPath); 
      Codec codec = Codec.getCodec("dir-rs");
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
          new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
        false, repl, repl);
      ParityFilePair parity = ParityFilePair.getParityFile(
          codec, stat, conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      Path destPath = new Path("/user/dikang/raidtest_new");
      
      // try to delete the parity
      try {
        raidFs.delete(srcParityPath);
        fail();
      } catch (IOException e) {
        LOG.warn("Expected exception: " + e.getMessage(), e);
        assertTrue(e.getMessage().startsWith(
            "You can not delete a parity file"));
      }
      
      // try to rename the parity
      try {
        raidFs.rename(srcParityPath, destPath);
        fail();
      } catch (IOException e) {
        LOG.warn("Expected exception: " + e.getMessage(), e);
        assertTrue(e.getMessage().startsWith(
            "You can not rename a parity file"));

View Full Code Here

      int[] seeds = new int[3];
      short repl = 1;
      Path dirPath = new Path("/user/dikang/raidtest");
      
      mySetup();
      DistributedRaidFileSystem raidFs = getRaidFS();
      Path[] files = TestRaidDfs.createTestFiles(dirPath,
          fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
      FileStatus stat = raidFs.getFileStatus(dirPath); 
      Codec codec = Codec.getCodec("dir-rs");
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
          new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
        false, repl, repl);
      
      ParityFilePair parity = ParityFilePair.getParityFile(
          codec, stat, conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));
      
      // delete one file
      assertTrue(raidFs.delete(files[0]));
      // verify the results
      assertFalse(raidFs.exists(files[0]));
      // we still have the parity file
      assertTrue(raidFs.exists(srcParityPath));
      
      // delete the left files
      assertTrue(raidFs.delete(files[1]));
      assertTrue(raidFs.delete(files[2]));
      
      // we will not touch the parity file.
      assertTrue(raidFs.exists(srcParityPath));
    } finally {
      stopCluster();
    }
  }

View Full Code Here

      int[] seeds = new int[3];
      short repl = 1;
      Path dirPath = new Path("/user/dikang/raidtest");
      
      mySetup();
      DistributedRaidFileSystem raidFs = getRaidFS();
      Path[] files = TestRaidDfs.createTestFiles(dirPath,
          fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
      FileStatus stat = raidFs.getFileStatus(dirPath); 
      Codec codec = Codec.getCodec("dir-rs");
      RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
          new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
        false, repl, repl);
      
      ParityFilePair parity = ParityFilePair.getParityFile(
          codec, stat, conf);
      Path srcParityPath = parity.getPath();
      assertTrue(raidFs.exists(srcParityPath));


      // do the delete file
      assertTrue(raidFs.delete(dirPath));


      // verify the results.
      assertFalse(raidFs.exists(dirPath));
      assertFalse(raidFs.exists(srcParityPath));


      // do the undelete using non-exist userName
      String nonExistedUser = UUID.randomUUID().toString();
      assertFalse(raidFs.undelete(dirPath, nonExistedUser));


      // verify the results
      assertFalse(raidFs.exists(dirPath));
      assertFalse(raidFs.exists(srcParityPath));


      // do the undelete file using current userName
      assertTrue(raidFs.undelete(dirPath, null));
      //verify the results.
      assertTrue(raidFs.exists(dirPath));
      assertTrue(raidFs.exists(srcParityPath));
    } finally {
      stopCluster();
    }
  }

View Full Code Here

    
    mySetup();
    //disable trash
    conf.setInt("fs.trash.interval", 0);
    
    DistributedRaidFileSystem raidFs = getRaidFS();
    Path[] files = TestRaidDfs.createTestFiles(dirPath,
        fileSizes, blockSizes, crcs, seeds, fileSys, (short)1);
    FileStatus stat = raidFs.getFileStatus(dirPath); 
    Codec codec = Codec.getCodec("dir-rs");
    RaidNode.doRaid(conf, stat, new Path(codec.parityDirectory), codec,
        new RaidNode.Statistics(), RaidUtils.NULL_PROGRESSABLE,
      false, repl, repl);
    
    try {
      raidFs.delete(files[0]);
      fail();
    } catch (Exception ex) {
      LOG.warn("Excepted error: " + ex.getMessage(), ex);
    } finally {
      stopCluster();

View Full Code Here

      }
      affectedFiles.add(bi.fileIdx);
    }
    // validate files
    if (validate) {
      DistributedRaidFileSystem raidfs = getRaidFS(fileSys, conf);
      for (Integer fid: affectedFiles) {
        FileStatus stat = lfs.get(fid);
        assertTrue(TestRaidDfs.validateFile(raidfs, stat.getPath(),
            lengths[fid], crcs[fid]));
        // test readFully
        byte[] filebytes = new byte[(int)stat.getLen()];
        FSDataInputStream stm = raidfs.open(stat.getPath());
        stm.readFully(0, filebytes);
        CRC32 crc = new CRC32();
        crc.update(filebytes, 0, filebytes.length);
        assertEquals(crcs[fid], crc.getValue());
      }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.hdfs.DistributedRaidFileSystem

org.apache.hadoop.conf.Configuration

org.apache.hadoop.fs.FileStatus

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FSDataInputStream

org.apache.hadoop.fs.FSDataOutputStream

org.apache.hadoop.fs.Path

org.apache.hadoop.fs.Trash

org.apache.hadoop.hdfs.DistributedFileSystem

org.apache.hadoop.hdfs.protocol.LocatedBlock

org.apache.hadoop.hdfs.protocol.LocatedBlocks

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.