Package org.apache.hadoop.zebra.tfile.TFile

Examples of org.apache.hadoop.zebra.tfile.TFile.Reader


  void readFile() throws IOException {
    long fileLength = fs.getFileStatus(path).getLen();
    int numSplit = 10;
    long splitSize = fileLength / numSplit + 1;

    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    long offset = 0;
    long rowCount = 0;
    BytesWritable key, value;
    for (int i = 0; i < numSplit; ++i, offset += splitSize) {
      Scanner scanner = reader.createScannerByByteRange(offset, splitSize);
      int count = 0;
      key = new BytesWritable();
      value = new BytesWritable();
      while (!scanner.atEnd()) {
        scanner.entry().get(key, value);
        ++count;
        scanner.advance();
      }
      scanner.close();
      Assert.assertTrue(count > 0);
      rowCount += count;
    }
    Assert.assertEquals(rowCount, reader.getEntryCount());
    reader.close();
  }
View Full Code Here


  /* Similar to readFile(), tests the scanner created
   * by record numbers rather than the offsets.
   */
  void readRowSplits(int numSplits) throws IOException {

    Reader reader =
      new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
   
    long totalRecords = reader.getEntryCount();
    for (int i=0; i<numSplits; i++) {
      long startRec = i*totalRecords/numSplits;
      long endRec = (i+1)*totalRecords/numSplits;
      if (i == numSplits-1) {
        endRec = totalRecords;
      }
      Scanner scanner = reader.createScannerByRecordNum(startRec, endRec);
      int count = 0;
      BytesWritable key = new BytesWritable();
      BytesWritable value = new BytesWritable();
      long x=startRec;
      while (!scanner.atEnd()) {
        assertEquals("Incorrect RecNum returned by scanner", scanner.getRecordNum(), x);
        scanner.entry().get(key, value);
        ++count;
        assertEquals("Incorrect RecNum returned by scanner", scanner.getRecordNum(), x);
        scanner.advance();
        ++x;
      }
      scanner.close();
      Assert.assertTrue(count == (endRec - startRec));
    }
    // make sure specifying range at the end gives zero records.
    Scanner scanner = reader.createScannerByRecordNum(totalRecords, -1);
    Assert.assertTrue(scanner.atEnd());
  }
View Full Code Here

    return String.format("%s%010d", prefix, value);
  }
 
  void checkRecNums() throws IOException {
    long fileLen = fs.getFileStatus(path).getLen();
    Reader reader = new Reader(fs.open(path), fileLen, conf);
    long totalRecs = reader.getEntryCount();
    long begin = random.nextLong() % (totalRecs / 2);
    if (begin < 0)
      begin += (totalRecs / 2);
    long end = random.nextLong() % (totalRecs / 2);
    if (end < 0)
      end += (totalRecs / 2);
    end += (totalRecs / 2) + 1;

    assertEquals("RecNum for offset=0 should be 0", 0, reader
        .getRecordNumNear(0));
    for (long x : new long[] { fileLen, fileLen + 1, 2 * fileLen }) {
      assertEquals("RecNum for offset>=fileLen should be total entries",
          totalRecs, reader.getRecordNumNear(x));
    }

    for (long i = 0; i < 100; ++i) {
      assertEquals("Locaton to RecNum conversion not symmetric", i, reader
          .getRecordNumByLocation(reader.getLocationByRecordNum(i)));
    }

    for (long i = 1; i < 100; ++i) {
      long x = totalRecs - i;
      assertEquals("Locaton to RecNum conversion not symmetric", x, reader
          .getRecordNumByLocation(reader.getLocationByRecordNum(x)));
    }

    for (long i = begin; i < end; ++i) {
      assertEquals("Locaton to RecNum conversion not symmetric", i, reader
          .getRecordNumByLocation(reader.getLocationByRecordNum(i)));
    }

    for (int i = 0; i < 1000; ++i) {
      long x = random.nextLong() % totalRecs;
      if (x < 0) x += totalRecs;
      assertEquals("Locaton to RecNum conversion not symmetric", x, reader
          .getRecordNumByLocation(reader.getLocationByRecordNum(x)));
    }
  }
View Full Code Here

  public void testFailureNegativeOffset() throws IOException {
    if (skip)
      return;
    writeRecords(2, true, true);

    Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Scanner scanner = reader.createScanner();
    byte[] buf = new byte[K];
    try {
      scanner.entry().getKey(buf, -1);
      Assert.fail("Failed to handle key negative offset.");
    }
    catch (Exception e) {
      // noop, expecting exceptions
    }
    finally {
    }
    scanner.close();
    reader.close();
  }
View Full Code Here

    fs.delete(path, true);
  }

  // we still can scan records in an unsorted TFile
  public void testFailureScannerWithKeys() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    try {
      Scanner scanner =
          reader.createScannerByKey("aaa".getBytes(), "zzz".getBytes());
      Assert
          .fail("Failed to catch creating scanner with keys on unsorted file.");
    }
    catch (RuntimeException e) {
    }
    finally {
      reader.close();
    }
  }
View Full Code Here

    }
  }

  // we still can scan records in an unsorted TFile
  public void testScan() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    Scanner scanner = reader.createScanner();

    try {

      // read key and value
      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

      scanner.advance();

      // now try get value first
      vbuf = new byte[BUF_SIZE];
      vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

      kbuf = new byte[BUF_SIZE];
      klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
    }
    finally {
      scanner.close();
      reader.close();
    }
  }
View Full Code Here

    }
  }

  // we still can scan records in an unsorted TFile
  public void testScanRange() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertFalse(reader.isSorted());
    Assert.assertEquals((int) reader.getEntryCount(), 4);

    Scanner scanner = reader.createScanner();

    try {

      // read key and value
      byte[] kbuf = new byte[BUF_SIZE];
      int klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyZ");

      byte[] vbuf = new byte[BUF_SIZE];
      int vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueZ");

      scanner.advance();

      // now try get value first
      vbuf = new byte[BUF_SIZE];
      vlen = scanner.entry().getValueLength();
      scanner.entry().getValue(vbuf);
      Assert.assertEquals(new String(vbuf, 0, vlen), "valueM");

      kbuf = new byte[BUF_SIZE];
      klen = scanner.entry().getKeyLength();
      scanner.entry().getKey(kbuf);
      Assert.assertEquals(new String(kbuf, 0, klen), "keyM");
    }
    finally {
      scanner.close();
      reader.close();
    }
  }
View Full Code Here

      reader.close();
    }
  }

  public void testFailureSeek() throws IOException {
    Reader reader =
        new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Scanner scanner = reader.createScanner();

    try {
      // can't find ceil
      try {
        scanner.lowerBound("keyN".getBytes());
        Assert.fail("Cannot search in a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }

      // can't find higher
      try {
        scanner.upperBound("keyA".getBytes());
        Assert.fail("Cannot search higher in a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }

      // can't seek
      try {
        scanner.seekTo("keyM".getBytes());
        Assert.fail("Cannot search a unsorted TFile!");
      }
      catch (Exception e) {
        // noop, expecting excetions
      }
      finally {
      }
    }
    finally {
      scanner.close();
      reader.close();
    }
  }
View Full Code Here

  public void testNoDataEntry() throws IOException {
    if (skip)
      return;
    closeOutput();

    Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Assert.assertTrue(reader.isSorted());
    Scanner scanner = reader.createScanner();
    Assert.assertTrue(scanner.atEnd());
    scanner.close();
    reader.close();
  }
View Full Code Here

 
  public void testLocate() throws IOException {
    if (skip)
      return;
    writeRecords(3 * records1stBlock);
    Reader reader = new Reader(fs.open(path), fs.getFileStatus(path).getLen(), conf);
    Scanner scanner = reader.createScanner();
    Location loc2 =
        locate(scanner, composeSortedKey(KEY, 3 * records1stBlock, 2)
            .getBytes());
    Location locLastIn1stBlock =
        locate(scanner, composeSortedKey(KEY, 3 * records1stBlock,
            records1stBlock - 1).getBytes());
    Location locFirstIn2ndBlock =
        locate(scanner, composeSortedKey(KEY, 3 * records1stBlock,
            records1stBlock).getBytes());
    Location locX = locate(scanner, "keyX".getBytes());
    Assert.assertEquals(scanner.endLocation, locX);
    scanner.close();
    reader.close();
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.zebra.tfile.TFile.Reader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.