Package org.apache.hadoop.hdfs.shortcircuit

Examples of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache


        (short)1, SEED);
    byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
    byte expected[] = DFSTestUtil.
        calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    final ShortCircuitCache cache =
        fs.dfs.getClientContext().getShortCircuitCache();
    final DatanodeInfo datanode =
        new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
    cache.getDfsClientShmManager().visit(new Visitor() {
      @Override
      public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info)
          throws IOException {
        Assert.assertEquals(1,  info.size());
        PerDatanodeVisitorInfo vinfo = info.get(datanode);
View Full Code Here


        (short)1, SEED);
    byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
    byte expected[] = DFSTestUtil.
        calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    final ShortCircuitCache cache =
        fs.dfs.getClientContext().getShortCircuitCache();
    Assert.assertEquals(null, cache.getDfsClientShmManager());
    cluster.shutdown();
  }
View Full Code Here

        (short)1, SEED);
    byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
    byte expected[] = DFSTestUtil.
        calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
    Assert.assertTrue(Arrays.equals(contents, expected));
    final ShortCircuitCache cache =
        fs.dfs.getClientContext().getShortCircuitCache();
    cache.close();
    Assert.assertTrue(cache.getDfsClientShmManager().
        getDomainSocketWatcher().isClosed());
    cluster.shutdown();
  }
View Full Code Here

        LOG.trace(this + ": " + pathInfo + " is not " +
            "usable for short circuit; giving up on BlockReaderLocal.");
      }
      return null;
    }
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
    ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
    InvalidToken exc = info.getInvalidTokenException();
    if (exc != null) {
      if (LOG.isTraceEnabled()) {
        LOG.trace(this + ": got InvalidToken exception while trying to " +
            "construct BlockReaderLocal via " + pathInfo.getPath());
View Full Code Here

      curPeer = nextDomainPeer();
      if (curPeer == null) break;
      if (curPeer.fromCache) remainingCacheTries--;
      DomainPeer peer = (DomainPeer)curPeer.peer;
      Slot slot = null;
      ShortCircuitCache cache = clientContext.getShortCircuitCache();
      try {
        MutableBoolean usedPeer = new MutableBoolean(false);
        slot = cache.allocShmSlot(datanode, peer, usedPeer,
            new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()),
            clientName);
        if (usedPeer.booleanValue()) {
          if (LOG.isTraceEnabled()) {
            LOG.trace(this + ": allocShmSlot used up our previous socket " +
              peer.getDomainSocket() + ".  Allocating a new one...");
          }
          curPeer = nextDomainPeer();
          if (curPeer == null) break;
          peer = (DomainPeer)curPeer.peer;
        }
        ShortCircuitReplicaInfo info = requestFileDescriptors(peer, slot);
        clientContext.getPeerCache().put(datanode, peer);
        return info;
      } catch (IOException e) {
        if (slot != null) {
          cache.freeSlot(slot);
        }
        if (curPeer.fromCache) {
          // Handle an I/O error we got when using a cached socket.
          // These are considered less serious, because the socket may be stale.
          if (LOG.isDebugEnabled()) {
View Full Code Here

   * @throws  IOException If we encountered an I/O exception while communicating
   *          with the datanode.
   */
  private ShortCircuitReplicaInfo requestFileDescriptors(DomainPeer peer,
          Slot slot) throws IOException {
    ShortCircuitCache cache = clientContext.getShortCircuitCache();
    final DataOutputStream out =
        new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
    SlotId slotId = slot == null ? null : slot.getSlotId();
    new Sender(out).requestShortCircuitFds(block, token, slotId, 1);
    DataInputStream in = new DataInputStream(peer.getInputStream());
View Full Code Here

      fsIn = null;
      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
      File dataFile = MiniDFSCluster.getBlockFile(0, block);
      File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block);

      ShortCircuitCache shortCircuitCache =
          ClientContext.getFromConf(conf).getShortCircuitCache();
      cluster.shutdown();
      cluster = null;
      test.setup(dataFile, checksum);
      FileInputStream streams[] = {
View Full Code Here

  private boolean printedConfWarning = false;

  private ClientContext(String name, Conf conf) {
    this.name = name;
    this.confString = confAsString(conf);
    this.shortCircuitCache = new ShortCircuitCache(
        conf.shortCircuitStreamsCacheSize,
        conf.shortCircuitStreamsCacheExpiryMs,
        conf.shortCircuitMmapCacheSize,
        conf.shortCircuitMmapCacheExpiryMs,
        conf.shortCircuitMmapCacheRetryTimeout,
View Full Code Here

    fsIn = fs.open(TEST_PATH);
    byte original[] = new byte[TEST_FILE_LENGTH];
    IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
    fsIn.close();
    fsIn = fs.open(TEST_PATH);
    final ShortCircuitCache cache = ClientContext.get(
        CONTEXT, new DFSClient.Conf(conf)). getShortCircuitCache();
    cache.accept(new CountingVisitor(0, 5, 5, 0));
    results[0] = fsIn.read(null, 4096,
        EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    fsIn.seek(0);
    results[1] = fsIn.read(null, 4096,
        EnumSet.of(ReadOption.SKIP_CHECKSUMS));

    // The mmap should be of the first block of the file.
    final ExtendedBlock firstBlock =
        DFSTestUtil.getFirstBlock(fs, TEST_PATH);
    cache.accept(new CacheVisitor() {
      @Override
      public void visit(int numOutstandingMmaps,
          Map<ExtendedBlockId, ShortCircuitReplica> replicas,
          Map<ExtendedBlockId, InvalidToken> failedLoads,
          Map<Long, ShortCircuitReplica> evictable,
          Map<Long, ShortCircuitReplica> evictableMmapped) {
        ShortCircuitReplica replica = replicas.get(
            new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId()));
        Assert.assertNotNull(replica);
        Assert.assertTrue(replica.hasMmap());
        // The replica should not yet be evictable, since we have it open.
        Assert.assertNull(replica.getEvictableTimeNs());
      }
    });

    // Read more blocks.
    results[2] = fsIn.read(null, 4096,
        EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    results[3] = fsIn.read(null, 4096,
        EnumSet.of(ReadOption.SKIP_CHECKSUMS));

    // we should have 3 mmaps, 1 evictable
    cache.accept(new CountingVisitor(3, 5, 2, 0));

    // After we close the cursors, the mmaps should be evictable for
    // a brief period of time.  Then, they should be closed (we're
    // using a very quick timeout)
    for (ByteBuffer buffer : results) {
      if (buffer != null) {
        fsIn.releaseBuffer(buffer);
      }
    }
    fsIn.close();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {
      public Boolean get() {
        final MutableBoolean finished = new MutableBoolean(false);
        cache.accept(new CacheVisitor() {
          @Override
          public void visit(int numOutstandingMmaps,
              Map<ExtendedBlockId, ShortCircuitReplica> replicas,
              Map<ExtendedBlockId, InvalidToken> failedLoads,
              Map<Long, ShortCircuitReplica> evictable,
              Map<Long, ShortCircuitReplica> evictableMmapped) {
            finished.setValue(evictableMmapped.isEmpty());
          }
        });
        return finished.booleanValue();
      }
    }, 10, 60000);

    cache.accept(new CountingVisitor(0, -1, -1, -1));
   
    fs.close();
    cluster.shutdown();
  }
View Full Code Here

    fsIn2.close();
   
    // check that the replica is anchored
    final ExtendedBlock firstBlock =
        DFSTestUtil.getFirstBlock(fs, TEST_PATH);
    final ShortCircuitCache cache = ClientContext.get(
        CONTEXT, new DFSClient.Conf(conf)). getShortCircuitCache();
    waitForReplicaAnchorStatus(cache, firstBlock, true, true, 1);
    // Uncache the replica
    fs.removeCacheDirective(directiveId);
    waitForReplicaAnchorStatus(cache, firstBlock, false, true, 1);
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.