Examples of BlockRecoveryInfo

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
              dnRegistration.getIpcPort() == id.getIpcPort()) {
            datanode = this;
          } else {
            datanode = DataNode.createInterDataNodeProtocolProxy(id, getConf());
          }
          BlockRecoveryInfo info = datanode.startBlockRecovery(block);
          if (info == null) {
            LOG.info("No block metadata found for block " + block + " on datanode "
                + id);
            continue;
          }
          if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
            LOG.info("Only old generation stamp " + info.getBlock().getGenerationStamp()
                + " found on datanode " + id + " (needed block=" +
                block + ")");
            continue;
          }
          blockRecords.add(new BlockRecord(id, datanode, info));

          if (info.wasRecoveredOnStartup()) {
            rwrCount++;
          } else {
            rbwCount++;
          }
        } catch (IOException e) {
          ++errorCount;
          InterDatanodeProtocol.LOG.warn(
              "Failed to getBlockMetaDataInfo for block (=" + block
              + ") from datanode (=" + id + ")", e);
        }
      }

      // If we *only* have replicas from post-DN-restart, then we should
      // include them in determining length. Otherwise they might cause us
      // to truncate too short.
      boolean shouldRecoverRwrs = (rbwCount == 0);
     
      List<BlockRecord> syncList = new ArrayList<BlockRecord>();
      long minlength = Long.MAX_VALUE;
     
      for (BlockRecord record : blockRecords) {
        BlockRecoveryInfo info = record.info;
        assert (info != null && info.getBlock().getGenerationStamp() >= block.getGenerationStamp());
        if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
          LOG.info("Not recovering replica " + record + " since it was recovered on "
              + "startup and we have better replicas");
          continue;
        }
        if (keepLength) {
          if (info.getBlock().getNumBytes() == block.getNumBytes()) {
            syncList.add(record);
          }
        } else {         
          syncList.add(record);
          if (info.getBlock().getNumBytes() < minlength) {
            minlength = info.getBlock().getNumBytes();
          }
        }
      }

      if (syncList.isEmpty() && errorCount > 0) {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
    synchronized (this) {
      ActiveFile activeFile = ongoingCreates.get(stored);
      boolean isRecovery = (activeFile != null) && activeFile.wasRecoveredOnStartup;
     
     
      BlockRecoveryInfo info = new BlockRecoveryInfo(
          stored, isRecovery);
      if (DataNode.LOG.isDebugEnabled()) {
        DataNode.LOG.debug("getBlockMetaDataInfo successful block=" + stored +
                  " length " + stored.getNumBytes() +
                  " genstamp " + stored.getGenerationStamp());
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

687
688
689
690
691
692
693
  @Override
  public BlockRecoveryInfo startBlockRecovery(long blockId)
      throws IOException {
    Block stored = getStoredBlock(blockId);
    return new BlockRecoveryInfo(stored, false);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
            datanode = DataNode.createInterDataNodeProtocolProxy(
      id, getConf(), socketTimeout);
            datanodeProxies.add(datanode);
          }
          throwIfAfterTime(deadline);
          BlockRecoveryInfo info = datanode.startBlockRecovery(namespaceId, block);
          if (info == null) {
            LOG.info("No block metadata found for block " + block + " on datanode "
                + id);
            continue;
          }
          if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
            LOG.info("Only old generation stamp " + info.getBlock().getGenerationStamp()
                + " found on datanode " + id + " (needed block=" +
                block + ")");
            continue;
          }
          blockRecords.add(new BlockRecord(id, datanode, info));

          if (info.wasRecoveredOnStartup()) {
            rwrCount++;
          } else {
            rbwCount++;
          }
        } catch (BlockRecoveryTimeoutException e) {
          throw e;
        } catch (IOException e) {
          ++errorCount;
          InterDatanodeProtocol.LOG.warn(
              "Failed to getBlockMetaDataInfo for block (=" + block
              + ") from datanode (=" + id + ")", e);
        }
      }

      // If we *only* have replicas from post-DN-restart, then we should
      // include them in determining length. Otherwise they might cause us
      // to truncate too short.
      boolean shouldRecoverRwrs = (rbwCount == 0);

      List<BlockRecord> syncList = new ArrayList<BlockRecord>();
      long minlength = Long.MAX_VALUE;

      for (BlockRecord record : blockRecords) {
        BlockRecoveryInfo info = record.info;
        assert (info != null && info.getBlock().getGenerationStamp() >= block.getGenerationStamp());
        if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
          LOG.info("Not recovering replica " + record + " since it was recovered on "
              + "startup and we have better replicas");
          continue;
        }
        if (keepLength) {
          if (info.getBlock().getNumBytes() == block.getNumBytes()) {
            syncList.add(record);
          }
        } else {
          syncList.add(record);
          if (info.getBlock().getNumBytes() < minlength) {
            minlength = info.getBlock().getNumBytes();
          }
        }
      }

      if (syncList.isEmpty() && errorCount > 0) {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
      ActiveFile activeFile = volumeMap.getOngoingCreates(namespaceId, stored);
      boolean isRecovery = (activeFile != null) && activeFile.wasRecoveredOnStartup;


      BlockRecoveryInfo info = new BlockRecoveryInfo(stored, isRecovery);
      if (DataNode.LOG.isDebugEnabled()) {
        DataNode.LOG.debug("getBlockMetaDataInfo successful block=" + stored +
                  " length " + stored.getNumBytes() +
                  " genstamp " + stored.getGenerationStamp());
      }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

691
692
693
694
695
696
697
  @Override
  public BlockRecoveryInfo startBlockRecovery(long blockId)
      throws IOException {
    Block stored = getStoredBlock(blockId);
    return new BlockRecoveryInfo(stored, false);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
      for (DatanodeInfo id : targets) {
        try {
          InterDatanodeProtocol datanode = dnRegistration.equals(id) ? this
            : DataNode.createInterDataNodeProtocolProxy(
                id, getConf(), socketTimeout, connectToDnViaHostname);
          BlockRecoveryInfo info = datanode.startBlockRecovery(block);
          if (info == null) {
            LOG.info("No block metadata found for block " + block + " on datanode "
                + id);
            continue;
          }
          if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
            LOG.info("Only old generation stamp " + info.getBlock().getGenerationStamp()
                + " found on datanode " + id + " (needed block=" +
                block + ")");
            continue;
          }
          blockRecords.add(new BlockRecord(id, datanode, info));

          if (info.wasRecoveredOnStartup()) {
            rwrCount++;
          } else {
            rbwCount++;
          }
        } catch (IOException e) {
          ++errorCount;
          InterDatanodeProtocol.LOG.warn(
              "Failed to getBlockMetaDataInfo for block (=" + block
              + ") from datanode (=" + id + ")", e);
        }
      }

      // If we *only* have replicas from post-DN-restart, then we should
      // include them in determining length. Otherwise they might cause us
      // to truncate too short.
      boolean shouldRecoverRwrs = (rbwCount == 0);
     
      List<BlockRecord> syncList = new ArrayList<BlockRecord>();
      long minlength = Long.MAX_VALUE;
     
      for (BlockRecord record : blockRecords) {
        BlockRecoveryInfo info = record.info;
        assert (info != null && info.getBlock().getGenerationStamp() >= block.getGenerationStamp());
        if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
          LOG.info("Not recovering replica " + record + " since it was recovered on "
              + "startup and we have better replicas");
          continue;
        }
        if (keepLength) {
          if (info.getBlock().getNumBytes() == block.getNumBytes()) {
            syncList.add(record);
          }
        } else {         
          syncList.add(record);
          if (info.getBlock().getNumBytes() < minlength) {
            minlength = info.getBlock().getNumBytes();
          }
        }
      }

      if (syncList.isEmpty() && errorCount > 0) {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
      for (DatanodeInfo id : targets) {
        try {
          InterDatanodeProtocol datanode = dnRegistration.equals(id) ? this
            : DataNode.createInterDataNodeProtocolProxy(
                id, getConf(), socketTimeout, connectToDnViaHostname);
          BlockRecoveryInfo info = datanode.startBlockRecovery(block);
          if (info == null) {
            LOG.info("No block metadata found for " + block + " on datanode "
                + id);
            continue;
          }
          if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
            LOG.info("Only old generation stamp " + info.getBlock().getGenerationStamp()
                + " found on datanode " + id + " (needed block=" +
                block + ")");
            continue;
          }
          blockRecords.add(new BlockRecord(id, datanode, info));

          if (info.wasRecoveredOnStartup()) {
            rwrCount++;
          } else {
            rbwCount++;
          }
        } catch (IOException e) {
          ++errorCount;
          InterDatanodeProtocol.LOG.warn(
              "Failed to getBlockMetaDataInfo for block (=" + block
              + ") from datanode (=" + id + ")", e);
        }
      }

      // If we *only* have replicas from post-DN-restart, then we should
      // include them in determining length. Otherwise they might cause us
      // to truncate too short.
      boolean shouldRecoverRwrs = (rbwCount == 0);
     
      List<BlockRecord> syncList = new ArrayList<BlockRecord>();
      long minlength = Long.MAX_VALUE;
     
      for (BlockRecord record : blockRecords) {
        BlockRecoveryInfo info = record.info;
        assert (info != null && info.getBlock().getGenerationStamp() >= block.getGenerationStamp());
        if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
          LOG.info("Not recovering replica " + record + " since it was recovered on "
              + "startup and we have better replicas");
          continue;
        }
        if (keepLength) {
          if (info.getBlock().getNumBytes() == block.getNumBytes()) {
            syncList.add(record);
          }
        } else {         
          syncList.add(record);
          if (info.getBlock().getNumBytes() < minlength) {
            minlength = info.getBlock().getNumBytes();
          }
        }
      }

      if (syncList.isEmpty() && errorCount > 0) {
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
    synchronized (this) {
      ActiveFile activeFile = ongoingCreates.get(stored);
      boolean isRecovery = (activeFile != null) && activeFile.wasRecoveredOnStartup;
     
     
      BlockRecoveryInfo info = new BlockRecoveryInfo(
          stored, isRecovery);
      if (DataNode.LOG.isDebugEnabled()) {
        DataNode.LOG.debug("getBlockMetaDataInfo successful block=" + stored +
                  " length " + stored.getNumBytes() +
                  " genstamp " + stored.getGenerationStamp());
View Full Code Here

Examples of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryInfo

2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
      for (DatanodeInfo id : targets) {
        try {
          InterDatanodeProtocol datanode = dnRegistration.equals(id) ? this
            : DataNode.createInterDataNodeProtocolProxy(
                id, getConf(), socketTimeout, connectToDnViaHostname);
          BlockRecoveryInfo info = datanode.startBlockRecovery(block);
          if (info == null) {
            LOG.info("No block metadata found for " + block + " on datanode "
                + id);
            continue;
          }
          if (info.getBlock().getGenerationStamp() < block.getGenerationStamp()) {
            LOG.info("Only old generation stamp " + info.getBlock().getGenerationStamp()
                + " found on datanode " + id + " (needed block=" +
                block + ")");
            continue;
          }
          blockRecords.add(new BlockRecord(id, datanode, info));

          if (info.wasRecoveredOnStartup()) {
            rwrCount++;
          } else {
            rbwCount++;
          }
        } catch (IOException e) {
          ++errorCount;
          InterDatanodeProtocol.LOG.warn(
              "Failed to getBlockMetaDataInfo for block (=" + block
              + ") from datanode (=" + id + ")", e);
        }
      }

      // If we *only* have replicas from post-DN-restart, then we should
      // include them in determining length. Otherwise they might cause us
      // to truncate too short.
      boolean shouldRecoverRwrs = (rbwCount == 0);
     
      List<BlockRecord> syncList = new ArrayList<BlockRecord>();
      long minlength = Long.MAX_VALUE;
     
      for (BlockRecord record : blockRecords) {
        BlockRecoveryInfo info = record.info;
        assert (info != null && info.getBlock().getGenerationStamp() >= block.getGenerationStamp());
        if (!shouldRecoverRwrs && info.wasRecoveredOnStartup()) {
          LOG.info("Not recovering replica " + record + " since it was recovered on "
              + "startup and we have better replicas");
          continue;
        }
        if (keepLength) {
          if (info.getBlock().getNumBytes() == block.getNumBytes()) {
            syncList.add(record);
          }
        } else {         
          syncList.add(record);
          if (info.getBlock().getNumBytes() < minlength) {
            minlength = info.getBlock().getNumBytes();
          }
        }
      }

      if (syncList.isEmpty() && errorCount > 0) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.