Package org.apache.nutch.storage

Examples of org.apache.nutch.storage.Host

338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
        // Create a new queue
        if (useHostSettings) {
          // Use host specific queue settings (if defined in the host table)
          try {
            String hostname = id.substring(id.indexOf("://")+3);
            Host host = hostDb.getByHostName(hostname);
            if (host != null) {
              fiq = new FetchItemQueue(conf,
                                       host.getInt("q_mt", maxThreads),
                                       host.getLong("q_cd", crawlDelay),
                                       host.getLong("q_mcd", minCrawlDelay));
            }
           
          } catch (IOException e) {
            LOG.error("Error while trying to access host settings", e);
          }
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
          metadata.put(metaname, metavalue);
        }
      }

      // now add the metadata
      Host host = new Host();

      Iterator<String> keysIter = metadata.keySet().iterator();
      while (keysIter.hasNext()) {
        String keymd = keysIter.next();
        String valuemd = metadata.get(keymd);
        host.putToMetadata(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes()));
      }
      String hostname;
      if (url.indexOf("://")> -1) {
        hostname=new URL(url).getHost();
      } else {
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
 
  public Host get(final String key) throws IOException {
    Callable<CacheHost> valueLoader = new Callable<CacheHost>() {
      @Override
      public CacheHost call() throws Exception {
        Host host = hostStore.get(key);
        if (host == null) return NULL_HOST;
        return new CacheHost(host, System.currentTimeMillis());
      } 
    };
    CacheHost cachedHost;
View Full Code Here
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
        // Create a new queue
        if (useHostSettings) {
          // Use host specific queue settings (if defined in the host table)
          try {
            String hostname = id.substring(id.indexOf("://")+3);
            Host host = hostDb.getByHostName(hostname);
            if (host != null) {
              fiq = new FetchItemQueue(conf,
                                       host.getInt("q_mt", maxThreads),
                                       host.getLong("q_cd", crawlDelay),
                                       host.getLong("q_mcd", minCrawlDelay));
            }
           
          } catch (IOException e) {
            LOG.error("Error while trying to access host settings", e);
          }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
    // query.setFields(Host._ALL_FIELDS);
    Result<String, Host> result = datastore.execute(query);

    while (result.next()) {
      String hostName = TableUtil.unreverseUrl(result.getKey());
      Host host = result.get();
      System.out.println(hostName);
      System.out.println(host);
    }
    result.close();
    datastore.close();
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
          metadata.put(metaname, metavalue);
        }
      }

      // now add the metadata
      Host host = new Host();

      Iterator<String> keysIter = metadata.keySet().iterator();
      while (keysIter.hasNext()) {
        String keymd = keysIter.next();
        String valuemd = metadata.get(keymd);
        host.putToMetadata(new Utf8(keymd), ByteBuffer.wrap(valuemd.getBytes()));
      }

      String hostkey = TableUtil.reverseHost(url);
      context.write(hostkey, host);
    }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
        }
      }
    }
   
    // output host data
    Host host = new Host();
    host.putToMetadata(new Utf8("p"),ByteBuffer.wrap(Integer.toString(numPages).getBytes()));
    if (numFetched > 0) {
      host.putToMetadata(new Utf8("f"),ByteBuffer.wrap(Integer.toString(numFetched).getBytes()));
    }
    for (String inlink: inlinkCount.getKeys()) {
      host.putToInlinks(new Utf8(inlink), new Utf8(Integer.toString(inlinkCount.getCount(inlink))));
    }
    for (String outlink: outlinkCount.getKeys()) {
      host.putToOutlinks(new Utf8(outlink), new Utf8(Integer.toString(outlinkCount.getCount(outlink))));
    }
   
    context.write(key.toString(), host);
  }
View Full Code Here
107
108
109
110
111
112
113
114
115
116
117
 
  public Host get(final String key) throws IOException {
    Callable<CacheHost> valueLoader = new Callable<CacheHost>() {
      @Override
      public CacheHost call() throws Exception {
        Host host = hostStore.get(key);
        if (host == null) return NULL_HOST;
        return new CacheHost(host, System.currentTimeMillis());
      } 
    };
    CacheHost cachedHost;
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
    Result<String, Host> result = datastore.execute(query);

    while (result.next()) {
      try {
        String hostName = TableUtil.unreverseUrl(result.getKey());
        Host host = result.get();
        System.out.println(hostName);
        System.out.println(host);
      } catch (Exception e) {
        e.printStackTrace();
      }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
 
  public Host get(final String key) throws IOException {
    Callable<CacheHost> valueLoader = new Callable<CacheHost>() {
      @Override
      public CacheHost call() throws Exception {
        Host host = hostStore.get(key);
        if (host == null) return NULL_HOST;
        return new CacheHost(host, System.currentTimeMillis());
      } 
    };
    CacheHost cachedHost;
View Full Code Here
TOP

Related Classes of org.apache.nutch.storage.Host

  • org.apache.avro.util.Utf8
  • org.apache.gora.persistency.impl.StateManagerImpl
  • org.apache.nutch.fetcher.FetcherReducer$FetchItemQueues
  • org.apache.nutch.host.HostDb
  • org.apache.nutch.host.HostDbReader
  • org.apache.nutch.host.HostDbUpdateReducer
  • org.apache.nutch.host.HostInjectorJob$UrlMapper
  • org.apache.avro.AvroRuntimeException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.