Package org.apache.nutch.storage

Examples of org.apache.nutch.storage.ParseStatus

125
126
127
128
129
130
131
132
133
134
135
136
137
        return;
      }
     

      parseUtil.process(key, page);
      ParseStatus pstatus = page.getParseStatus();
      if (pstatus != null) {
        context.getCounter("ParserStatus",
            ParseStatusCodes.majorCodes[pstatus.getMajorCode()]).increment(1);
      }

      context.write(key, page);
    }   
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
    };

    @Override
    public void map(String key, WebPage page, Context context)
    throws IOException, InterruptedException {
      ParseStatus pstatus = page.getParseStatus();
      if (pstatus == null || !ParseStatusUtils.isSuccess(pstatus)
          || pstatus.getMinorCode() == ParseStatusCodes.SUCCESS_REDIRECT) {
        return; // filter urls not parsed
      }

      Utf8 mark = Mark.UPDATEDB_MARK.checkMark(page);
      if (!batchId.equals(REINDEX)) {
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
    if (outlinks.size() > 0) {
      Outlink[] old = parse.getOutlinks();
      String title = parse.getTitle();
      List<Outlink> list = Arrays.asList(old);
      outlinks.addAll(list);
      ParseStatus status = parse.getParseStatus();
      String text = parse.getText();
      Outlink[] newlinks = outlinks.toArray(new Outlink[outlinks.size()]);
      return new Parse(text, title, newlinks, status);
    }
    return parse;
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
          res.put(f, simpleMeta);
        } else if ("protocolStatus".equals(f)) {
          ProtocolStatus ps = page.getProtocolStatus();
          res.put(f, ProtocolStatusUtils.toString(ps));
        } else if ("parseStatus".equals(f)) {
          ParseStatus ps = page.getParseStatus();
          res.put(f, ParseStatusUtils.toString(ps));
        } else if ("signature".equals(f)) {
          ByteBuffer bb = page.getSignature();
          res.put(f, StringUtil.toHexString(bb));
        } else if ("content".equals(f)) {
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
      if (LOG.isTraceEnabled()) {
        LOG.trace("found "+outlinks.length+" outlinks in "+ url);
      }
    }

    ParseStatus status = new ParseStatus();
    status.setMajorCode(ParseStatusCodes.SUCCESS);
    if (metaTags.getRefresh()) {
      status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT);
      status.addToArgs(new Utf8(metaTags.getRefreshHref().toString()));
      status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime())));
    }

    Parse parse = new Parse(text, title, outlinks, status);
    parse = htmlParseFilters.filter(url, page, parse, metaTags, root);
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
    if (outlinks.length == 0) {
      outlinks = OutlinkExtractor.getOutlinks(text, getConf());
    }

    ParseStatus status = ParseStatusUtils.STATUS_SUCCESS;
    if (metaTags.getRefresh()) {
      status.setMinorCode(ParseStatusCodes.SUCCESS_REDIRECT);
      status.addToArgs(new Utf8(metaTags.getRefreshHref().toString()));
      status.addToArgs(new Utf8(Integer.toString(metaTags.getRefreshTime())));
    }

    Parse parse = new Parse(text, title, outlinks, status);
    parse = htmlParseFilters.filter(url, page, parse, metaTags, root);
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    if (outlinks.size() > 0) {
      Outlink[] old = parse.getOutlinks();
      String title = parse.getTitle();
      List<Outlink> list = Arrays.asList(old);
      outlinks.addAll(list);
      ParseStatus status = parse.getParseStatus();
      String text = parse.getText();
      Outlink[] newlinks = outlinks.toArray(new Outlink[outlinks.size()]);
      return new Parse(text, title, newlinks, status);
    }
    return parse;
View Full Code Here
119
120
121
122
123
124
125
126
127
128
129
130
131
132
        return;
      }
     

      URLWebPage redirectedPage = parseUtil.process(key, page);
      ParseStatus pstatus = page.getParseStatus();
      if (pstatus != null) {
        context.getCounter("ParserStatus",
            ParseStatusCodes.majorCodes[pstatus.getMajorCode()]).increment(1);
      }

      if (redirectedPage != null) {
        context.write(TableUtil.reverseUrl(redirectedPage.getUrl()),
                      redirectedPage.getDatum());
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
          res.put(f, simpleMeta);
        } else if ("protocolStatus".equals(f)) {
          ProtocolStatus ps = page.getProtocolStatus();
          res.put(f, ProtocolStatusUtils.toString(ps));
        } else if ("parseStatus".equals(f)) {
          ParseStatus ps = page.getParseStatus();
          res.put(f, ParseStatusUtils.toString(ps));
        } else if ("signature".equals(f)) {
          ByteBuffer bb = page.getSignature();
          res.put(f, StringUtil.toHexString(bb.array()));
        } else if ("content".equals(f)) {
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
    }
    return null;
  }

  public static Parse getEmptyParse(Exception e, Configuration conf) {
    ParseStatus status = new ParseStatus();
    status.setMajorCode(ParseStatusCodes.FAILED);
    status.setMinorCode(ParseStatusCodes.FAILED_EXCEPTION);
    status.addToArgs(new Utf8(e.toString()));

    return new Parse("", "", new Outlink[0], status);
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.storage.ParseStatus

  • org.apache.gora.persistency.impl.StateManagerImpl
  • org.apache.nutch.api.DbReader$DbIterator
  • org.apache.nutch.indexer.IndexerJob$IndexerMapper
  • org.apache.nutch.parse.html.HtmlParser
  • org.apache.nutch.parse.js.JSParseFilter
  • org.apache.nutch.parse.ParserJob$ParserMapper
  • org.apache.nutch.parse.ParseStatusUtils
  • org.apache.nutch.parse.tika.TikaParser
  • org.apache.avro.AvroRuntimeException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.