Package fr.eolya.extraction.tika

Examples of fr.eolya.extraction.tika.TikaWrapper.process()

430
431
432
433
434
435
436
437
438
439
440
                  String swfToHtmlPath = Utils.getValidPropertyPath(config.getProperty("/crawler/param[@name='swfToHtmlPath']", ""), null, "HOME");

                  TikaWrapper tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_HTML);
                  tikaWrapper.setSwfToHtmlPath(swfToHtmlPath);
                  tikaWrapper.process(urlLoader.getStream(), TikaWrapper.CONTENT_TYPE_SWF);
                  rawPage = tikaWrapper.getText();
                 
                  //MultiFormatTextExtractor extractor = new MultiFormatTextExtractor();
                  //extractor.setSwfToHtmlPath(swfToHtmlPath);
                  //rawPage = extractor.swfInputStreamToHtml(urlLoader.getStream());
View Full Code Here
366
367
368
369
370
371
372
373
374
375
376
          wrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_TEXT);
        }
      }
     
      wrapper.setTempPath(tmpPath);
      wrapper.process(input, tikaContentType);
     
      parserText = wrapper.getText();
      parserContentType = wrapper.getMetaContentType();
      if (contentType.startsWith("text/html")) {
        parserTitle = htmlParser.getBestTitle(wrapper.getMetaTitle());
View Full Code Here
231
232
233
234
235
236
237
238
239
240
241
                //ws.clear();
               
                //String rawPage = extractor.htmlPageToText(data, page, "");
                //String title = extractor.getTitle();
        TikaWrapper tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_HTML);
        tikaWrapper.process(urlLoader.getStream());
        String rawPage = tikaWrapper.getText();
        String title = tikaWrapper.getMetaTitle();

                ret += "<page_0><![CDATA[" + rawPage + "]]>" + "</page_0>";
                ret += "<title_0><![CDATA[" + title + "]]>" + "</title_0>";
View Full Code Here
241
242
243
244
245
246
247
248
249
250
251
                ret += "<title_0><![CDATA[" + title + "]]>" + "</title_0>";
               
                //rawPage = extractor.htmlPageToText(data, page, "boilerpipe_article");
                //title = extractor.getTitle();
        tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_TEXT_MAIN_BOILERPIPE_ARTICLE, TikaWrapper.CONTENT_TYPE_HTML);
        tikaWrapper.process(urlLoader.getStream());
        rawPage = tikaWrapper.getText();
        title = tikaWrapper.getMetaTitle();
                ret += "<page_1><![CDATA[" + rawPage + "]]>" + "</page_1>";
                ret += "<title_1><![CDATA[" + title + "]]>" + "</title_1>";
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
                ret += "<title_1><![CDATA[" + title + "]]>" + "</title_1>";
                //rawPage = extractor.htmlPageToText(data, page, "boilerpipe_default");
                //title = extractor.getTitle();
        tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_TEXT_MAIN_BOILERPIPE_DEFAULT, TikaWrapper.CONTENT_TYPE_HTML);
        tikaWrapper.process(urlLoader.getStream());
        rawPage = tikaWrapper.getText();
        title = tikaWrapper.getMetaTitle();
                ret += "<page_2><![CDATA[" + rawPage + "]]>" + "</page_2>";
                ret += "<title_2><![CDATA[" + title + "]]>" + "</title_2>";
View Full Code Here
259
260
261
262
263
264
265
266
267
268
269
                ret += "<title_2><![CDATA[" + title + "]]>" + "</title_2>";
                //rawPage = extractor.htmlPageToText(data, page, "boilerpipe_canola");
                //title = extractor.getTitle();
        tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_TEXT_MAIN_BOILERPIPE_CANOLA, TikaWrapper.CONTENT_TYPE_HTML);
        tikaWrapper.process(urlLoader.getStream());
        rawPage = tikaWrapper.getText();
        title = tikaWrapper.getMetaTitle();
                ret += "<page_3><![CDATA[" + rawPage + "]]>" + "</page_3>";
                ret += "<title_3><![CDATA[" + title + "]]>" + "</title_3>";
View Full Code Here
268
269
270
271
272
273
274
275
276
277
278
                ret += "<title_3><![CDATA[" + title + "]]>" + "</title_3>";

                //rawPage = extractor.htmlPageToText(data, page, "snacktory");
                //title = extractor.getTitle();
        tikaWrapper = new TikaWrapper(TikaWrapper.OUTPUT_FORMAT_TEXT_MAIN_SNACKTORY, TikaWrapper.CONTENT_TYPE_HTML);
        tikaWrapper.process(urlLoader.getStream());
        rawPage = tikaWrapper.getText();
        title = tikaWrapper.getMetaTitle();
                ret += "<page_4><![CDATA[" + rawPage + "]]>" + "</page_4>";
                ret += "<title_4><![CDATA[" + title + "]]>" + "</title_4>";
               
View Full Code Here
390
391
392
393
394
395
396
397
398
399
400
                } else {
                    //text = extractor.htmlPageToText(page, "", "");
                  in = IOUtils.toInputStream(page);
                }
               
        tikaWrapper.process(in, TikaWrapper.CONTENT_TYPE_HTML);
        text = tikaWrapper.getText();
       
                if (title==null || "".equals(title))
                    title = tikaWrapper.getMetaTitle();
               
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.