Package org.archive.io.warc

Examples of org.archive.io.warc.WARCReader

37
38
39
40
41
42
43
44
45
46
47
48
      ARCReader reader = ARCReaderFactory.get(file,offset);
      r = ARCArchiveRecordToResource(reader.get(),reader);

    } else if (isWarc(name)) {

      WARCReader reader = WARCReaderFactory.get(file,offset);
      r = WARCArchiveRecordToResource(reader.get(),reader);

    } else {
      throw new ResourceNotAvailableException("Unknown extension");
    }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
      ARCReader reader = ARCReaderFactory.get(url, offset);
      r = ARCArchiveRecordToResource(reader.get(),reader);

    } else if (isWarc(name)) {

      WARCReader reader = WARCReaderFactory.get(url, offset);
      r = WARCArchiveRecordToResource(reader.get(),reader);

    } else {
      throw new ResourceNotAvailableException("Unknown extension");
    }
    return r;
View Full Code Here
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    WARCRecordToSearchResultAdapter adapter2 =
      new WARCRecordToSearchResultAdapter();
    adapter2.setCanonicalizer(canonicalizer);
   
    WARCReader reader = WARCReaderFactory.get(warc);
   
    ArchiveReaderCloseableIterator itr1 =
      new ArchiveReaderCloseableIterator(reader,reader.iterator());

    CloseableIterator<WARCRecord> itr2 =
      new AdaptedIterator<ArchiveRecord, WARCRecord>(itr1, adapter1);

    return new AdaptedIterator<WARCRecord, SearchResult>(itr2, adapter2);
View Full Code Here
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
   public void transform(final File warc, final File dir, final String prefix,
           final String suffix, final boolean force)
   throws IOException, java.text.ParseException {
       FileUtils.assertReadable(warc);
       FileUtils.assertReadable(dir);
       WARCReader reader = WARCReaderFactory.get(warc);
       List<String> metadata =  new ArrayList<String>();
       metadata.add("Made from " + reader.getReaderIdentifier() + " by " +
           this.getClass().getName() + "/" + getRevision());
       ARCWriter writer =
           new ARCWriter(
                   new AtomicInteger(),
                   new WriterPoolSettingsData(
                           prefix,
                           suffix,
                           -12,
                           reader.isCompressed(),
                           Arrays.asList(new File [] {dir}),
                           metadata));
       transform(reader, writer);
   }
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
293
            // check the warc records
            List<File> warcDirs = warcWriter.calcOutputDirs();
            assertEquals(1, warcDirs.size());
            String[] warcs = warcDirs.get(0).list();
            assertEquals(1, warcs.length);
            WARCReader warcReader = WARCReaderFactory.get(new File(warcDirs.get(0), warcs[0]));
            Iterator<ArchiveRecord> recordIterator = warcReader.iterator();
           
            ArchiveRecord record = recordIterator.next();
            assertEquals(WARCRecordType.warcinfo.toString(), record.getHeader().getHeaderValue(HEADER_KEY_TYPE));
           
            assertTrue(recordIterator.hasNext());
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
207
    if(reader instanceof ARCReader) {
      ARCReader areader = (ARCReader) reader;
      r = ARCArchiveRecordToResource(areader.get(),areader);
   
    } else if(reader instanceof WARCReader) {
      WARCReader wreader = (WARCReader) reader;
      r = WARCArchiveRecordToResource(wreader.get(),wreader);
     
    } else {
      throw new ResourceNotAvailableException("Unknown ArchiveReader");
    }
    long elapsed = System.currentTimeMillis() - start;
View Full Code Here
TOP

Related Classes of org.archive.io.warc.WARCReader

  • org.archive.io.Warc2Arc
  • org.archive.modules.recrawl.ContentDigestHistoryTest
  • org.archive.wayback.resourcestore.ResourceFactory
  • org.archive.wayback.resourcestore.resourcefile.ResourceFactory
  • org.archive.wayback.resourcestore.WarcIndexer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.