Package org.archive.io

Examples of org.archive.io.ReplayInputStream

221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
   * 3. Content - The HTTP response content, plus content-type.
   *
   * For more on this format, see '?'.
   */
  protected void writeMimeFile(CrawlURI curi) throws IOException {
      ReplayInputStream ris = null;
      OutputStream out = null;
               
      try {
          String boundary = BOUNDARY_START + stringToMD5(curi.toString());
          ris = curi.getRecorder().getRecordedInput().
              getReplayInputStream();
          out = initOutputStream(curi);
         
          // Part 1: Archive info
          writeArchiveInfoPart(boundary, curi, ris, out);

          // Part 2: Header info + HTTP header
          writeHeaderPart(boundary, ris, out);

          // Part 3: Content info + HTTP content
          writeContentPart(boundary, curi, ris, out);

          // And finally the terminator string
          String terminator = "\n--" + boundary + "--\n";
          out.write(terminator.getBytes());
      } finally {
          if (ris != null)
              ris.close();
          if (out != null)
              out.close();
      }
  }
View Full Code Here
342
343
344
345
346
347
348
349
350
351
352
        String ip = (String)curi.getData().get(A_DNS_SERVER_IP_LABEL);
        if (ip != null && ip.length() > 0) {
            recordInfo.addExtraHeader(HEADER_KEY_IP, ip);
        }
       
        ReplayInputStream ris =
            curi.getRecorder().getRecordedInput().getReplayInputStream();
        recordInfo.setContentStream(ris);
       
        try {
            w.writeRecord(recordInfo);
View Full Code Here
371
372
373
374
375
376
377
378
379
380
381
        Object whoisServerIP = curi.getData().get(CoreAttributeConstants.A_WHOIS_SERVER_IP);
        if (whoisServerIP != null) {
            recordInfo.addExtraHeader(HEADER_KEY_IP, whoisServerIP.toString());
        }
       
        ReplayInputStream ris =
            curi.getRecorder().getRecordedInput().getReplayInputStream();
        recordInfo.setContentStream(ris);
       
        try {
            w.writeRecord(recordInfo);
View Full Code Here
521
522
523
524
525
526
527
528
529
530
531
        recordInfo.setEnforceLength(true);
       
        final URI uid = qualifyRecordID(baseid, TYPE, WARCRecordType.request.toString());
        recordInfo.setRecordId(uid);
       
        ReplayInputStream
            ris = curi.getRecorder().getRecordedOutput().getReplayInputStream();
        recordInfo.setContentStream(ris);
       
        try {
            w.writeRecord(recordInfo);
View Full Code Here
558
559
560
561
562
563
564
565
566
567
568
        recordInfo.setRecordId(baseid);
        recordInfo.setExtraHeaders(namedFields);
        recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize());
        recordInfo.setEnforceLength(true);
       
        ReplayInputStream ris =
            curi.getRecorder().getRecordedInput().getReplayInputStream();
        recordInfo.setContentStream(ris);
       
        try {
            w.writeRecord(recordInfo);
View Full Code Here
586
587
588
589
590
591
592
593
594
595
596
        recordInfo.setRecordId(baseid);
        recordInfo.setExtraHeaders(namedFields);
        recordInfo.setContentLength(curi.getRecorder().getRecordedInput().getSize());
        recordInfo.setEnforceLength(true);
       
        ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream();
        recordInfo.setContentStream(ris);
        try {
            w.writeRecord(recordInfo);
        } finally {
            IOUtils.closeQuietly(ris);
View Full Code Here
644
645
646
647
648
649
650
651
652
653
654
          for ( String key : revisitHeaders.keySet()) {
              headers.addLabelValue(key, revisitHeaders.get(key));         
          }
        }
       
    ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream();
    recordInfo.setContentStream(ris);

    try {
      w.writeRecord(recordInfo);
    } finally {
View Full Code Here
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
       the file rename fails
    */
    private void writeToPath(RecordingInputStream recis, File dest)
        throws IOException {
        File tf = new File (dest.getPath() + "N");
        ReplayInputStream replayis = null;
        FileOutputStream fos = null;
        try {
            replayis = recis.getMessageBodyReplayInputStream();
            fos = new FileOutputStream(tf);

            replayis.readFullyTo(fos);
        } finally {
            IOUtils.closeQuietly(replayis);
            IOUtils.closeQuietly(fos);
        }
        if (!tf.renameTo(dest)) {
View Full Code Here
509
510
511
512
513
514
515
516
517
518
519
    public void testGapError() throws IOException {
      ARCWriter writer = createArcWithOneRecord("testGapError", true);
        String content = getContent();
        // Make a 'weird' RIS that returns bad 'remaining' length
        // awhen remaining should be 0
        ReplayInputStream ris = new ReplayInputStream(content.getBytes(),
                content.length(), null) {
            public long remaining() {
                return (super.remaining()==0) ? -1 : super.remaining();
            }
        };
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
    protected ProcessResult innerProcessResult(CrawlURI puri) {
        CrawlURI curi = (CrawlURI)puri;
       
        long recordLength = getRecordedSize(curi);
       
        ReplayInputStream ris = null;
        try {
            if (shouldWrite(curi)) {
                ris = curi.getRecorder().getRecordedInput()
                        .getReplayInputStream();
                return write(curi, recordLength, ris, getHostAddress(curi));
View Full Code Here
TOP

Related Classes of org.archive.io.ReplayInputStream

  • org.archive.io.arc.ARCWriterTest
  • org.archive.modules.extractor.ExtractorDOC
  • org.archive.modules.writer.ARCWriterProcessor
  • org.archive.modules.writer.Kw3WriterProcessor
  • org.archive.modules.writer.MirrorWriterProcessor
  • org.archive.modules.writer.WARCWriterProcessor
  • org.archive.wayback.liveweb.URLtoARCCacher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.