Package edu.uci.ics.crawler4j.crawler

Examples of edu.uci.ics.crawler4j.crawler.CrawlController.start()

61
62
63
64
65
66
67
68
69
    // controller.setProxy("proxyserver.example.com", 8080, username,
    // password);

    MyImageCrawler.configure(crawlDomains, storageFolder);

    controller.start(MyImageCrawler.class, numberOfCrawlers);
  }

}
View Full Code Here
102
103
104
105
106
107
108
109
110
111
      /*
       * Start the crawl. This is a blocking operation, meaning
       * that your code will reach the line after this only when
       * crawling is finished.
       */
      controller.start(MyCrawler.class, numberOfCrawlers);
    }

}

View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
      String rootFolder = args[0];
      int numberOfCrawlers = Integer.parseInt(args[1]);
     
      CrawlController controller = new CrawlController(rootFolder);   
      controller.addSeed("http://www.ics.uci.edu/");
      controller.start(MyCrawler.class, numberOfCrawlers); 
     
      List<Object> crawlersLocalData = controller.getCrawlersLocalData();
      long totalLinks = 0;
      long totalTextSize = 0;
      int totalProcessedPages = 0;
View Full Code Here
107
108
109
110
111
112
113
114
    /*
     * Start the crawl. This is a blocking operation, meaning that your code
     * will reach the line after this only when crawling is finished.
     */
    controller.start(StatusHandlerCrawler.class, numberOfCrawlers);
  }
}
View Full Code Here
66
67
68
69
70
71
72
73
74
      controller.addSeed(domain);
    }

    ImageCrawler.configure(crawlDomains, storageFolder);

    controller.start(ImageCrawler.class, numberOfCrawlers);
  }

}
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

    controller.addSeed("http://www.ics.uci.edu/");
    controller.start(LocalDataCollectorCrawler.class, numberOfCrawlers);

    List<Object> crawlersLocalData = controller.getCrawlersLocalData();
    long totalLinks = 0;
    long totalTextSize = 0;
    int totalProcessedPages = 0;
View Full Code Here
108
109
110
111
112
113
114
115
    /*
     * Start the crawl. This is a blocking operation, meaning that your code
     * will reach the line after this only when crawling is finished.
     */
    controller.start(BasicCrawler.class, numberOfCrawlers);
  }
}
View Full Code Here
25
26
27
28
29
30
31
32
33
34
35
      controller.addSeed("http://zookeeper.apache.org/");
      controller.setPolitenessDelay(1000);
      controller.setMaximumCrawlDepth(2);
      controller.setMaximumPagesToFetch(1);

      controller.start(MyCrawler.class, numberOfCrawlers);
    }


  public static class MyCrawler extends WebCrawler {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.