public static void main(String[] args) throws Exception {
String rootFolder = "/tmp";
int numberOfCrawlers = 1;
CrawlController controller = new CrawlController(rootFolder);
controller.addSeed("http://hadoop.apache.org/");
controller.addSeed("http://hadoop.apache.org/common/");
controller.addSeed("http://hadoop.apache.org/hdfs/");
controller.addSeed("http://hadoop.apache.org/mapreduce/");
controller.addSeed("http://avro.apache.org/");
controller.addSeed("http://hbase.apache.org/");
controller.addSeed("http://hive.apache.org/");
controller.addSeed("http://pig.apache.org/");
controller.addSeed("http://zookeeper.apache.org/");
controller.setPolitenessDelay(1000);
controller.setMaximumCrawlDepth(2);
controller.setMaximumPagesToFetch(1);
controller.start(MyCrawler.class, numberOfCrawlers);
}