controller.addSeed("http://hadoop.apache.org/mapreduce/");
controller.addSeed("http://avro.apache.org/");
controller.addSeed("http://hbase.apache.org/");
controller.addSeed("http://hive.apache.org/");
controller.addSeed("http://pig.apache.org/");
controller.addSeed("http://zookeeper.apache.org/");
controller.setPolitenessDelay(1000);
controller.setMaximumCrawlDepth(2);
controller.setMaximumPagesToFetch(1);
controller.start(MyCrawler.class, numberOfCrawlers);