.build();
DefaultCrawler crawler =
new DefaultCrawler(new ResponseFetcher(processor), Executors.newFixedThreadPool(10),
new CompositeURLParser(new FramePageURLParser(), new AhrefPageURLParser()));
crawler.getUrls(apiConfig);
crawler.shutdown();
}
private class ResponseFetcher extends HTTPClientResponseFetcher {
private final DocumentProcessor processor;