long startTime = System.currentTimeMillis();
ConfigurableWorker worker = new ConfigurableWorker("W001");
worker.setTaskProcessor(new PageURLMiningProcessor());
addTask2Worker(worker, new PageURLMiningTask("http://www.taobao.com"));
addTask2Worker(worker, new PageURLMiningTask("http://www.xinhuanet.com"));
addTask2Worker(worker, new PageURLMiningTask("http://www.zol.com.cn"));
addTask2Worker(worker, new PageURLMiningTask("http://www.163.com"));
LinearURLMiningMain mainListener = new LinearURLMiningMain();
worker.addListener(mainListener);
worker.start();
String targetURL = EMPTY_STRING;
while (foundURLs.size() < URL_SIZE_TO_MINE) {
targetURL = foundURLs.pollFirst();
if (targetURL == null) {
TimeUnit.MILLISECONDS.sleep(50);
continue;
}
PageURLMiningTask task = new PageURLMiningTask(targetURL);
taskID2TaskMap.putIfAbsent(worker.addTask(task), task);
TimeUnit.MILLISECONDS.sleep(100);
}