Package com.zhangwoo.spider.po

Examples of com.zhangwoo.spider.po.UrlState

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
  }

  @Override
  public void messageReceived(IoSession session, Object message)
      throws Exception {
    UrlState state=(UrlState) message;
    int index=TaskCenter.urlState.indexOf(state);
    if(index!=-1){
      UrlState oldState=TaskCenter.urlState.remove(index);
      state.setBeginTime(oldState.getBeginTime());
      state.setEndTime(DateUtil.formatDateTime());
      state.setRuninfo(((InetSocketAddress) session.getRemoteAddress()).getAddress().getHostAddress());
      TaskCenter.taskState.get(state.getUrlReq().getTask()).add(state);
      TaskCenter.taskCount.put(state.getUrlReq().getTask(),
          TaskCenter.taskCount.get(state.getUrlReq().getTask())-1);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    if (TaskCenter.taskQueue.contains(urlReq) // 待运行URL队列包含该URL
        || TaskCenter.urlState.contains(urlReq) // 正在运行的URL队列中包含该URL
        || urlReq.getTask() == null // 不归属任何任务
        || TaskCenter.taskState.get(urlReq.getTask()) == null
        || TaskCenter.taskState.get(urlReq.getTask()).contains(
            new UrlState(urlReq)) // 已运行的任务URL队列包含此URL
    ) {
      return false; // 以上情况均放弃该URL
    }
    TaskCenter.taskQueue.add(urlReq);
    TaskCenter.taskCount.put(urlReq.getTask(),
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
   */
  public static UrlRequest getUrl() {
    // logger.debug("now need poll , size "+TaskCenter.taskQueue.size());
    UrlRequest urlReq = TaskCenter.taskQueue.poll();
    if (urlReq!=null) {
      UrlState state = new UrlState(urlReq);
      state.setBeginTime(DateUtil.formatDateTime());
      urlState.add(state); // 正在运行的任务
    }
    return urlReq;
  }
View Full Code Here
176
177
178
179
180
181
182
183
    }
    if ((urlReqResult == null || urlReqResult.size() == 0) && (convsResult == null || convsResult.size() == 0)) {
      logger.info("url : " + urlReq.getUrl() + " get no reuslt!  ");
    }
    ClientMessageCenter.getInstance().stateSend(
        new UrlState(html.length(), urlReqResult.size(), convsResult
            .size(), urlReq));
  }
View Full Code Here
TOP

Related Classes of com.zhangwoo.spider.po.UrlState

  • com.zhangwoo.spider.client.process.SpiderThread
  • com.zhangwoo.spider.server.TaskCenter
  • com.zhangwoo.spider.server.TaskStateHandler
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.