Package org.vietspider.html

Examples of org.vietspider.html.NodeIterator

36
37
38
39
40
41
42
43
44
45
46
47
48
    return node;
  }

  public StringBuilder getTextContent(HTMLNode node){
    StringBuilder value = new StringBuilder();
    NodeIterator iterator = node.iterator();
    while(iterator.hasNext()) {
      HTMLNode n = iterator.next();
      if(n.isNode(Name.CONTENT)) {
        value.append(n.getValue());   
      }   
    }
    /*if(node.getConfig().name() == Name.CONTENT){
View Full Code Here
15
16
17
18
19
20
21
22
23
24
25
26
27
  public static void main(String[] args) throws Exception {
    URL url = new URL("http://java.sun.com/");
    HTMLDocument document = new HTMLParser2().createDocument(url.openStream(), "utf-8");
   
    NodeIterator iterator =  document.getRoot().iterator();
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(node.isNode(Name.SCRIPT)) {
        if(node.hasChildren() && node.getChildren().size() > 0) {
          System.out.println("===================================================");
          System.out.println(node.getChild(0).getTextValue());
        }
View Full Code Here
37
38
39
40
41
42
43
44
45
46
47
48
49
50
    }
  }
 
  private void searchDataNode(HTMLNode root) {*/
    NodePathParser pathParser = new NodePathParser();
    NodeIterator iterator = root.iterator();
  
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(!node.isNode(Name.DIV)) continue;
      Attributes attributes = node.getAttributes();
      Attribute attribute = attributes.get("id");
      if(attribute == null) continue;
      String value = attribute.getValue();
View Full Code Here
128
129
130
131
132
133
134
135
136
137
138
139
140
  }
 
  private HTMLNode searchPageNode(HTMLNode root) {
    HTMLNode parent = root.getParent();
    if(parent == null) return null;
    NodeIterator iterator = parent.iterator();
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(!node.isNode(Name.CONTENT)) continue;
      String content  = node.getTextValue().toLowerCase();
      if(content.indexOf("page") < 0
          && content.indexOf("trang") < 0) continue;
      HTMLNode table = upParent(node, Name.TABLE);
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
154
155
    }
    return null;
  }
 
  private HTMLNode searchUserNode(HTMLNode root) {
    NodeIterator iterator = root.iterator();
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(!node.isNode(Name.A)) continue;
      Attributes attributes = node.getAttributes();
      Attribute attribute = attributes.get("class");
      if(attribute == null) continue;
      String value = attribute.getValue();
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
170
171
    }
    return null;
  }
 
  private HTMLNode searchContentNode(HTMLNode root, String clazz) {
    NodeIterator iterator = root.iterator();
    while(iterator.hasNext()) {
      HTMLNode node = iterator.next();
      if(!node.isNode(Name.DIV)) continue;
      Attributes attributes = node.getAttributes();
      Attribute attribute = attributes.get("id");
      if(attribute == null) continue;
      String value = attribute.getValue();
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
188
189
    }
    return null;
  }
 
  private boolean isPageList(HTMLNode node) {
    NodeIterator iterator = node.iterator();
    while(iterator.hasNext()) {
      HTMLNode n = iterator.next();
      if(!n.isNode(Name.A)) continue;
      Attributes attributes = n.getAttributes();
      Attribute attribute = attributes.get("href");
      if(attribute == null) continue;
      String value = attribute.getValue();
View Full Code Here
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
    }
    return false;
  }
 
  private HTMLNode searchPageNode2(HTMLNode node) {
    NodeIterator iterator = node.iterator();
    HTMLNode table = null;
    while(iterator.hasNext()) {
      HTMLNode n = iterator.next();
      if(!n.isNode(Name.A)) continue;
      List<HTMLNode> children = n.getChildren();
      if(children == null
          || children.size() != 1
          || !children.get(0).isNode(Name.CONTENT)) continue;
      String text = children.get(0).getTextValue();
      try {
        Integer.parseInt(text.trim());
        table = upParent(n, Name.TABLE);
        break;
      } catch (Exception e) {
      }
    }
    if(table == null) return null;
    NodeHandler nodeHandler = new NodeHandler();
    iterator = node.iterator();
    while(iterator.hasNext()) {
      HTMLNode n = iterator.next();
      if(!n.isNode(Name.CONTENT)) continue;
      String text = n.getTextValue().toLowerCase().trim();
      if(text.startsWith("trang") || text.startsWith("page")) {
        if(nodeHandler.count(text) < 5) return table;
      }
View Full Code Here
248
249
250
251
252
253
254
255
256
257
258
259
260
      return null;
    }
   
    HTMLNode parent = node.getParent();
    if(parent == null) return null;
    NodeIterator iterator = parent.iterator();
    while(iterator.hasNext()) {
      HTMLNode n = iterator.next();
      if(!n.isNode(Name.CONTENT)) continue;
      if(n.getParent().isNode(Name.A)) continue;
      String content = n.getTextValue();
      if(indexOf(title, content)) return upParent(n, Name.TD, Name.DIV, Name.STRONG);
    }
View Full Code Here
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
  public void remove(HTMLNode root, HTMLNode first, HTMLNode last) {
    boolean remove = true;
   
    List<HTMLNode> removes = new ArrayList<HTMLNode>();
    NodeIterator nodeIterator = root.iterator();
//    System.out.println(new String(first.getValue()));
    while(nodeIterator.hasNext()) {
      HTMLNode node = nodeIterator.next();
      switch (node.getName()) {
      case UL:
        if(isLinkContainer(node)) removes.add(node);
        break;
      case DIV:
View Full Code Here
TOP

Related Classes of org.vietspider.html.NodeIterator

  • AttributeExplorer
  • JavascriptExtractor
  • org.vietspider.html.renderer.checker.ContentChecker
  • org.vietspider.html.renderer.checker.FormChecker
  • org.vietspider.html.renderer.checker.FormNodeRemover
  • org.vietspider.html.renderer.checker.LinkBlockChecker
  • org.vietspider.html.renderer.checker.ULNodeChecker
  • org.vietspider.html.renderer.ContentRegionSearcher
  • org.vietspider.html.renderer.ContentRegionSearcher2
  • org.vietspider.html.renderer.ContentRendererBak
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.