Package net.yacy.cora.protocol

Examples of net.yacy.cora.protocol.RequestHeader

885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
        return log;
    }

    private Iterator<String> downloadSeedFile(final DigestURI seedURL) throws IOException {
        // Configure http headers
        final RequestHeader reqHeader = new RequestHeader();
        reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
        reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary?
        reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());

        final HTTPClient client = new HTTPClient();
        client.setHeader(reqHeader.entrySet());
        byte[] content = null;
        try {
            // send request
          content = client.GETbytes(seedURL);
        } catch (final Exception e) {
View Full Code Here
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
        Iterator<String>   enu;
        int                lc;
        final int          sc = this.peers.sizeConnected();
        ResponseHeader header;

        final RequestHeader reqHeader = new RequestHeader();
        reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
        reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache");
        reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());
        final HTTPClient client = new HTTPClient();
        client.setHeader(reqHeader.entrySet());
        client.setTimout((int) getConfigLong("bootstrapLoadTimeout", 20000));

        Network.log.logInfo("BOOTSTRAP: " + sc + " seeds known from previous run");

        // - use the superseed to further fill up the seedDB
View Full Code Here
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
    /**
     * @param url
     * @return
     */
    public static Map<String, String> loadFileAsMap(final DigestURI url) {
        final RequestHeader reqHeader = new RequestHeader();
        reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());
        final HTTPClient client = new HTTPClient();
        client.setHeader(reqHeader.entrySet());
        try {
            // sending request
            final Map<String, String> result = FileUtils.table(client.GETbytes(url));
            return (result == null) ? new HashMap<String, String>() : result;
        } catch (final Exception e) {
View Full Code Here
TOP

Related Classes of net.yacy.cora.protocol.RequestHeader

  • de.anomic.crawler.retrieval.FileLoader
  • de.anomic.crawler.retrieval.FTPLoader
  • de.anomic.crawler.retrieval.HTTPLoader
  • de.anomic.crawler.retrieval.Response
  • de.anomic.crawler.retrieval.SMBLoader
  • de.anomic.crawler.RobotsTxt
  • de.anomic.http.server.HTTPDemon
  • de.anomic.http.server.ServerSideIncludes
  • de.anomic.search.Switchboard
  • de.anomic.server.serverSwitch
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.