Examples of net.yacy.cora.document.MultiProtocolURI

net.yacy.cora.document.MultiProtocolURI
MultiProtocolURI provides a URL object for multiple protocols like http, https, ftp, smb and file

                                access = Access.denied;
                            }
                        }
                        if (this.service.getProtocol() == Protocol.smb) {
                            try {
                                MultiProtocolURI uri = new MultiProtocolURI(this.service.toString());
                                String[] list = uri.list();
                                access = list == null || list.length == 0 ? Access.empty : Access.granted;
                            } catch (IOException e) {
                                access = Access.denied;
                            }
                        }

View Full Code Here

            final int startRecord,
            final int maximumRecords,
            final CacheStrategy cacheStrategy,
            final boolean global,
            final String userAgent) throws IOException {
        MultiProtocolURI uri = null;
        try {
            uri = new MultiProtocolURI(rssSearchServiceURL);
        } catch (final MalformedURLException e) {
            throw new IOException("cora.Search failed asking peer '" + rssSearchServiceURL + "': bad url, " + e.getMessage());
        }


        // send request
        byte[] result = new byte[0];
        try {
            final LinkedHashMap<String,ContentBody> parts = new LinkedHashMap<String,ContentBody>();
            parts.put("query", UTF8.StringBody(query));
            parts.put("startRecord", UTF8.StringBody(Integer.toString(startRecord)));
            parts.put("maximumRecords", UTF8.StringBody(Long.toString(maximumRecords)));
            parts.put("verify", cacheStrategy == null ? UTF8.StringBody("false") : UTF8.StringBody(cacheStrategy.toName()));
            parts.put("resource", UTF8.StringBody(global ? "global" : "local"));
            parts.put("nav", UTF8.StringBody("none"));
            // result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts);
            final HTTPClient httpClient = new HTTPClient(userAgent == null ? ClientIdentification.getUserAgent() : userAgent, (int) timeout);
            result = httpClient.POSTbytes(new MultiProtocolURI(rssSearchServiceURL), uri.getHost(), parts, false);


            final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
            if (reader == null) {
                throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (1), reader == null");
            }
            final RSSFeed feed = reader.getFeed();
            if (feed == null) {
                // case where the rss reader does not understand the content
                throw new IOException("cora.Search failed asking peer '" + uri.getHost() + "': probably bad response from remote peer (2)");
            }
            return feed;
        } catch (final IOException e) {
            throw new IOException("cora.Search error asking peer '" + uri.getHost() + "':" + e.toString());
        }
    }

View Full Code Here

    
    public MultiProtocolURI referer() {
        String referer = get(REFERER, null);
        if (referer == null) return null;
        try {
            return new MultiProtocolURI(referer);
        } catch (MalformedURLException e) {
            return null;
        }
    }

View Full Code Here

            return null;
        }
    }
    
    public String refererHost() {
        final MultiProtocolURI url = referer();
        if (url == null) return null;
        return url.getHost();
    }

View Full Code Here


      final Pattern p = Pattern.compile("(href=\"|src=\")([^\"]+)|(href='|src=')([^']+)|(url\\(')([^']+)|(url\\(\")([^\"]+)|(url\\()([^\\)]+)");
      final Matcher m = p.matcher(sbuffer);
      final StringBuffer result = new StringBuffer(80);
      String init, url;
      MultiProtocolURI target;
      while (m.find()) {
        init = null;
        if(m.group(1) != null) init = m.group(1);
        if(m.group(3) != null) init = m.group(3);
        if(m.group(5) != null) init = m.group(5);
        if(m.group(7) != null) init = m.group(7);
        if(m.group(9) != null) init = m.group(9);
        url = null;
        if(m.group(2) != null) url = m.group(2);
        if(m.group(4) != null) url = m.group(4);
        if(m.group(6) != null) url = m.group(6);
        if(m.group(8) != null) url = m.group(8);
        if(m.group(10) != null) url = m.group(10);
        if (url.startsWith("data:") || url.startsWith("#") || url.startsWith("mailto:") || url.startsWith("javascript:")) {
          String newurl = init + url;
          newurl = newurl.replaceAll("\\$","\\\\\\$");
            m.appendReplacement(result, newurl);


        } else if (url.startsWith("http")) {
          // absoulte url of form href="http://domain.com/path"
          if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) {
            if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(url)) != null) {
              continue;
            }
          }


          String newurl = init + "/proxy.html?url=" + url;
          newurl = newurl.replaceAll("\\$","\\\\\\$");
            m.appendReplacement(result, newurl);


        } else if (url.startsWith("//")) {
          // absoulte url but same protocol of form href="//domain.com/path"
          final String complete_url = proxyurl.getProtocol() + ":" +  url;
          if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) {
            if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(complete_url)) != null) {
              continue;
            }
          }


          String newurl = init + "/proxy.html?url=" + complete_url;
          newurl = newurl.replaceAll("\\$","\\\\\\$");
            m.appendReplacement(result, newurl);


        } else if (url.startsWith("/")) {
          // absolute path of form href="/absolute/path/to/linked/page"
          String newurl = init + "/proxy.html?url=http://" + host + url;
          newurl = newurl.replaceAll("\\$","\\\\\\$");
            m.appendReplacement(result, newurl);


        } else {
          // relative path of form href="relative/path"
          try {
            target = new MultiProtocolURI("http://" + host + directory + "/" + url);
            String newurl = init + "/proxy.html?url=" + target.toString();
            newurl = newurl.replaceAll("\\$","\\\\\\$");
            m.appendReplacement(result, newurl);
          }
          catch (final MalformedURLException e) {}

View Full Code Here

                    }
                  prop.put("robots-allowed", robotsEntry == null ? 1 : robotsEntry.isDisallowed(theURL) ? 0 : 1);
                    prop.putHTML("robotsInfo", robotsEntry.getInfo());


                    // get the sitemap URL of the domain
                    final MultiProtocolURI sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap();
                    prop.putXML("sitemap", sitemapURL == null ? "" : sitemapURL.toString());
                } catch (final MalformedURLException e) {
                    Log.logException(e);
                }
            }
            if (actions.indexOf("oai",0) >= 0) {

View Full Code Here

            prop.put("imgurl", "1");
            int c = 0;
            final int yOffset = embed ? 0 : 70;
            for (int i = 0; i < fifoSize; i++) {


                final MultiProtocolURI baseURL = origins[i].baseURL;
                final MultiProtocolURI imageURL = origins[i].imageEntry.url();


                // check if this loads a page from localhost, which must be prevented to protect the server
                // against attacks to the administration interface when localhost access is granted
                if ((Domains.isLocal(baseURL.getHost(), null) || Domains.isLocal(imageURL.getHost(), null)) &&
                    sb.getConfigBool("adminAccountForLocalhost", false)) continue;


                final long z = imgZIndex[i];
                prop.put("imgurl_list_" + c + "_url",
                       "<a href=\"" + baseURL.toNormalform(true, false) + "\">"
                       + "<img src=\"" + imageURL.toNormalform(true, false) + "\" "
                       + "style=\""
                       + ((imgWidth[i] == 0 || imgHeight[i] == 0) ? "" : "width:" + imgWidth[i] + "px;height:" + imgHeight[i] + "px;")
                       + "position:absolute;top:" + (imgPosY[i] + yOffset)
                       + "px;left:" + imgPosX[i]
                       + "px;z-index:" + z + "\" "

View Full Code Here

public class cytag {
    
    public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
        
        final Switchboard sb = (Switchboard)env;
        final MultiProtocolURI referer = header.referer();
        
        // harvest request information
        StringBuilder connect = new StringBuilder();
        connect.append('{');
        appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format());
        appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false, false));
        appendJSON(connect, "nick",  (post == null) ? "" : post.get("nick", ""));
        appendJSON(connect, "tag",   (post == null) ? "" : post.get("tag", ""));
        appendJSON(connect, "icon",  (post == null) ? "" : post.get("icon", ""));
        appendJSON(connect, "ip",    header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, ""));
        appendJSON(connect, "agent", header.get("User-Agent", ""));

View Full Code Here

        // return variable that accumulates replacements
        final serverObjects prop = new serverObjects();
        final Switchboard sb = (Switchboard) env;


        // get referer for backlink
        final MultiProtocolURI referer = header.referer();
        prop.put("referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true, true));
        //if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString());


        if (post == null) {
            prop.put("info", "1");//no information submitted
            return prop;

View Full Code Here




    public static void main(final String[] args) {
        final File image = new File(args[0]);
        final genericImageParser parser = new genericImageParser();
        MultiProtocolURI uri;
        try {
            uri = new MultiProtocolURI("http://localhost/" + image.getName());
            final Document[] document = parser.parse(uri, "image/" + uri.getFileExtension(), "UTF-8", new FileInputStream(image));
            System.out.println(document[0].toString());
        } catch (final MalformedURLException e) {
            e.printStackTrace();
        } catch (final FileNotFoundException e) {
            e.printStackTrace();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of net.yacy.cora.document.MultiProtocolURI

Collage

cytag

de.anomic.crawler.RobotsTxt

de.anomic.crawler.RobotsTxtEntry

de.anomic.data.BookmarkHelper

de.anomic.http.server.HTTPDFileHandler

de.anomic.search.QueryParams

de.anomic.search.Switchboard

de.anomic.yacy.graphics.WebStructureGraph

de.anomic.yacy.yacyClient

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.