Examples of DomProfile


Examples of de.anomic.crawler.CrawlProfile.DomProfile

        }

        // deny urls that exceed allowed number of occurrences
        final int maxAllowedPagesPerDomain = profile.domMaxPages();
        if (maxAllowedPagesPerDomain < Integer.MAX_VALUE) {
            final DomProfile dp = profile.getDom(url.getHost());
            if (dp != null && dp.count >= maxAllowedPagesPerDomain) {
                if (this.log.isFine()) this.log.logFine("URL '" + urlstring + "' appeared too often in crawl stack, a maximum of " + profile.domMaxPages() + " is allowed.");
                return "crawl stack domain counter exceeded";
            }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.