public boolean insert(LinkRelevance linkRelevance) {
        Context timerContext = insertTimer.time();
        try {
            if (linkRelevance == null) {
                return false;
            }
            boolean insert = isRelevant(linkRelevance);
            if (insert) {
                if (downloadRobots) {
                    URL url = linkRelevance.getURL();
                    String hostName = url.getHost();
                    if (!hostsManager.isKnown(hostName)) {
                        hostsManager.insert(hostName);
                        try {
                            URL robotsUrl = new URL(url.getProtocol(), url.getHost(), url.getPort(), "/robots.txt");
                            LinkRelevance sitemap = LinkRelevance.createRobots(robotsUrl.toString(), 299);
                            frontier.insert(sitemap);
                        } catch (Exception e) {
                            logger.warn("Failed to insert robots.txt for host: " + hostName, e);
                        }
                    }
                }
                insert = frontier.insert(linkRelevance);
                scheduler.notifyLinkInserted();
            }
            return insert;
        } finally {
            timerContext.stop();
        }
    }
This class manages the crawler frontier

@author Luciano Barbosa
@version 1.0
ache/src/main/java/achecrawler/link/frontier/FrontierManager.java