try {
BaseRobotRules rules = RobotUtils.getRobotRules(fetcher, parser, new URL("http://localhost:8089/robots.txt"));
Assert.assertTrue(rules.isAllowAll());
} finally {
server.stop();
}
}
@Test
public void testMatchAgainstEmailAddress() throws Exception {