diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 0a19dca..eee6ac1 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -100,7 +100,8 @@ class IWLAPreAnalysisRobots(IPlugin): first_page = super_hit['requests'][0] if self.robot_re.match(first_page['http_user_agent']) or\ - self.crawl_re.match(first_page['http_user_agent']): + self.crawl_re.match(first_page['http_user_agent']) or\ + self.compatible_re.match(first_page['http_user_agent']): self.logger.debug(first_page['http_user_agent']) self._setRobot(k, super_hit) continue