diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 94a3adf..d4a0d6c 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -64,6 +64,7 @@ class IWLAPreAnalysisRobots(IPlugin): self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE) self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE) self.logger = logging.getLogger(self.__class__.__name__) + return True def _setRobot(self, k, super_hit): @@ -83,6 +84,10 @@ class IWLAPreAnalysisRobots(IPlugin): self.logger.debug('%s is a robot' % (k)) continue + if super_hit.get('feed_parser', False): + self.logger.debug('%s is feed parser' % (k)) + continue + isRobot = False referers = 0