soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Don't launch robot analysis rules for feed parsers

Browse Source
This commit is contained in:
Gregory Soutade 2022-11-16 21:10:11 +01:00
parent f8b37a625c
commit 975cc66bd5
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
plugins/pre_analysis/robots.py
View File

@ -64,6 +64,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE) self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE) self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
self.logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
return True return True
def _setRobot(self, k, super_hit): def _setRobot(self, k, super_hit):
@ -83,6 +84,10 @@ class IWLAPreAnalysisRobots(IPlugin):
self.logger.debug('%s is a robot' % (k)) self.logger.debug('%s is a robot' % (k))
continue continue
if super_hit.get('feed_parser', False):
self.logger.debug('%s is feed parser' % (k))
continue
isRobot = False isRobot = False
referers = 0 referers = 0