Add "compatible" as a criteria for robot

This commit is contained in:
Gregory Soutade 2023-02-18 08:49:14 +01:00
parent a5bef4ece6
commit c8dfdd17f7

View File

@ -100,7 +100,8 @@ class IWLAPreAnalysisRobots(IPlugin):
first_page = super_hit['requests'][0]
if self.robot_re.match(first_page['http_user_agent']) or\
self.crawl_re.match(first_page['http_user_agent']):
self.crawl_re.match(first_page['http_user_agent']) or\
self.compatible_re.match(first_page['http_user_agent']):
self.logger.debug(first_page['http_user_agent'])
self._setRobot(k, super_hit)
continue