From c8dfdd17f77c1eff024701d966d20f25c119bc03 Mon Sep 17 00:00:00 2001 From: Gregory Soutade Date: Sat, 18 Feb 2023 08:49:14 +0100 Subject: [PATCH] Add "compatible" as a criteria for robot --- plugins/pre_analysis/robots.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 0a19dca..eee6ac1 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -100,7 +100,8 @@ class IWLAPreAnalysisRobots(IPlugin): first_page = super_hit['requests'][0] if self.robot_re.match(first_page['http_user_agent']) or\ - self.crawl_re.match(first_page['http_user_agent']): + self.crawl_re.match(first_page['http_user_agent']) or\ + self.compatible_re.match(first_page['http_user_agent']): self.logger.debug(first_page['http_user_agent']) self._setRobot(k, super_hit) continue