diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index f4ea99d..0c4bb96 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -100,7 +100,7 @@ class IWLAPreAnalysisRobots(IPlugin): for hit in super_hit['requests']: # 3) /robots.txt read - if hit['extract_request']['http_uri'] == '/robots.txt': + if hit['extract_request']['http_uri'].endswith('/robots.txt'): isRobot = True break