[pre_analysis/robots] Don't checks for /robots.txt request, but endswith /robots.txt for robot detection

2015-04-06 17:52:31 +02:00
parent 2ed7551049
commit df78a3f4cb
1 changed files with 1 additions and 1 deletions
--- a/plugins/pre_analysis/robots.py
+++ b/plugins/pre_analysis/robots.py
@@ -100,7 +100,7 @@ class IWLAPreAnalysisRobots(IPlugin):

            for hit in super_hit['requests']:
 # 3) /robots.txt read
-                if hit['extract_request']['http_uri'] == '/robots.txt':
+                if hit['extract_request']['http_uri'].endswith('/robots.txt'):
                    isRobot = True
                    break