From df78a3f4cbefd3ec26375a6bd4605a6f149fc729 Mon Sep 17 00:00:00 2001
From: Gregory Soutade <gregory@soutade.fr>
Date: Mon, 6 Apr 2015 17:52:31 +0200
Subject: [PATCH] [pre_analysis/robots] Don't checks for /robots.txt request,
 but endswith /robots.txt for robot detection

---
 plugins/pre_analysis/robots.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py
index f4ea99d..0c4bb96 100644
--- a/plugins/pre_analysis/robots.py
+++ b/plugins/pre_analysis/robots.py
@@ -100,7 +100,7 @@ class IWLAPreAnalysisRobots(IPlugin):
 
             for hit in super_hit['requests']:
 # 3) /robots.txt read
-                if hit['extract_request']['http_uri'] == '/robots.txt':
+                if hit['extract_request']['http_uri'].endswith('/robots.txt'):
                     isRobot = True
                     break