Add debug traces in robots plugin

plugins/pre_analysis/robots.py

@@ -19,6 +19,7 @@
 #
 
 import re
+import logging
 
 from iwla import IWLA
 from iplugin import IPlugin
@@ -61,13 +62,20 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
         self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
         self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
+        self.logger = logging.getLogger(self.__class__.__name__)
         return True
 
+    def _setRobot(self, k, super_hit):
+        self.logger.debug('%s is a robot' % (k))
+        super_hit['robot'] = 1
+
 # Basic rule to detect robots
     def hook(self):
         hits = self.iwla.getCurrentVisists()
         for (k, super_hit) in hits.items():
-            if super_hit['robot']: continue
+            if super_hit['robot']:
+                self.logger.debug('%s is a robot' % (k))
+                continue
 
             isRobot = False
             referers = 0
@@ -76,7 +84,7 @@ class IWLAPreAnalysisRobots(IPlugin):
 
             if self.robot_re.match(first_page['http_user_agent']) or\
                     self.crawl_re.match(first_page['http_user_agent']):
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             for r in self.awstats_robots:
@@ -85,7 +93,7 @@ class IWLAPreAnalysisRobots(IPlugin):
                     break
 
             if isRobot:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
 # 1) no pages view --> robot
@@ -95,13 +103,13 @@ class IWLAPreAnalysisRobots(IPlugin):
 
 # 2) pages without hit --> robot
             if not super_hit['viewed_hits']:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             for hit in super_hit['requests']:
 # 3) /robots.txt read
                 if hit['extract_request']['http_uri'].endswith('/robots.txt'):
-                    isRobot = True
+                    self._setRobot(k, super_hit)
                     break
 
 # 4) Any referer for hits
@@ -109,10 +117,10 @@ class IWLAPreAnalysisRobots(IPlugin):
                     referers += 1
 
             if isRobot:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             if not super_hit['viewed_pages'] and \
                     (super_hit['viewed_hits'] and not referers):
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue