From 62be78845a7d4e1defbfa3c6ce54b91e4743cf5a Mon Sep 17 00:00:00 2001
From: Gregory Soutade <gregory@soutade.fr>
Date: Wed, 13 May 2015 18:13:18 +0200
Subject: [PATCH 1/2] Add debug traces in robots plugin

---
 plugins/pre_analysis/robots.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py
index 0c4bb96..41c744e 100644
--- a/plugins/pre_analysis/robots.py
+++ b/plugins/pre_analysis/robots.py
@@ -19,6 +19,7 @@
 #
 
 import re
+import logging
 
 from iwla import IWLA
 from iplugin import IPlugin
@@ -61,13 +62,20 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
         self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
         self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
+        self.logger = logging.getLogger(self.__class__.__name__)
         return True
 
+    def _setRobot(self, k, super_hit):
+        self.logger.debug('%s is a robot' % (k))
+        super_hit['robot'] = 1
+
 # Basic rule to detect robots
     def hook(self):
         hits = self.iwla.getCurrentVisists()
         for (k, super_hit) in hits.items():
-            if super_hit['robot']: continue
+            if super_hit['robot']:
+                self.logger.debug('%s is a robot' % (k))
+                continue
 
             isRobot = False
             referers = 0
@@ -76,7 +84,7 @@ class IWLAPreAnalysisRobots(IPlugin):
 
             if self.robot_re.match(first_page['http_user_agent']) or\
                     self.crawl_re.match(first_page['http_user_agent']):
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             for r in self.awstats_robots:
@@ -85,7 +93,7 @@ class IWLAPreAnalysisRobots(IPlugin):
                     break
 
             if isRobot:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
 # 1) no pages view --> robot
@@ -95,13 +103,13 @@ class IWLAPreAnalysisRobots(IPlugin):
 
 # 2) pages without hit --> robot
             if not super_hit['viewed_hits']:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             for hit in super_hit['requests']:
 # 3) /robots.txt read
                 if hit['extract_request']['http_uri'].endswith('/robots.txt'):
-                    isRobot = True
+                    self._setRobot(k, super_hit)
                     break
 
 # 4) Any referer for hits
@@ -109,10 +117,10 @@ class IWLAPreAnalysisRobots(IPlugin):
                     referers += 1
 
             if isRobot:
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue
 
             if not super_hit['viewed_pages'] and \
                     (super_hit['viewed_hits'] and not referers):
-                super_hit['robot'] = 1
+                self._setRobot(k, super_hit)
                 continue

From 86fc5f2189c07bf34c7d7e1a4885bf68c4609d07 Mon Sep 17 00:00:00 2001
From: Gregory Soutade <gregory@soutade.fr>
Date: Thu, 14 May 2015 09:54:25 +0200
Subject: [PATCH 2/2] Add FileIter to iwla, allowing to specify multiple files
 to analyse

---
 iwla.py | 38 +++++++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/iwla.py b/iwla.py
index 7bb08bc..fd5b2ab 100755
--- a/iwla.py
+++ b/iwla.py
@@ -720,6 +720,38 @@ class IWLA(object):
         else:
             self.logger.info('==> Analyse not started : nothing new')
 
+
+class FileIter(object):
+    def __init__(self, filenames):
+        self.filenames = [f for f in filenames.split(',') if f]
+        for f in self.filenames:
+            if not os.path.exists(f):
+                print 'No such file \'%s\'' % (f)
+                sys.exit(-1)
+        self.cur_file = None
+        self._openNextFile()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self.next()
+
+    def _openNextFile(self):
+        if self.cur_file:
+            self.cur_file.close()
+            self.cur_file = None
+        if not self.filenames:
+            raise StopIteration()
+        self.cur_file = open(self.filenames.pop(0))
+
+    def next(self):
+        l = self.cur_file.readline()
+        if not l:
+            self._openNextFile()
+            l = self.cur_file.readline()
+        return l[:-1]
+    
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Intelligent Web Log Analyzer')
 
@@ -775,8 +807,4 @@ if __name__ == '__main__':
         iwla.start(sys.stdin)
     else:
         filename = args.file or conf.analyzed_filename
-        if not os.path.exists(filename):
-            print 'No such file \'%s\'' % (filename)
-            sys.exit(-1)
-        with open(filename) as f:
-            iwla.start(f)
+        iwla.start(FileIter(filename))