Initial commit

plugins/pre_analysis/__init__.py

@@ -0,0 +1 @@
+#

plugins/pre_analysis/page_to_hit.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Pre analysis hook
+Change page into hit and hit into page into statistics
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    page_to_hit_conf*
+    hit_to_page_conf*
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+visits :
+   remote_addr =>
+       is_page
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPreAnalysisPageToHit(IPlugin):
+
+    def __init__(self, iwla):
+        super(IWLAPreAnalysisPageToHit, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        # Page to hit
+        self.ph_regexps = self.iwla.getConfValue('page_to_hit_conf', [])
+        if not self.ph_regexps: return False
+        self.ph_regexps = map(lambda(r): re.compile(r), self.ph_regexps)
+
+        # Hit to page
+        self.hp_regexps = self.iwla.getConfValue('hit_to_page_conf', [])
+        if not self.hp_regexps: return False
+        self.hp_regexps = map(lambda(r): re.compile(r), self.hp_regexps)
+
+        return True
+
+    def hook(self):
+        hits = self.iwla.getCurrentVisists()
+
+        for (k, super_hit) in hits.items():
+            if super_hit['robot']: continue
+
+            for request in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(request):
+                    break
+
+                if not self.iwla.hasBeenViewed(request):
+                    continue
+
+                uri = request['extract_request']['extract_uri']
+
+                if request['is_page']:
+                    # Page to hit
+                    for regexp in self.ph_regexps:
+                        if regexp.match(uri):
+                            #print '%s is a hit' % (uri )
+                            request['is_page'] = False
+                            super_hit['viewed_pages'] -= 1
+                            super_hit['viewed_hits'] += 1
+                            break
+                else:
+                    # Hit to page
+                    for regexp in self.hp_regexps:
+                        if regexp.match(uri):
+                            #print '%s is a page' % (uri )
+                            request['is_page'] = True
+                            super_hit['viewed_pages'] += 1
+                            super_hit['viewed_hits'] -= 1
+                            break

plugins/pre_analysis/robots.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+import awstats_data
+
+"""
+Pre analysis hook
+
+Filter robots
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    page_to_hit_conf*
+    hit_to_page_conf*
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+visits :
+   remote_addr =>
+       robot
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPreAnalysisRobots(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPreAnalysisRobots, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
+    
+        return True
+
+# Basic rule to detect robots
+    def hook(self):
+        hits = self.iwla.getCurrentVisists()
+        for (k, super_hit) in hits.items():
+            if super_hit['robot']: continue
+
+            isRobot = False
+            referers = 0
+
+            first_page = super_hit['requests'][0]
+            if not self.iwla.isValidForCurrentAnalysis(first_page): continue
+
+            for r in self.awstats_robots:
+                if r.match(first_page['http_user_agent']):
+                    isRobot = True
+                    break
+
+            if isRobot:
+                super_hit['robot'] = 1
+                continue
+
+# 1) no pages view --> robot
+            # if not super_hit['viewed_pages']:
+            #     super_hit['robot'] = 1
+            #     continue
+
+# 2) pages without hit --> robot
+            if not super_hit['viewed_hits']:
+                super_hit['robot'] = 1
+                continue
+
+            for hit in super_hit['requests']:
+# 3) /robots.txt read
+                if hit['extract_request']['http_uri'] == '/robots.txt':
+                    isRobot = True
+                    break
+
+# 4) Any referer for hits
+                if not hit['is_page'] and hit['http_referer']:
+                    referers += 1
+
+            if isRobot:
+                super_hit['robot'] = 1
+                continue
+
+            if not super_hit['viewed_pages'] and \
+                    (super_hit['viewed_hits'] and not referers):
+                super_hit['robot'] = 1
+                continue