Initial commit

plugins/__init__.py

@@ -0,0 +1,2 @@
+
+__all__ = ['pre_analysis', 'post_analysis', 'display']

plugins/display/__init__.py

@@ -0,0 +1 @@
+#

plugins/display/all_visits.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create All visits page
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    display_visitor_ip*
+
+Output files :
+    OUTPUT_ROOT/year/month/all_visits.html
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayAllVisits(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayAllVisits, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        hits = self.iwla.getValidVisitors()
+        display_visitor_ip = self.iwla.getConfValue('display_visitor_ip', False)
+
+        last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
+
+        title = createCurTitle(self.iwla, u'All visits')
+
+        filename = 'all_visits.html'
+        path = self.iwla.getCurDisplayPath(filename)
+
+        page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+        table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Last seen'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Bandwidth'), self.iwla._(u'Last seen')])
+        table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
+        
+        for super_hit in last_access:
+            address = super_hit['remote_addr']
+            if display_visitor_ip and\
+                    super_hit.get('dns_name_replaced', False):
+                address = '%s [%s]' % (address, super_hit['remote_ip'])
+
+            row = [
+                address,
+                super_hit['viewed_pages'],
+                super_hit['viewed_hits'],
+                bytesToStr(super_hit['bandwidth']),
+                time.asctime(super_hit['last_access'])
+                ]
+            table.appendRow(row)
+        page.appendBlock(table)
+
+        display.addPage(page)
+
+        index = self.iwla.getDisplayIndex()
+        link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All visits'))
+        block = index.getBlock(self.iwla._(u'Top visitors'))
+        if block:
+            block.setTitle('%s - %s' % (block.getTitle(), link))
+        else:
+            block = display.createBlock(DisplayHTMLRawBlock)
+            block.setRawHTML(link)
+            index.appendBlock(block)

plugins/display/istats_diff.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+import logging
+
+"""
+Display hook itnerface
+
+Enlight new and updated statistics
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayStatsDiff(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayStatsDiff, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.month_stats_key = None
+        # Set >= if month_stats[self.month_stats_key] is a list or a tuple
+        self.stats_index = -1
+        self.filename = None
+        self.block_name = None
+        self.logger = logging.getLogger(__name__)
+
+    def load(self):
+        if not self.month_stats_key or not self.filename or\
+                not self.block_name:
+            self.logger('Bad parametrization')
+            return False
+        month_stats = self.iwla.getMonthStats()
+        self.cur_stats = {k:v for (k,v) in month_stats.get(self.month_stats_key, {}).items()}
+        return True
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        month_stats = self.iwla.getMonthStats()
+
+        path = self.iwla.getCurDisplayPath(self.filename)
+        page = display.getPage(path)
+        if not page: return
+        title = self.iwla._(self.block_name)
+        block = page.getBlock(title)
+        if not block:
+            self.logger.error('Block %s not found' % (title))
+            return
+
+        stats_diff = {}
+        for (k, v) in month_stats[self.month_stats_key].items():
+            new_value = self.cur_stats.get(k, 0)
+            if new_value:
+                if self.stats_index != -1:
+                    if new_value[self.stats_index] != v[self.stats_index]:
+                        stats_diff[k] = 'iwla_update'
+                else:
+                    if new_value != v:
+                        stats_diff[k] = 'iwla_update'
+            else:
+                stats_diff[k] = 'iwla_new'
+
+        for (idx, row) in enumerate(block.rows):
+            if row[0] in stats_diff.keys():
+                block.setCellCSSClass(idx, 0, stats_diff[row[0]])

plugins/display/referers.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create Referers page
+
+Plugin requirements :
+    post_analysis/referers
+
+Conf values needed :
+    max_referers_displayed*
+    create_all_referers_page*
+    max_key_phrases_displayed*
+    create_all_key_phrases_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/referers.html
+    OUTPUT_ROOT/year/month/key_phrases.html
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayReferers(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayReferers, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisReferers']
+        self.max_referers = self.iwla.getConfValue('max_referers_displayed', 0)
+        self.create_all_referers = self.iwla.getConfValue('create_all_referers_page', True)
+        self.max_key_phrases = self.iwla.getConfValue('max_key_phrases_displayed', 0)
+        self.create_all_key_phrases = self.iwla.getConfValue('create_all_key_phrases_page', True)
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        month_stats = self.iwla.getMonthStats()
+        referers = month_stats.get('referers', {})
+        robots_referers = month_stats.get('robots_referers', {})
+        search_engine_referers = month_stats.get('search_engine_referers', {})
+        key_phrases = month_stats.get('key_phrases', {})
+
+        top_referers = [(k, referers[k]['pages']) for k in referers.keys()]
+        top_referers = sorted(top_referers, key=lambda t: t[1], reverse=True)
+
+        top_robots_referers = [(k, robots_referers[k]['pages']) for k in robots_referers.keys()]
+        top_robots_referers = sorted(top_robots_referers, key=lambda t: t[1], reverse=True)
+
+        top_search_engine_referers = [(k, search_engine_referers[k]['pages']) for k in search_engine_referers.keys()]
+        top_search_engine_referers = sorted(top_search_engine_referers, key=lambda t: t[1], reverse=True)
+
+        top_key_phrases = key_phrases.items()
+        top_key_phrases = sorted(top_key_phrases, key=lambda t: t[1], reverse=True)
+
+        cur_time = self.iwla.getCurTime()
+        index = self.iwla.getDisplayIndex()
+
+        # All referers in a file
+        if self.create_all_referers:
+            title = createCurTitle(self.iwla, u'Connexion from')
+
+            filename = 'referers.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Connexion from'), [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
+            table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
+
+            total_search = [0]*3
+            table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
+            new_list = self.max_referers and top_search_engine_referers[:self.max_referers] or top_search_engine_referers
+            for r,_ in new_list:
+                row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
+                total_search[1] += search_engine_referers[r]['pages']
+                total_search[2] += search_engine_referers[r]['hits']
+                table.appendRow(row)
+            if self.max_referers:
+                others = 0
+                for (uri, entrance) in top_referers[self.max_referers:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+            total_external = [0]*3
+            table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
+            new_list = self.max_referers and top_referers[:self.max_referers] or top_referers
+            for r,_ in new_list:
+                row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
+                total_external[1] += referers[r]['pages']
+                total_external[2] += referers[r]['hits']
+                table.appendRow(row)
+            if self.max_referers:
+                others = 0
+                for (uri, entrance) in top_referers[self.max_referers:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+            total_robot = [0]*3
+            table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
+            new_list = self.max_referers and top_robots_referers[:self.max_referers] or top_robots_referers
+            for r,_ in new_list:
+                row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
+                total_robot[1] += robots_referers[r]['pages']
+                total_robot[2] += robots_referers[r]['hits']
+                table.appendRow(row)
+            if self.max_referers:
+                others = 0
+                for (uri, entrance) in top_referers[self.max_referers:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+            page.appendBlock(table)
+
+            display.addPage(page)
+
+        title = self.iwla._(u'Top Referers')
+        if self.create_all_referers:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Referers'))
+            title = '%s - %s' % (title, link)
+
+        # Top referers in index
+        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
+        table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
+
+        table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
+        for r,_ in top_search_engine_referers[:10]:
+            row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
+            total_search[1] -= search_engine_referers[r]['pages']
+            total_search[2] -= search_engine_referers[r]['hits']
+            table.appendRow(row)
+        if total_search[1] or total_search[2]:
+            total_search[0] = self.iwla._(u'Others')
+            table.appendRow(total_search)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+        table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
+        for r,_ in top_referers[:10]:
+            row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
+            total_external[1] -= referers[r]['pages']
+            total_external[2] -= referers[r]['hits']            
+            table.appendRow(row)
+        if total_external[1] or total_external[2]:
+            total_external[0] = self.iwla._(u'Others')
+            table.appendRow(total_external)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+        table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
+        for r,_ in top_robots_referers[:10]:
+            row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
+            total_robot[1] -= robots_referers[r]['pages']
+            total_robot[2] -= robots_referers[r]['hits']
+            table.appendRow(row)
+        if total_robot[1] or total_robot[2]:
+            total_robot[0] = self.iwla._(u'Others')
+            table.appendRow(total_robot)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+
+        index.appendBlock(table)
+
+        # All key phrases in a file
+        if self.create_all_key_phrases:
+            title = createCurTitle(self.iwla, u'Key Phrases')
+
+            filename = 'key_phrases.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            total_search = [0]*2
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top key phrases'), [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
+            table.setColsCSSClass(['', 'iwla_search'])
+            new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases
+            for phrase in new_list:
+                table.appendRow([phrase[0], phrase[1]])
+                total_search[1] += phrase[1]
+            page.appendBlock(table)
+
+            display.addPage(page)
+
+        title = self.iwla._(u'Top key phrases')
+        if self.create_all_key_phrases:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All key phrases'))
+            title = '%s - %s' % (title, link)
+
+        # Top key phrases in index
+        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
+        table.setColsCSSClass(['', 'iwla_search'])
+        for phrase in top_key_phrases[:10]:
+            table.appendRow([phrase[0], phrase[1]])
+            total_search[1] -= phrase[1]
+        if total_search[1]:
+            total_search[0] = self.iwla._(u'Others')
+            table.appendRow(total_search)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+        index.appendBlock(table)

plugins/display/referers_diff.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from istats_diff import IWLADisplayStatsDiff
+from display import *
+
+"""
+Display hook
+
+Enlight new and updated key phrases in in all_key_phrases.html
+
+Plugin requirements :
+    display/referers
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayReferersDiff(IWLADisplayStatsDiff):
+    def __init__(self, iwla):
+        super(IWLADisplayReferersDiff, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLADisplayReferers']
+        self.month_stats_key = 'key_phrases'
+        self.filename = 'key_phrases.html'
+        self.block_name = u'Key phrases'
+
+    def load(self):
+        if not self.iwla.getConfValue('create_all_key_phrases_page', True):
+            return False
+        return super(IWLADisplayReferersDiff, self).load()

plugins/display/top_downloads.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create TOP downloads page
+
+Plugin requirements :
+    post_analysis/top_downloads
+
+Conf values needed :
+    max_downloads_displayed*
+    create_all_downloads_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/top_downloads.html
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopDownloads(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopDownloads, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisTopDownloads']
+        self.max_downloads = self.iwla.getConfValue('max_downloads_displayed', 0)
+        self.create_all_downloads = self.iwla.getConfValue('create_all_downloads_page', True)
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        top_downloads = self.iwla.getMonthStats()['top_downloads']
+        top_downloads = sorted(top_downloads.items(), key=lambda t: t[1], reverse=True)
+
+        # All in a file
+        if self.create_all_downloads:
+            filename = 'top_downloads.html'
+            path = self.iwla.getCurDisplayPath(filename)
+            title = createCurTitle(self.iwla, u'All Downloads')
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Downloads'), [self.iwla._(u'URI'), self.iwla._(u'Hit')])
+            table.setColsCSSClass(['', 'iwla_hit'])
+        
+            total_entrance = [0]*2
+            new_list = self.max_downloads and top_downloads[:self.max_downloads] or top_downloads
+            for (uri, entrance) in new_list:
+                table.appendRow([generateHTMLLink(uri), entrance])
+                total_entrance[1] += entrance
+            if self.max_downloads:
+                others = 0
+                for (uri, entrance) in top_downloads[self.max_downloads:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+            page.appendBlock(table)
+                        
+            display.addPage(page)
+
+        title = self.iwla._(u'Top Downloads')
+        if self.create_all_downloads:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Downloads'))
+            title = '%s - %s' % (title, link)
+
+        # Top in index
+        index = self.iwla.getDisplayIndex()
+        
+        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Hits')])
+        table.setColsCSSClass(['', 'iwla_hit'])
+        for (uri, entrance) in top_downloads[:10]:
+            table.appendRow([generateHTMLLink(uri), entrance])
+            total_entrance[1] -= entrance
+        if total_entrance[1]:
+            total_entrance[0] = self.iwla._(u'Others')
+            table.appendRow(total_entrance)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+        index.appendBlock(table)

plugins/display/top_hits.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create TOP hits page
+
+Plugin requirements :
+    post_analysis/top_hits
+
+Conf values needed :
+    max_hits_displayed*
+    create_all_hits_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/top_hits.html
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopHits(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopHits, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisTopHits']
+        self.max_hits = self.iwla.getConfValue('max_hits_displayed', 0)
+        self.create_all_hits = self.iwla.getConfValue('create_all_hits_page', True)
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        top_hits = self.iwla.getMonthStats()['top_hits']
+        top_hits = sorted(top_hits.items(), key=lambda t: t[1], reverse=True)
+
+        # All in a file
+        if self.create_all_hits:        
+            title = createCurTitle(self.iwla, u'All Hits')
+            filename = 'top_hits.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Hits'), [self.iwla._(u'URI'), self.iwla._(u'Entrance')])        
+            table.setColsCSSClass(['', 'iwla_hit'])
+            total_hits = [0]*2
+            new_list = self.max_hits and top_hits[:self.max_hits] or top_hits
+            for (uri, entrance) in new_list:
+                table.appendRow([generateHTMLLink(uri), entrance])
+                total_hits[1] += entrance
+            if self.max_hits:
+                others = 0
+                for (uri, entrance) in top_hits[self.max_hits:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+                
+            page.appendBlock(table)
+                        
+            display.addPage(page)
+
+        title = 'Top Hits'
+        if self.create_all_hits:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Hits'))
+            title = '%s - %s' % (title, link)
+
+        # Top in index
+        index = self.iwla.getDisplayIndex()
+        
+        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Entrance')])        
+        table.setColsCSSClass(['', 'iwla_hit'])
+        for (uri, entrance) in top_hits[:10]:
+            table.appendRow([generateHTMLLink(uri), entrance])
+            total_hits[1] -= entrance
+        if total_hits[1]:
+            total_hits[0] = self.iwla._(u'Others')
+            table.appendRow(total_hits)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+        index.appendBlock(table)

plugins/display/top_pages.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create TOP pages page
+
+Plugin requirements :
+    post_analysis/top_pages
+
+Conf values needed :
+    max_pages_displayed*
+    create_all_pages_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/top_pages.html
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopPages(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopPages, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisTopPages']
+        self.max_pages = self.iwla.getConfValue('max_pages_displayed', 0)
+        self.create_all_pages = self.iwla.getConfValue('create_all_pages_page', True)
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        top_pages = self.iwla.getMonthStats()['top_pages']
+        top_pages = sorted(top_pages.items(), key=lambda t: t[1], reverse=True)
+
+        # All in a page
+        if self.create_all_pages:
+            title = createCurTitle(self.iwla, u'All Pages')
+            filename = 'top_pages.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All Pages'), [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
+            table.setColsCSSClass(['', 'iwla_hit'])
+            total_hits = [0]*2        
+            new_list = self.max_pages and top_pages[:self.max_pages] or top_pages
+            for (uri, entrance) in new_list:
+                table.appendRow([generateHTMLLink(uri), entrance])
+                total_hits[1] += entrance
+            if self.max_pages:
+                others = 0
+                for (uri, entrance) in top_pages[self.max_pages:]:
+                    others += entrance
+                table.appendRow([self.iwla._(u'Others'), others])
+                table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+            page.appendBlock(table)
+
+            display.addPage(page)
+
+        title = self.iwla._(u'Top Pages')
+        if self.create_all_pages:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Pages'))
+            title = '%s - %s' % (title, link)
+
+        # Top in index
+        index = self.iwla.getDisplayIndex()
+        
+        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'URI'), self.iwla._(u'Entrance')])
+        table.setColsCSSClass(['', 'iwla_hit'])
+        for (uri, entrance) in top_pages[:10]:
+            table.appendRow([generateHTMLLink(uri), entrance])
+            total_hits[1] -= entrance
+        if total_hits[1]:
+            total_hits[0] = self.iwla._(u'Others')
+            table.appendRow(total_hits)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+        index.appendBlock(table)

plugins/display/top_visitors.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+"""
+Display hook
+
+Create TOP visitors block
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    display_visitor_ip*
+
+Output files :
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopVisitors(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopVisitors, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def hook(self):
+        hits = self.iwla.getValidVisitors()
+        display = self.iwla.getDisplay()
+        display_visitor_ip = self.iwla.getConfValue('display_visitor_ip', False)
+
+        total = [0]*5
+        for super_hit in hits.values():
+            total[1] += super_hit['viewed_pages']
+            total[2] += super_hit['viewed_hits']
+            total[3] += super_hit['bandwidth']
+
+        top_bandwidth = [(k,v['bandwidth']) for (k,v) in hits.items()]
+        top_bandwidth = sorted(top_bandwidth, key=lambda t: t[1], reverse=True)
+        top_visitors = [hits[h[0]] for h in top_bandwidth[:10]]
+
+        index = self.iwla.getDisplayIndex()
+        table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top visitors'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Bandwidth'), self.iwla._(u'Last seen')])
+        table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', ''])
+        for super_hit in top_visitors:
+            address = super_hit['remote_addr']
+            if display_visitor_ip and\
+                    super_hit.get('dns_name_replaced', False):
+                address = '%s [%s]' % (address, super_hit['remote_ip'])
+
+            row = [
+                address,
+                super_hit['viewed_pages'],
+                super_hit['viewed_hits'],
+                bytesToStr(super_hit['bandwidth']),
+                time.asctime(super_hit['last_access'])
+                ]
+            total[1] -= super_hit['viewed_pages']
+            total[2] -= super_hit['viewed_hits']
+            total[3] -= super_hit['bandwidth']
+            table.appendRow(row)
+        if total[1] or total[2] or total[3]:
+            total[0] = self.iwla._(u'Others')
+            total[3] = bytesToStr(total[3])
+            total[4] = ''
+            table.appendRow(total)
+            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
+        index.appendBlock(table)

plugins/post_analysis/__init__.py

@@ -0,0 +1 @@
+#

plugins/post_analysis/referers.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+import urllib
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+import awstats_data
+
+"""
+Post analysis hook
+
+Extract referers and key phrases from requests
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    domain_name
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+month_stats :
+   referers =>
+       pages
+       hits
+   robots_referers =>
+       pages
+       hits
+   search_engine_referers =>
+       pages
+       hits
+   key_phrases =>
+       phrase
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisReferers(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisReferers, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.conf_requires = ['domain_name']
+
+    def _getSearchEngine(self, hashid):
+        for (k, e) in self.search_engines.items():
+            for (h,h_re) in e['hashid']:
+                if hashid == h:
+                    return k
+        return None
+
+    def load(self):
+        domain_name = self.iwla.getConfValue('domain_name', '')
+        
+        if not domain_name:
+            print 'domain_name must not be empty !'
+            return False
+
+        self.own_domain_re = re.compile(r'.*%s.*' % (domain_name))
+        self.search_engines = {}
+
+        for (hashid, name) in awstats_data.search_engines_hashid.items():
+            hashid_re = re.compile(r'.*%s.*' % (hashid))
+            if not name in self.search_engines.keys():
+                self.search_engines[name] = {
+                    'hashid' : [(hashid, hashid_re)]
+                    }
+            else:
+                self.search_engines[name]['hashid'].append((hashid, hashid_re))
+            #print 'Hashid %s => %s' % (name, hashid)
+
+        for (name, known_url) in awstats_data.search_engines_knwown_url.items():
+            self.search_engines[name]['known_url'] = re.compile(known_url + '(?P<key_phrase>.+)')
+
+        for (engine, not_engine) in awstats_data.not_search_engines_keys.items():
+            not_engine_re = re.compile(r'.*%s.*' % (not_engine))
+            key = self._getSearchEngine(engine)
+            if key:
+                self.search_engines[key]['not_search_engine'] = not_engine_re
+
+        return True
+
+    def _extractKeyPhrase(self, key_phrase_re, parameters, key_phrases):
+        if not parameters or not key_phrase_re: return
+
+        for p in parameters.split('&'):
+            groups = key_phrase_re.match(p)
+            if groups:
+                key_phrase = groups.groupdict()['key_phrase']
+                key_phrase = urllib.unquote_plus(key_phrase).decode('utf8')
+                if not key_phrase in key_phrases.keys():
+                    key_phrases[key_phrase] = 1
+                else:
+                    key_phrases[key_phrase] += 1
+                break
+
+    def hook(self):
+        stats = self.iwla.getCurrentVisists()
+        month_stats = self.iwla.getMonthStats()
+        
+        referers = month_stats.get('referers', {})
+        robots_referers = month_stats.get('robots_referers', {})
+        search_engine_referers = month_stats.get('search_engine_referers', {})
+        key_phrases = month_stats.get('key_phrases', {})
+
+        for (k, super_hit) in stats.items():
+            for r in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(r): break
+                if not r['http_referer']: continue
+                
+                uri = r['extract_referer']['extract_uri']
+                if self.own_domain_re.match(uri): continue
+
+                is_search_engine = False
+                for (name, engine) in self.search_engines.items():
+                    for (hashid, hashid_re) in engine['hashid']:
+                        if not hashid_re.match(uri): continue
+
+                        not_engine = engine.get('not_search_engine', None)
+                        # Try not engine
+                        if not_engine and not_engine.match(uri): break
+                        is_search_engine = True
+                        uri = name
+
+                        parameters = r['extract_referer'].get('extract_parameters', None)
+                        key_phrase_re = engine.get('known_url', None)
+
+                        self._extractKeyPhrase(key_phrase_re, parameters, key_phrases)
+                        break
+
+                if is_search_engine:
+                    dictionary = search_engine_referers
+                elif super_hit['robot']:
+                    dictionary = robots_referers
+                    # print '%s => %s' % (uri, super_hit['remote_ip'])
+                else:
+                    dictionary = referers
+                if r['is_page']:
+                    key = 'pages'
+                else:
+                    key = 'hits'
+                if not uri in dictionary: dictionary[uri] = {'pages':0, 'hits':0}
+                dictionary[uri][key] += 1
+
+        month_stats['referers'] = referers
+        month_stats['robots_referers'] = robots_referers
+        month_stats['search_engine_referers'] = search_engine_referers
+        month_stats['key_phrases'] = key_phrases

plugins/post_analysis/reverse_dns.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import socket
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Post analysis hook
+
+Replace IP by reverse DNS names
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    reverse_dns_timeout*
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+valid_visitors:
+   remote_addr
+   dns_name_replaced
+   dns_analyzed
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisReverseDNS(IPlugin):
+    DEFAULT_DNS_TIMEOUT = 0.5
+
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisReverseDNS, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        timeout = self.iwla.getConfValue('reverse_dns_timeout',
+                                         IWLAPostAnalysisReverseDNS.DEFAULT_DNS_TIMEOUT)
+        socket.setdefaulttimeout(timeout)
+        return True
+        
+    def hook(self):
+        hits = self.iwla.getValidVisitors()
+        for (k, hit) in hits.items():
+            if hit.get('dns_analysed', False): continue
+            try:
+                name, _, _ = socket.gethostbyaddr(k)
+                hit['remote_addr'] = name.lower()
+                hit['dns_name_replaced'] = True
+            except:
+                pass
+            finally:
+                hit['dns_analysed'] = True
+

plugins/post_analysis/top_downloads.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Post analysis hook
+
+Count TOP downloads
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+month_stats:
+   top_downloads =>
+       uri
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisTopDownloads(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisTopDownloads, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.conf_requires = ['multimedia_files', 'viewed_http_codes']
+
+    def hook(self):
+        stats = self.iwla.getCurrentVisists()
+        month_stats = self.iwla.getMonthStats()
+
+        multimedia_files = self.iwla.getConfValue('multimedia_files')
+        viewed_http_codes = self.iwla.getConfValue('viewed_http_codes')
+
+        top_downloads = month_stats.get('top_downloads', {})
+
+        for (k, super_hit) in stats.items():
+            if super_hit['robot']: continue
+            for r in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(r):
+                    break
+                if not self.iwla.hasBeenViewed(r) or\
+                        r['is_page']:
+                    continue
+
+                uri = r['extract_request']['extract_uri'].lower()
+
+                isMultimedia = False
+                for ext in multimedia_files:
+                    if uri.endswith(ext):
+                        isMultimedia = True
+                        break
+
+                if isMultimedia: continue
+
+                uri = "%s%s" % (r.get('server_name', ''),
+                                r['extract_request']['extract_uri'])
+
+                if not uri in top_downloads.keys():
+                    top_downloads[uri] = 1
+                else:
+                    top_downloads[uri] += 1
+
+        month_stats['top_downloads'] = top_downloads

plugins/post_analysis/top_hits.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Post analysis hook
+
+Count TOP hits
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+month_stats:
+   top_hits =>
+       uri
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisTopHits(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisTopHits, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def hook(self):
+        stats = self.iwla.getCurrentVisists()
+        month_stats = self.iwla.getMonthStats()
+        
+        top_hits = month_stats.get('top_hits', {})
+
+        for (k, super_hit) in stats.items():
+            if super_hit['robot']: continue
+            for r in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(r):
+                    break
+                if not self.iwla.hasBeenViewed(r) or\
+                        r['is_page']:
+                    continue
+
+                uri = r['extract_request']['extract_uri'].lower()
+                uri = "%s%s" % (r.get('server_name', ''), uri)
+
+                if not uri in top_hits.keys():
+                    top_hits[uri] = 1
+                else:
+                    top_hits[uri] += 1
+
+        month_stats['top_hits'] = top_hits

plugins/post_analysis/top_pages.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Post analysis hook
+
+Count TOP pages
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+month_stats:
+   top_pages =>
+       uri
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisTopPages(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisTopPages, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        self.index_re = re.compile(r'/index.*')
+        return True
+        
+    def hook(self):
+        stats = self.iwla.getCurrentVisists()
+        month_stats = self.iwla.getMonthStats()
+        
+        top_pages = month_stats.get('top_pages', {})
+
+        for (k, super_hit) in stats.items():
+            if super_hit['robot']: continue
+            for r in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(r):
+                    break
+                if not self.iwla.hasBeenViewed(r) or\
+                        not r['is_page']:
+                    continue
+
+                uri = r['extract_request']['extract_uri']
+                if self.index_re.match(uri):
+                    uri = '/'
+                
+                uri = "%s%s" % (r.get('server_name', ''), uri)
+
+                if not uri in top_pages.keys():
+                    top_pages[uri] = 1
+                else:
+                    top_pages[uri] += 1
+
+        month_stats['top_pages'] = top_pages

plugins/pre_analysis/__init__.py

@@ -0,0 +1 @@
+#

plugins/pre_analysis/page_to_hit.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+"""
+Pre analysis hook
+Change page into hit and hit into page into statistics
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    page_to_hit_conf*
+    hit_to_page_conf*
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+visits :
+   remote_addr =>
+       is_page
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPreAnalysisPageToHit(IPlugin):
+
+    def __init__(self, iwla):
+        super(IWLAPreAnalysisPageToHit, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        # Page to hit
+        self.ph_regexps = self.iwla.getConfValue('page_to_hit_conf', [])
+        if not self.ph_regexps: return False
+        self.ph_regexps = map(lambda(r): re.compile(r), self.ph_regexps)
+
+        # Hit to page
+        self.hp_regexps = self.iwla.getConfValue('hit_to_page_conf', [])
+        if not self.hp_regexps: return False
+        self.hp_regexps = map(lambda(r): re.compile(r), self.hp_regexps)
+
+        return True
+
+    def hook(self):
+        hits = self.iwla.getCurrentVisists()
+
+        for (k, super_hit) in hits.items():
+            if super_hit['robot']: continue
+
+            for request in super_hit['requests'][::-1]:
+                if not self.iwla.isValidForCurrentAnalysis(request):
+                    break
+
+                if not self.iwla.hasBeenViewed(request):
+                    continue
+
+                uri = request['extract_request']['extract_uri']
+
+                if request['is_page']:
+                    # Page to hit
+                    for regexp in self.ph_regexps:
+                        if regexp.match(uri):
+                            #print '%s is a hit' % (uri )
+                            request['is_page'] = False
+                            super_hit['viewed_pages'] -= 1
+                            super_hit['viewed_hits'] += 1
+                            break
+                else:
+                    # Hit to page
+                    for regexp in self.hp_regexps:
+                        if regexp.match(uri):
+                            #print '%s is a page' % (uri )
+                            request['is_page'] = True
+                            super_hit['viewed_pages'] += 1
+                            super_hit['viewed_hits'] -= 1
+                            break

plugins/pre_analysis/robots.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+import awstats_data
+
+"""
+Pre analysis hook
+
+Filter robots
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    page_to_hit_conf*
+    hit_to_page_conf*
+
+Output files :
+    None
+
+Statistics creation :
+    None
+
+Statistics update :
+visits :
+   remote_addr =>
+       robot
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPreAnalysisRobots(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPreAnalysisRobots, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
+    
+        return True
+
+# Basic rule to detect robots
+    def hook(self):
+        hits = self.iwla.getCurrentVisists()
+        for (k, super_hit) in hits.items():
+            if super_hit['robot']: continue
+
+            isRobot = False
+            referers = 0
+
+            first_page = super_hit['requests'][0]
+            if not self.iwla.isValidForCurrentAnalysis(first_page): continue
+
+            for r in self.awstats_robots:
+                if r.match(first_page['http_user_agent']):
+                    isRobot = True
+                    break
+
+            if isRobot:
+                super_hit['robot'] = 1
+                continue
+
+# 1) no pages view --> robot
+            # if not super_hit['viewed_pages']:
+            #     super_hit['robot'] = 1
+            #     continue
+
+# 2) pages without hit --> robot
+            if not super_hit['viewed_hits']:
+                super_hit['robot'] = 1
+                continue
+
+            for hit in super_hit['requests']:
+# 3) /robots.txt read
+                if hit['extract_request']['http_uri'] == '/robots.txt':
+                    isRobot = True
+                    break
+
+# 4) Any referer for hits
+                if not hit['is_page'] and hit['http_referer']:
+                    referers += 1
+
+            if isRobot:
+                super_hit['robot'] = 1
+                continue
+
+            if not super_hit['viewed_pages'] and \
+                    (super_hit['viewed_hits'] and not referers):
+                super_hit['robot'] = 1
+                continue