Add Operating Systems post analysis and display

conf.py

@@ -10,8 +10,8 @@ display_visitor_ip = True
 
 # Hooks used
 pre_analysis_hooks = ['page_to_hit', 'robots']
-post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'reverse_dns']
-display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff']
+post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers'] #, 'reverse_dns']
+display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers']
 
 # Reverse DNS timeout
 reverse_dns_timeout = 0.2

plugins/display/operating_systems.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+import awstats_data
+
+"""
+Display hook
+
+Add operating systems statistics
+
+Plugin requirements :
+    post_analysis/operating_systems
+
+Conf values needed :
+    create_families_page*
+
+Output files :
+    OUTPUT_ROOT/year/month/index.html
+
+Statistics creation :
+    None
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLADisplayTopOperatingSystems(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayTopOperatingSystems, self).__init__(iwla)
+        self.API_VERSION = 1
+        self.requires = ['IWLAPostAnalysisOperatingSystems']
+
+    def load(self):
+        self.icon_path = self.iwla.getConfValue('icon_path', '/')
+        self.create_families_pages = self.iwla.getConfValue('create_families_pages_page', True)
+        self.icon_names = {v:k for (k, v) in awstats_data.operating_systems_family.items()}
+
+        return True
+
+    def hook(self):
+        display = self.iwla.getDisplay()
+        os_families = self.iwla.getMonthStats()['os_families']
+        os_families = sorted(os_families.items(), key=lambda t: t[1], reverse=True)
+        operating_systems = self.iwla.getMonthStats()['operating_systems']
+        operating_systems = sorted(operating_systems.items(), key=lambda t: t[1], reverse=True)
+
+        # All in a page
+        if self.create_families_pages:
+            title = createCurTitle(self.iwla, u'All Operating Systems')
+            filename = 'operating_systems.html'
+            path = self.iwla.getCurDisplayPath(filename)
+
+            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Operating Systems'), ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')])
+            table.setColsCSSClass(['', '', 'iwla_hit'])
+            for (os_name, entrance) in operating_systems:
+                icon = '<img src="/%s/os/%s.png"/>' % (self.icon_path, os_name)
+                table.appendRow([icon, os_name, entrance])
+            page.appendBlock(table)
+
+            display.addPage(page)
+
+        # Families in index
+        title = self.iwla._(u'Operating Systems')
+        if self.create_families_pages:
+            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
+            title = '%s - %s' % (title, link)
+
+        index = self.iwla.getDisplayIndex()
+        
+        table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')])        
+        table.setColsCSSClass(['', '', 'iwla_hit'])
+        for (family, entrance) in os_families:
+            icon = '<img src="/%s/os/%s.png"/>' % (self.icon_path, self.icon_names[family])
+            table.appendRow([icon, family, entrance])
+        index.appendBlock(table)

plugins/post_analysis/operating_systems.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright Grégory Soutadé 2015
+
+# This file is part of iwla
+
+# iwla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# iwla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+
+from iwla import IWLA
+from iplugin import IPlugin
+
+import awstats_data
+
+"""
+Post analysis hook
+
+Detect operating systems from requests
+
+Plugin requirements :
+    None
+
+Conf values needed :
+    None
+
+Output files :
+    None
+
+Statistics creation :
+visits :
+   remote_addr =>
+       operating_system
+
+month_stats :
+    operating_systems =>
+        operating_system => count
+
+    os_families =>
+        family => count
+
+Statistics update :
+    None
+
+Statistics deletion :
+    None
+"""
+
+class IWLAPostAnalysisOperatingSystems(IPlugin):
+    def __init__(self, iwla):
+        super(IWLAPostAnalysisOperatingSystems, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        self.operating_systems = []
+        self.os_family = {}
+
+        for hashid in awstats_data.operating_systems:
+            hashid_re = re.compile(r'.*%s.*' % (hashid), re.IGNORECASE)
+
+            if hashid in awstats_data.operating_systems_hashid.keys():
+                self.operating_systems.append((hashid_re, awstats_data.operating_systems_hashid[hashid]))
+
+        for (name, family) in awstats_data.operating_systems_family.items():
+            name_re = re.compile(r'.*%s.*' % (name))
+            self.os_family[name_re] = family
+
+        return True
+
+    def hook(self):
+        stats = self.iwla.getValidVisitors()
+        month_stats = self.iwla.getMonthStats()
+        
+        operating_systems = month_stats.get('operating_systems', {})
+
+        os_stats = {}
+        family_stats = {}
+
+        for (k, super_hit) in stats.items():
+            if not 'operating_system' in super_hit:
+                for r in super_hit['requests'][::-1]:
+                    user_agent = r['http_user_agent']
+                    if not user_agent: continue
+                
+                    os_name = 'unknown'
+                    for (hashid_re, operating_system) in self.operating_systems:
+                        if hashid_re.match(user_agent):
+                            os_name = operating_system
+                            break
+                    super_hit['operating_system'] = os_name
+                    break
+            else:
+                os_name = super_hit['operating_system']
+
+            os_family = ''
+            if os_name != 'unknown':
+                for (name_re, family) in self.os_family.items():
+                    if name_re.match(os_name):
+                        os_family = family
+                        break
+
+            if not os_name in os_stats.keys():
+                os_stats[os_name] = 1
+            else:
+                os_stats[os_name] += 1
+
+            if os_family:
+                if not os_family in family_stats.keys():
+                    family_stats[os_family] = 1
+                else:
+                    family_stats[os_family] += 1
+
+        month_stats['operating_systems'] = os_stats
+        month_stats['os_families'] = family_stats

tools/iwla_convert.pl

@@ -1,9 +1,9 @@
 #!/usr/bin/perl
 
-my $awstats_lib_root = './';
-my @awstats_libs = ('search_engines.pm', 'robots.pm');
+my $awstats_lib_root = '/usr/share/awstats/lib/';
+# my $awstats_lib_root = './';
+my @awstats_libs = ('search_engines.pm', 'robots.pm', 'operating_systems.pm', 'browsers.pm');
 
-# my $awstats_lib_root = '/usr/share/awstats/lib/';
 # my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
 
 foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
@@ -51,6 +51,8 @@ sub dumpHash {
 # Robots
 open($FIC,">", "awstats_data.py") or die $!;
 
+print $FIC "#This file was automatically generated by iwla_convert.pl. Do not edit manually.\n\n";
+
 print $FIC "robots = [";
 dumpList(\@RobotsSearchIDOrder_list1, $FIC, 1);
 dumpList(\@RobotsSearchIDOrder_list2, $FIC, 0);
@@ -76,4 +78,28 @@ print $FIC "search_engines_knwown_url = {";
 dumpHash(\%SearchEnginesKnownUrl, $FIC, 1);
 print $FIC "}\n\n";
 
+print $FIC "operating_systems = [";
+dumpList(\@OSSearchIDOrder, $FIC, 1);
+print $FIC "]\n\n";
+
+print $FIC "operating_systems_hashid = {";
+dumpHash(\%OSHashID, $FIC, 1);
+print $FIC "}\n\n";
+
+print $FIC "operating_systems_family = {";
+dumpHash(\%OSFamily, $FIC, 1);
+print $FIC "}\n\n";
+
+print $FIC "browsers = [";
+dumpList(\@BrowsersSearchIDOrder, $FIC, 1);
+print $FIC "]\n\n";
+
+print $FIC "browsers_hashid = {";
+dumpHash(\%BrowsersHashIDLib, $FIC, 1);
+print $FIC "}\n\n";
+
+print $FIC "browsers_icons = {";
+dumpHash(\%BrowsersHashIcon, $FIC, 1);
+print $FIC "}\n\n";
+
 close($FIC);