iwla/plugins/post_analysis/operating_systems.py

# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015

# This file is part of iwla

# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
#

import re

from iwla import IWLA
from iplugin import IPlugin

import awstats_data

"""
Post analysis hook

Detect operating systems from requests

Plugin requirements :
    None

Conf values needed :
    None

Output files :
    None

Statistics creation :
visits :
   remote_ip =>
       operating_system

month_stats :
    operating_systems =>
        operating_system => count

    os_families =>
        family => count

Statistics update :
    None

Statistics deletion :
    None
"""

class IWLAPostAnalysisOperatingSystems(IPlugin):
    def __init__(self, iwla):
        super(IWLAPostAnalysisOperatingSystems, self).__init__(iwla)
        self.API_VERSION = 1

    def load(self):
        self.operating_systems = []
        self.os_family = {}

        for hashid in awstats_data.operating_systems:
            hashid_re = re.compile(r'.*%s.*' % (hashid), re.IGNORECASE)

            if hashid in awstats_data.operating_systems_hashid.keys():
                self.operating_systems.append((hashid_re, awstats_data.operating_systems_hashid[hashid]))

        for (name, family) in awstats_data.operating_systems_family.items():
            name_re = re.compile(r'.*%s.*' % (name))
            self.os_family[name_re] = family

        return True

    def hook(self):
        stats = self.iwla.getValidVisitors()
        month_stats = self.iwla.getMonthStats()
        
        operating_systems = month_stats.get('operating_systems', {})

        os_stats = {}
        family_stats = {}

        for (k, super_hit) in stats.items():
            if not 'operating_system' in super_hit:
                for r in super_hit['requests'][::-1]:
                    user_agent = r['http_user_agent']
                    if not user_agent: continue
                
                    os_name = 'unknown'
                    for (hashid_re, operating_system) in self.operating_systems:
                        if hashid_re.match(user_agent):
                            os_name = operating_system
                            break
                    super_hit['operating_system'] = os_name
                    break
            else:
                os_name = super_hit['operating_system']

            os_family = ''
            if os_name != 'unknown':
                for (name_re, family) in self.os_family.items():
                    if name_re.match(os_name):
                        os_family = family
                        break

            if not os_name in os_stats.keys():
                os_stats[os_name] = 1
            else:
                os_stats[os_name] += 1

            if os_family:
                if not os_family in family_stats.keys():
                    family_stats[os_family] = 1
                else:
                    family_stats[os_family] += 1

        month_stats['operating_systems'] = os_stats
        month_stats['os_families'] = family_stats