iwla/iwla.py

#!/usr/bin/env python

import os
import re
import time
import pickle
import gzip
import importlib 

import default_conf as conf
import conf as _
conf.__dict__.update(_.__dict__)
del _

from iplugin import *
from display import *

class IWLA(object):

    ANALYSIS_CLASS = 'HTTP'
    API_VERSION = 1

    def __init__(self):
        print '==> Start'

        self.meta_infos = {}
        self.analyse_started = False
        self.current_analysis = {}
        self.cache_plugins = {}
        self.display = DisplayHTMLBuild()
        self.valid_visitors = None

        self.log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', conf.log_format)
        self.log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', self.log_format_extracted)
        self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
        self.log_re = re.compile(self.log_format_extracted)
        self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?')
        self.plugins = {conf.PRE_HOOK_DIRECTORY     : conf.pre_analysis_hooks,
                        conf.POST_HOOK_DIRECTORY    : conf.post_analysis_hooks,
                        conf.DISPLAY_HOOK_DIRECTORY : conf.display_hooks}

    def getConfValue(self, key, default):
        if not key in dir(conf):
            return default
        else:
            return conf.__dict__[key]

    def _clearVisits(self):
        self.current_analysis = {
            'days_stats' : {},
            'month_stats' : {},
            'visits' : {}
            }
        self.valid_visitors = None
        return self.current_analysis

    def getDaysStats(self):
        return self.current_analysis['days_stats']

    def getMonthStats(self):
        return self.current_analysis['month_stats']

    def getCurrentVisists(self):
        return self.current_analysis['visits']

    def getValidVisitors(self):
        return self.valid_visitors

    def getDisplay(self):
        return self.display

    def getCurTime(self):
        return self.meta_infos['last_time']

    def _clearMeta(self):
        self.meta_infos = {
            'last_time' : None
            }
        return self.meta_infos

    def _clearDisplay(self):
        self.display = DisplayHTMLBuild()
	return self.display

    def getDBFilename(self, time):
        return (conf.DB_ROOT + '%d/%d_%s') % (time.tm_year, time.tm_mon, conf.DB_FILENAME)

    def _serialize(self, obj, filename):
        base = os.path.dirname(filename)
        if not os.path.exists(base):
            os.makedirs(base)

        # TODO : remove return
        #return

        with open(filename + '.tmp', 'wb+') as f:
            pickle.dump(obj, f)
            f.seek(0)
            with gzip.open(filename, 'w') as fzip:
                fzip.write(f.read())
        os.remove(filename + '.tmp')

    def _deserialize(self, filename):
        if not os.path.exists(filename):
            return None

        with gzip.open(filename, 'r') as f:
            return pickle.load(f)
        return None

    def _callPlugins(self, root, *args):
        print '==> Call plugins (%s)' % root
        for p in self.plugins[root]:
            print '\t%s' % (p)
            mod = self.cache_plugins[root + '.' + p]
            mod.hook(*args)

    def isPage(self, request):
        for e in conf.pages_extensions:
            if request.endswith(e):
                return True

        return False

    def _appendHit(self, hit):
        remote_addr = hit['remote_addr']
	
        if not remote_addr in self.current_analysis['visits'].keys():
            self._createVisitor(hit)
            return
	
        super_hit = self.current_analysis['visits'][remote_addr]
        super_hit['requests'].append(hit)
        super_hit['bandwidth'] += int(hit['body_bytes_sent'])
        super_hit['last_access'] = self.meta_infos['last_time']

        request = hit['extract_request']

        if 'extract_uri' in request.keys():
            uri = request['extract_uri']
        else:
            uri = request['http_uri']

        hit['is_page'] = self.isPage(uri)

        status = int(hit['status'])
        if status not in conf.viewed_http_codes:
            return

        if super_hit['robot'] or\
                not status in conf.viewed_http_codes:
            page_key = 'not_viewed_pages'
            hit_key  = 'not_viewed_hits'
        else:
            page_key = 'viewed_pages'
            hit_key  = 'viewed_hits'

        if hit['is_page']:
            super_hit[page_key] += 1
        else:
            super_hit[hit_key] += 1

    def _createVisitor(self, hit):
        super_hit = self.current_analysis['visits'][hit['remote_addr']] = {}
        super_hit['remote_addr'] = hit['remote_addr']
        super_hit['remote_ip'] = hit['remote_addr']
        super_hit['viewed_pages'] = 0
        super_hit['viewed_hits'] = 0
        super_hit['not_viewed_pages'] = 0
        super_hit['not_viewed_hits'] = 0
        super_hit['bandwidth'] = 0
        super_hit['last_access'] = self.meta_infos['last_time']
        super_hit['requests'] = []
        super_hit['robot'] = False
        super_hit['hit_only'] = 0
        self._appendHit(hit)

    def _decodeHTTPRequest(self, hit):
        if not 'request' in hit.keys(): return False

        groups = self.http_request_extracted.match(hit['request'])

        if groups:
            hit['extract_request'] = groups.groupdict()
            uri_groups = self.uri_re.match(hit['extract_request']['http_uri'])
            if uri_groups:
                d = uri_groups.groupdict()
                hit['extract_request']['extract_uri'] = d['extract_uri']
                if 'extract_parameters' in d.keys():
                    hit['extract_request']['extract_parameters'] = d['extract_parameters']
        else:
            print "Bad request extraction " + hit['request']
            return False

        if hit['http_referer']:
            referer_groups = self.uri_re.match(hit['http_referer'])
            if referer_groups:
                hit['extract_referer'] = referer_groups.groupdict()
        return True

    def _decodeTime(self, hit):
        hit['time_decoded'] = time.strptime(hit['time_local'], conf.time_format)

    def getDisplayIndex(self):
        cur_time = self.meta_infos['last_time']
        filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)

        return self.display.getPage(filename)

    def _generateDisplayDaysStat(self):
        cur_time = self.meta_infos['last_time']
        title = 'Stats %d/%d' % (cur_time.tm_mon, cur_time.tm_year)
        filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
        print '==> Generate display (%s)' % (filename)
        page = DisplayHTMLPage(title, filename)

        days = DisplayHTMLBlockTable('By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'])

        keys = self.current_analysis['days_stats'].keys()
        keys.sort()
        nb_visits = 0
        for k in keys:
            stats = self.current_analysis['days_stats'][k]
            row = [k, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'],
                   bytesToStr(stats['viewed_bandwidth']), bytesToStr(stats['not_viewed_bandwidth'])]
            days.appendRow(row)
            nb_visits += stats['nb_visitors']

        stats = self.current_analysis['month_stats']

        nb_days = len(keys)
        row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
        if nb_days:
            average_row = map(lambda(v): int(v/nb_days), row)
        else:
            average_row = map(lambda(v): 0, row)

        average_row[0] = 'Average'
        average_row[4] = bytesToStr(average_row[4])
        average_row[5] = bytesToStr(average_row[5])
        days.appendRow(average_row)

        row[0] = 'Total'
        row[4] = bytesToStr(row[4])
        row[5] = bytesToStr(row[5])
        days.appendRow(row)
        page.appendBlock(days)
        self.display.addPage(page)

    def _generateDisplay(self):
        self._generateDisplayDaysStat()
        self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY, self)
        self.display.build(conf.DISPLAY_ROOT)

    def _generateStats(self, visits):
        stats = {}
        stats['viewed_bandwidth'] = 0
        stats['not_viewed_bandwidth'] = 0
        stats['viewed_pages'] = 0
        stats['viewed_hits'] = 0
        #stats['requests'] = set()
        stats['nb_visitors'] = 0

        for k in visits.keys():
            super_hit = visits[k]
            if super_hit['robot']:
                stats['not_viewed_bandwidth'] += super_hit['bandwidth']
                continue

            #print "[%s] =>\t%d/%d" % (k, super_hit['viewed_pages'], super_hit['viewed_hits'])

            if not super_hit['hit_only']:
                stats['nb_visitors'] += 1
            stats['viewed_bandwidth'] += super_hit['bandwidth']
            stats['viewed_pages'] += super_hit['viewed_pages']
            stats['viewed_hits'] += super_hit['viewed_hits']

	    # for p in super_hit['requests']:
	    #     if not p['is_page']: continue
	    #     req = p['extract_request']
	    #     stats['requests'].add(req['extract_uri'])

        return stats

    def _generateMonthStats(self):
        self._clearDisplay()

        visits = self.current_analysis['visits']

        stats = self._generateStats(visits)

        cur_time = self.meta_infos['last_time']
        print "== Stats for %d/%d ==" % (cur_time.tm_year, cur_time.tm_mon)
        print stats

        self.current_analysis['month_stats'] = stats

        self.valid_visitors = {k: v for (k,v) in visits.items() if not visits[k]['robot']}
        self._callPlugins(conf.POST_HOOK_DIRECTORY, self)

        path = self.getDBFilename(cur_time)
        if os.path.exists(path):
            os.remove(path)

        print "==> Serialize to %s" % path

        self._serialize(self.current_analysis, path)

        self._generateDisplay()

    def _generateDayStats(self):
        visits = self.current_analysis['visits']

        self._callPlugins(conf.PRE_HOOK_DIRECTORY, self)

        stats = self._generateStats(visits)

        cur_time = self.meta_infos['last_time']
        print "== Stats for %d/%d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)

        if cur_time.tm_mday > 1:
            last_day = cur_time.tm_mday - 1
            while last_day:
                if last_day in self.current_analysis['days_stats'].keys():
                    break
                last_day -= 1
            if last_day:
                for k in stats.keys():
                    stats[k] -= self.current_analysis['days_stats'][last_day][k]
                stats['nb_visitors'] = 0
                for k in visits.keys():
                    if visits[k]['robot']: continue
                    if visits[k]['last_access'].tm_mday == cur_time.tm_mday:
                        stats['nb_visitors'] += 1
        print stats

        self.current_analysis['days_stats'][cur_time.tm_mday] = stats

    def _newHit(self, hit):
        self._decodeTime(hit)

        t = hit['time_decoded']

        cur_time = self.meta_infos['last_time']

        if cur_time == None:
            self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
            self.analyse_started = True
        else:
            if not self.analyse_started:
                if time.mktime(cur_time) >= time.mktime(t):
                    return False
                else:
                    self.analyse_started = True
            if cur_time.tm_mon != t.tm_mon:
                self._generateMonthStats()
                self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
            elif cur_time.tm_mday != t.tm_mday:
                self._generateDayStats()

        self.meta_infos['last_time'] = t

        if not self._decodeHTTPRequest(hit): return False

        for k in hit.keys():
            if hit[k] == '-': hit[k] = ''

        self._appendHit(hit)

        return True

    def start(self):
        self.cache_plugins = preloadPlugins(self.plugins, self)

        print '==> Analyse previous database'

        self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta()
        if self.meta_infos['last_time']:
            self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
        else:
            self._clearVisits()

        print '==> Analysing log'

        with open(conf.analyzed_filename) as f:
            for l in f:
                # print "line " + l

                groups = self.log_re.match(l)

                if groups:
                    if not self._newHit(groups.groupdict()):
                        break
                else:
                    print "No match for " + l
                #break

        if self.analyse_started:
            self._generateDayStats()
            self._generateMonthStats()
            self._serialize(self.meta_infos, conf.META_PATH)
        else:
            print '==> Analyse not started : nothing to do'
            self._generateMonthStats()

if __name__ == '__main__':
    iwla = IWLA()
    iwla.start()