iwla/iwla.py

#!/usr/bin/env python

import os
import re
import time
import glob
import imp
import pickle
import gzip

# Default configuration

DB_ROOT = './output/'
DISPLAY_ROOT = './output/'

log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local] ' +\
    '"$request" $status $body_bytes_sent ' +\
    '"$http_referer" "$http_user_agent"';

time_format = '%d/%b/%Y:%H:%M:%S +0100'

pre_analysis_hooks = []
post_analysis_hooks = []
display_hooks = []

from conf import *

print '==> Start'

meta_visit = {}
analyse_started = False
current_visits = {}
cache_plugins = {}
display = {}

log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', log_format);
log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', log_format_extracted)
http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')

log_re = re.compile(log_format_extracted)
uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
viewed_http_codes = [200]

HOOKS_ROOT = './plugins/'
PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/'
POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/'
DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/'
META_PATH = DB_ROOT + 'meta.db'
DB_FILENAME = 'iwla.db'

plugins = {PRE_HOOK_DIRECTORY : pre_analysis_hooks, POST_HOOK_DIRECTORY : post_analysis_hooks, DISPLAY_HOOK_DIRECTORY : display_hooks}

ANALYSIS_CLASS = 'HTTP'
API_VERSION = 1

def preloadPlugins():
    for root in plugins.keys():
        for plugin_name in plugins[root]:
            p = root + '/' + plugin_name
            try:
                mod = cache_plugins[p] = imp.load_source('hook', p)
                infos = mod.get_plugins_infos()
                if infos['class'] != ANALYSIS_CLASS or \
                        API_VERSION < infos['min_version']  or\
                        (infos['max_version'] != -1 and (API_VERSION > infos['max_version'])):
                    del cache_plugins[p]
                elif not mod.load():
                    del cache_plugins[p]
            except Exception as e:
                print 'Error loading \'%s\' => %s' % (p, e)
                return False
    return True
            

def createEmptyVisits():
    visits = {'days_stats' : {}, 'month_stats' : {}, 'visits' : {}}
    return visits

def createEmptyMeta():
    meta = {'last_time' : None}
    return meta

def createEmptyDisplay():
    display = {}
    return display

def createPage(filename, title):
    page = {}
    page['title'] = title;
    page['blocks'] = []
    display[filename] = page

    return page

def getDBFilename(time):
    return (DB_ROOT + '%d/%d_%s') % (time.tm_year, time.tm_mon, DB_FILENAME)

def serialize(obj, filename):
    base = os.path.dirname(filename)
    if not os.path.exists(base):
        os.makedirs(base)

    # TODO : remove return
    return

    with open(filename + '.tmp', 'wb+') as f:
        pickle.dump(obj, f)
        f.seek(0)
        with gzip.open(filename, 'w') as fzip:
            fzip.write(f.read())
        os.remove(filename + '.tmp')

def deserialize(filename):
    if not os.path.exists(filename):
        return None

    with gzip.open(filename, 'r') as f:
        return pickle.load(f)
    return None

def callPlugins(root, *kwargs):
    print '==> Call plugins (%s)' % root
    for p in plugins[root]:
        print '\t%s' % (p)
        mod = cache_plugins[root + '/' + p]
        mod.hook(*kwargs)

def isPage(request):
    for e in pages_extensions:
        if request.endswith(e):
            return True

    return False

def appendHit(hit):
    remote_addr = hit['remote_addr']

    if not remote_addr in current_visits['visits'].keys():
        createUser(hit)
        return

    super_hit = current_visits['visits'][remote_addr]
    super_hit['pages'].append(hit)
    super_hit['bandwith'] += int(hit['body_bytes_sent'])
    super_hit['last_access'] = meta_visit['last_time']

    request = hit['extract_request']

    if 'extract_uri' in request.keys():
        uri = request['extract_uri']
    else:
        uri = request['http_uri']

    hit['is_page'] = isPage(uri)

    # Don't count 3xx status
    status = int(hit['status'])
    if status >= 300 and status < 400: return

    if super_hit['robot'] or\
            not status in viewed_http_codes:
        page_key = 'not_viewed_pages'
        hit_key  = 'not_viewed_hits'
    else:
        page_key = 'viewed_pages'
        hit_key  = 'viewed_hits'
    
    if hit['is_page']:
        super_hit[page_key] += 1
    else:
        super_hit[hit_key] += 1
    
def createUser(hit):
    super_hit = current_visits['visits'][hit['remote_addr']] = {}
    super_hit['viewed_pages'] = 0;
    super_hit['viewed_hits'] = 0;
    super_hit['not_viewed_pages'] = 0;
    super_hit['not_viewed_hits'] = 0;
    super_hit['bandwith'] = 0;
    super_hit['last_access'] = meta_visit['last_time']
    super_hit['pages'] = [];
    super_hit['robot'] = False
    super_hit['hit_only'] = 0;
    appendHit(hit)

def decodeHTTPRequest(hit):
    if not 'request' in hit.keys(): return False

    groups = http_request_extracted.match(hit['request'])
    
    if groups:
        hit['extract_request'] = groups.groupdict()
        uri_groups = uri_re.match(hit['extract_request']['http_uri']);
        if uri_groups:
            d = uri_groups.groupdict()
            hit['extract_request']['extract_uri'] = d['extract_uri']
            if 'extract_parameters' in d.keys():
                hit['extract_request']['extract_parameters'] = d['extract_parameters']
    else:
        print "Bad request extraction " + hit['request']
        return False

    referer_groups = uri_re.match(hit['http_referer']);
    if referer_groups:
        referer = hit['extract_referer'] = referer_groups.groupdict()
    return True

def decodeTime(hit):
    t = hit['time_local']

    hit['time_decoded'] = time.strptime(t, time_format)

def buildPages():
    for filename in display.keys():
        page = display[filename]
        with open(DISPLAY_ROOT + filename, 'w') as f:
            f.write('<html><title>%s</title><body>' % (page['title']))
            for block in page['blocks']:
                if block['type'] == 'html':
                    f.write(block['value'])
                elif block['type'] == 'table':
                    f.write('<table>')
                    f.write('<tr>')
                    for title in block['cols']:
                        f.write('<th>%s</th>' % (title))
                    f.write('</tr>')
                    for row in block['rows']:
                        f.write('<tr>')
                        for v in row:
                            f.write('<td>%s</td>' % (v))
                        f.write('</tr>')
                    f.write('</table>')
            f.write('</body></html>')

def generateDisplayDaysStat():
    cur_time = meta_visit['last_time']
    title = 'Stats %d/%d' % (cur_time.tm_mon, cur_time.tm_year)
    filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
    page = createPage(filename, title)
    
    days = {'type' : 'table', 'title' : 'By day'}
    days['cols'] = ['Day', 'Visits', 'Pages', 'Hits', 'Bandwith', 'Robot Bandwith']
    days['rows'] = []
    keys = current_visits['days_stats'].keys()
    keys.sort()
    nb_visits = 0
    for k in keys:
        stats = current_visits['days_stats'][k]
        row = [k, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
        row = map(lambda(v): str(v), row)
        days['rows'].append(row)
        nb_visits += stats['nb_visitors']

    stats = current_visits['month_stats']

    nb_days = len(keys)
    row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
    if nb_days:
        row = map(lambda(v): str(int(v/nb_days)), row)
    else:
        row = map(lambda(v): '0', row)

    row[0] = 'Average'
    days['rows'].append(row)

    row = ['Total', nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
    row = map(lambda(v): str(v), row)
    days['rows'].append(row)
    page['blocks'].append(days)

def generateDisplay():
    generateDisplayDaysStat()
    callPlugins(DISPLAY_HOOK_DIRECTORY, current_visits, display)
    buildPages()

def generateStats(visits):
    stats = {}
    stats['viewed_bandwidth'] = 0
    stats['not_viewed_bandwidth'] = 0
    stats['viewed_pages'] = 0
    stats['viewed_hits'] = 0
    #stats['pages'] = set()
    stats['nb_visitors'] = 0

    for k in visits.keys():
        super_hit = visits[k]
        if super_hit['robot']:
            stats['not_viewed_bandwidth'] += super_hit['bandwith']
            continue

        print "[%s] =>\t%d/%d" % (k, super_hit['viewed_pages'], super_hit['viewed_hits'])
        
        if not super_hit['hit_only']:
            stats['nb_visitors'] += 1
        stats['viewed_bandwidth'] += super_hit['bandwith']
        stats['viewed_pages'] += super_hit['viewed_pages']
        stats['viewed_hits'] += super_hit['viewed_hits']

        # for p in super_hit['pages']:
        #     if not p['is_page']: continue
        #     req = p['extract_request']
        #     stats['pages'].add(req['extract_uri'])

    return stats
            
def generateMonthStats():
    display = createEmptyDisplay()

    visits = current_visits['visits']
    
    stats = generateStats(visits)
    
    cur_time = meta_visit['last_time']
    print "== Stats for %d/%d ==" % (cur_time.tm_year, cur_time.tm_mon)
    print stats

    valid_visitors = {k: v for (k,v) in visits.items() if not visits[k]['robot']}
    callPlugins(POST_HOOK_DIRECTORY, valid_visitors)

    current_visits['month_stats'] = stats

    path = getDBFilename(cur_time)
    if os.path.exists(path):
        os.remove(path)

    print "==> Serialize to %s" % path

    serialize(current_visits, path)

    generateDisplay()

def generateDayStats():
    visits = current_visits['visits']
    
    callPlugins(PRE_HOOK_DIRECTORY, visits)

    stats = generateStats(visits)
    
    cur_time = meta_visit['last_time']
    print "== Stats for %d/%d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)

    if cur_time.tm_mday > 1:
        last_day = cur_time.tm_mday - 1
        while last_day:
            if last_day in current_visits['days_stats'].keys():
                break
            last_day -= 1
        if last_day:
            for k in stats.keys():
                stats[k] -= current_visits['days_stats'][last_day][k]
            stats['nb_visitors'] = 0
            for k in visits.keys():
                if visits[k]['robot']: continue
                if visits[k]['last_access'].tm_mday == cur_time.tm_mday:
                    stats['nb_visitors'] += 1
    print stats

    current_visits['days_stats'][cur_time.tm_mday] = stats

def newHit(hit):
    global current_visits
    global analyse_started

    decodeTime(hit)

    t = hit['time_decoded']

    cur_time = meta_visit['last_time']

    if cur_time == None:
        current_visits = deserialize(getDBFilename(t)) or createEmptyVisits()
        analyse_started = True
    else:
        if not analyse_started:
            if time.mktime(cur_time) >= time.mktime(t):
                return
            else:
                analyse_started = True
                current_visits = deserialize(getDBFilename(t)) or createEmptyVisits()
        if cur_time.tm_mon != t.tm_mon:
            generateMonthStats()
            current_visits = deserialize(getDBFilename(t)) or createEmptyVisits()
        elif cur_time.tm_mday != t.tm_mday:
            generateDayStats()

    meta_visit['last_time'] = t

    if not decodeHTTPRequest(hit): return False

    for k in hit.keys():
        if hit[k] == '-': hit[k] = ''

    appendHit(hit)

    return True

preloadPlugins()

print '==> Analysing log'

meta_visit = deserialize(META_PATH) or createEmptyMeta()

current_visits = createEmptyVisits()

f = open(analyzed_filename)
for l in f:
    # print "line " + l;
    
    groups = log_re.match(l)
    
    if groups:
        if not newHit(groups.groupdict()):
            break
    else:
        print "No match " + l
f.close();

if analyse_started:
    generateDayStats()
    generateMonthStats()
    serialize(meta_visit, META_PATH)
else:
    print '==> Analyse not started : nothing to do'