Initial commit

iwla.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+
+import os
+import re
+import time
+import glob
+import imp
+from robots import awstats_robots;
+
+print '==> Start'
+
+current_visit = {}
+
+log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local] ' +\
+    '"$request" $status $body_bytes_sent ' +\
+    '"$http_referer" "$http_user_agent"';
+
+log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', log_format);
+log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', log_format_extracted)
+http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
+#09/Nov/2014:06:35:16 +0100
+time_format = '%d/%b/%Y:%H:%M:%S +0100'
+#print "Log format : " + log_format_extracted
+
+log_re = re.compile(log_format_extracted)
+uri_re = re.compile(r'(?P<extract_uri>[^\?]*)\?(?P<extract_parameters>.*)')
+pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
+viewed_http_codes = [200]
+
+cur_time = None
+
+print '==> Generating robot dictionary'
+
+awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
+
+def isPage(request):
+    for e in pages_extensions:
+        if request.endswith(e):
+            return True
+
+    return False
+
+def appendHit(hit):
+    super_hit = current_visit[hit['remote_addr']]
+    super_hit['pages'].append(hit)
+    super_hit['bandwith'] += int(hit['body_bytes_sent'])
+    
+    request = hit['extract_request']
+
+    if 'extract_uri' in request.keys():
+        uri = request['extract_uri']
+    else:
+        uri = request['http_uri']
+
+    hit['is_page'] = isPage(uri)
+
+    # Don't count redirect status
+    if int(hit['status']) == 302: return
+
+    if super_hit['robot'] or\
+            not int(hit['status']) in viewed_http_codes:
+        page_key = 'not_viewed_pages'
+        hit_key  = 'not_viewed_hits'
+    else:
+        page_key = 'viewed_pages'
+        hit_key  = 'viewed_hits'
+    
+    if hit['is_page']:
+        super_hit[page_key] += 1
+    else:
+        super_hit[hit_key] += 1
+    
+def createGeneric(hit):
+    super_hit = current_visit[hit['remote_addr']] = {}
+    super_hit['viewed_pages'] = 0;
+    super_hit['viewed_hits'] = 0;
+    super_hit['not_viewed_pages'] = 0;
+    super_hit['not_viewed_hits'] = 0;
+    super_hit['bandwith'] = 0;
+    super_hit['pages'] = [];
+    
+    return super_hit
+
+def createUser(hit, robot):
+    super_hit = createGeneric(hit)
+    super_hit['robot'] = robot;
+    appendHit(hit)
+
+def isRobot(hit):
+    for r in awstats_robots:
+        if r.match(hit['http_user_agent']):
+            return True
+    return False
+
+def decode_http_request(hit):
+    if not 'request' in hit.keys(): return False
+
+    groups = http_request_extracted.match(hit['request'])
+    
+    if groups:
+        hit['extract_request'] = groups.groupdict()
+        uri_groups = uri_re.match(hit['extract_request']['http_uri']);
+        if uri_groups:
+            hit['extract_request']['extract_uri'] = uri_groups.group('extract_uri')
+            hit['extract_request']['extract_parameters'] = uri_groups.group('extract_parameters')
+    else:
+        print "Bad request extraction " + hit['request']
+        return False
+
+    referer_groups = uri_re.match(hit['http_referer']);
+    if referer_groups:
+            hit['extract_referer']['extract_uri'] = referer_groups.group('extract_uri')
+            hit['extract_referer']['extract_parameters'] = referer_groups.group('extract_parameters')
+    return True
+
+def decode_time(hit):
+    t = hit['time_local']
+
+    hit['time_decoded'] = time.strptime(t, time_format)
+
+
+def newHit(hit):
+    global cur_time
+
+    if not decode_http_request(hit): return
+
+    for k in hit.keys():
+        if hit[k] == '-': hit[k] = ''
+
+    decode_time(hit)
+
+    t = hit['time_decoded']
+
+    current_visit['last_time'] = t
+
+    if cur_time == None:
+        cur_time = t
+    else:
+        if cur_time.tm_mday != t.tm_mday:
+            return False
+
+    remote_addr = hit['remote_addr']
+    if remote_addr in current_visit.keys():
+        appendHit(hit)
+    else:
+        createUser(hit, isRobot(hit))
+
+    return True
+
+print '==> Analysing log'
+f = open("access.log")
+for l in f:
+    # print "line " + l;
+    
+    groups = log_re.match(l)
+    
+    if groups:
+        if not newHit(groups.groupdict()):
+            break
+    else:
+        print "No match " + l
+f.close();
+
+print '==> Call plugins'
+plugins = glob.glob('./hooks_pre/*.py')
+plugins.sort()
+for p in plugins:
+    print '\t%s' % (p)
+    mod = imp.load_source('hook', p)
+    mod.hook(current_visit)
+
+for ip in current_visit.keys():
+    hit = current_visit[ip]
+    if hit['robot']: continue
+    print "%s =>" % (ip)
+    for k in hit.keys():
+        if k != 'pages':
+            print "\t%s : %s" % (k, current_visit[ip][k])

iwla_convert.pl

@@ -0,0 +1,34 @@
+#!/usr/bin/perl
+
+my $awstats_lib_root = '/usr/share/awstats/lib/';
+my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
+
+foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
+
+open($FIC,">", "robots.py") or die $!;
+
+print $FIC "awstats_robots = [";
+$first = 0;
+foreach $r (@RobotsSearchIDOrder_list1)
+{
+    $r =~ s/\'/\\\'/g;
+    if ($first != 0)
+    {
+	print $FIC ", ";
+    }
+    else
+    {
+	$first = 1;
+    }
+    print $FIC "'.*$r.*'";
+}
+foreach $r (@RobotsSearchIDOrder_list2)
+{
+    $r =~ s/\'/\\\'/g;
+    print $FIC ", '.*$r.*'";
+}
+print $FIC "]\n\n";
+
+close($FIC);
+
+