Do a lot of things

conf.py

@@ -8,12 +8,17 @@ time_format = '%d/%b/%Y:%H:%M:%S +0100'
 
 analyzed_filename = 'access.log'
 
+domain_name = 'soutade.fr'
+
+display_visitor_ip = True
+
 DB_ROOT = './output/'
 DISPLAY_ROOT = './output/'
 
 pre_analysis_hooks = ['page_to_hit', 'robots']
-post_analysis_hooks = ['top_visitors', 'reverse_dns']
-display_hooks = ['top_visitors', 'all_visits']
+post_analysis_hooks = ['top_visitors']
+# post_analysis_hooks = ['top_visitors', 'reverse_dns']
+display_hooks = ['top_visitors', 'all_visits', 'referers']
 
 reverse_dns_timeout = 0.2
 page_to_hit_conf = [r'^.+/logo/$']

default_conf.py

@@ -21,4 +21,4 @@ post_analysis_hooks = []
 display_hooks = []
 
 pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
-viewed_http_codes = [200]
+viewed_http_codes = [200, 304]

iwla.py

@@ -34,7 +34,7 @@ class IWLA(object):
         self.log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', self.log_format_extracted)
         self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
         self.log_re = re.compile(self.log_format_extracted)
-        self.uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
+        self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?')
         self.plugins = {conf.PRE_HOOK_DIRECTORY     : conf.pre_analysis_hooks,
                         conf.POST_HOOK_DIRECTORY    : conf.post_analysis_hooks,
                         conf.DISPLAY_HOOK_DIRECTORY : conf.display_hooks}
@@ -143,9 +143,9 @@ class IWLA(object):
 
         hit['is_page'] = self.isPage(uri)
 
-        # Don't count 3xx status
         status = int(hit['status'])
-        if status >= 300 and status < 400: return
+        if status not in conf.viewed_http_codes:
+            return
 
         if super_hit['robot'] or\
                 not status in conf.viewed_http_codes:
@@ -163,6 +163,7 @@ class IWLA(object):
     def _createVisitor(self, hit):
         super_hit = self.current_analysis['visits'][hit['remote_addr']] = {}
         super_hit['remote_addr'] = hit['remote_addr']
+        super_hit['remote_ip'] = hit['remote_addr']
         super_hit['viewed_pages'] = 0
         super_hit['viewed_hits'] = 0
         super_hit['not_viewed_pages'] = 0
@@ -191,9 +192,10 @@ class IWLA(object):
             print "Bad request extraction " + hit['request']
             return False
 
-        referer_groups = self.uri_re.match(hit['http_referer'])
-        if referer_groups:
-            referer = hit['extract_referer'] = referer_groups.groupdict()
+        if hit['http_referer']:
+            referer_groups = self.uri_re.match(hit['http_referer'])
+            if referer_groups:
+                hit['extract_referer'] = referer_groups.groupdict()
         return True
 
     def _decodeTime(self, hit):
@@ -229,13 +231,13 @@ class IWLA(object):
         nb_days = len(keys)
         row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
         if nb_days:
-            average_row = map(lambda(v): str(int(v/nb_days)), row)
+            average_row = map(lambda(v): int(v/nb_days), row)
         else:
-            average_row = map(lambda(v): '0', row)
+            average_row = map(lambda(v): 0, row)
 
         average_row[0] = 'Average'
-        average_row[4] = bytesToStr(row[4])
-        average_row[5] = bytesToStr(row[5])
+        average_row[4] = bytesToStr(average_row[4])
+        average_row[5] = bytesToStr(average_row[5])
         days.appendRow(average_row)
 
         row[0] = 'Total'

iwla_convert.pl

@@ -1,34 +1,79 @@
 #!/usr/bin/perl
 
-my $awstats_lib_root = '/usr/share/awstats/lib/';
-my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
+my $awstats_lib_root = './';
+my @awstats_libs = ('search_engines.pm', 'robots.pm');
+
+# my $awstats_lib_root = '/usr/share/awstats/lib/';
+# my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm');
 
 foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;}
 
-open($FIC,">", "robots.py") or die $!;
+sub dumpList {
+    my @list = @{$_[0]};
+    my $FIC = $_[1];
+    my $first = $_[2];
 
-print $FIC "awstats_robots = [";
-$first = 0;
-foreach $r (@RobotsSearchIDOrder_list1)
-{
-    $r =~ s/\'/\\\'/g;
-    if ($first != 0)
+    foreach $r (@list)
     {
-	print $FIC ", ";
+	$r =~ s/\'/\\\'/g;
+	if ($first == 0)
+	{
+	    print $FIC ", ";
+	}
+	else
+	{
+	    $first = 0;
+	}
+	print $FIC "'.*$r.*'";
     }
-    else
-    {
-	$first = 1;
+}
+
+sub dumpHash {
+    my %hash = %{$_[0]};
+    my $FIC = $_[1];
+    my $first = $_[2];
+
+    while( my ($k,$v) = each(%hash) ) {
+	$k =~ s/\'/\\\'/g;
+	$v =~ s/\'/\\\'/g;
+	if ($first == 0)
+	{
+	    print $FIC ", ";
+	}
+	else
+	{
+	    $first = 0;
+	}
+	print $FIC "'.*$k.*' : '$v'";
     }
-    print $FIC "'.*$r.*'";
-}
-foreach $r (@RobotsSearchIDOrder_list2)
-{
-    $r =~ s/\'/\\\'/g;
-    print $FIC ", '.*$r.*'";
 }
+
+# Robots
+open($FIC,">", "awstats_data.py") or die $!;
+
+print $FIC "robots = [";
+dumpList(\@RobotsSearchIDOrder_list1, $FIC, 1);
+dumpList(\@RobotsSearchIDOrder_list2, $FIC, 0);
 print $FIC "]\n\n";
 
+print $FIC "search_engines = [";
+dumpList(\@SearchEnginesSearchIDOrder_list1, $FIC, 1);
+print $FIC "]\n\n";
+
+print $FIC "search_engines_2 = [";
+dumpList(\@SearchEnginesSearchIDOrder_list2, $FIC, 1);
+print $FIC "]\n\n";
+
+print $FIC "not_search_engines_keys = {";
+dumpHash(\%NotSearchEnginesKeys, $FIC, 1);
+print $FIC "}\n\n";
+
+print $FIC "search_engines_hashid = {";
+dumpHash(\%SearchEnginesHashID, $FIC, 1);
+print $FIC "}\n\n";
+
+print $FIC "search_engines_knwown_url = {";
+dumpHash(\%SearchEnginesKnownUrl, $FIC, 1);
+print $FIC "}\n\n";
+
 close($FIC);
-
-

plugins/display/all_visits.py

@@ -14,7 +14,7 @@ class IWLADisplayAllVisits(IPlugin):
         last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
 
         cur_time = self.iwla.getCurTime()
-        title = time.strftime('All visits %B %Y', cur_time)
+        title = time.strftime('All visits - %B %Y', cur_time)
 
         filename = 'all_visits_%d.html' % (cur_time.tm_mon)
         path = '%d/%s' % (cur_time.tm_year, filename)
@@ -22,8 +22,13 @@ class IWLADisplayAllVisits(IPlugin):
         page = DisplayHTMLPage(title, path)
         table = DisplayHTMLBlockTable('Last seen', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
         for super_hit in last_access:
+            address = super_hit['remote_addr']
+            if self.iwla.getConfValue('display_visitor_ip', False) and\
+                    super_hit.get('dns_name_replaced', False):
+                address = '%s [%s]' % (address, super_hit['remote_ip'])
+
             row = [
-                super_hit['remote_addr'],
+                address,
                 super_hit['viewed_pages'],
                 super_hit['viewed_hits'],
                 bytesToStr(super_hit['bandwidth']),

plugins/display/referers.py

@@ -0,0 +1,199 @@
+import time
+import re
+import HTMLParser
+
+from iwla import IWLA
+from iplugin import IPlugin
+from display import *
+
+import awstats_data
+
+class IWLADisplayReferers(IPlugin):
+    def __init__(self, iwla):
+        super(IWLADisplayReferers, self).__init__(iwla)
+        self.API_VERSION = 1
+
+    def load(self):
+        domain_name = self.iwla.getConfValue('domain_name', '')
+        
+        if not domain_name:
+            print 'domain_name required in conf'
+            return False
+
+        self.own_domain_re = re.compile('.*%s.*' % (domain_name))
+        self.search_engines = {}
+
+        for engine in awstats_data.search_engines:
+            self.search_engines[engine] = {
+                're' : re.compile(engine, re.IGNORECASE)
+                }
+            
+        for (engine, not_engine) in awstats_data.not_search_engines_keys.items():
+            if not engine in self.search_engines: continue
+            self.search_engines[engine]['not_search_engine'] = \
+                re.compile(not_engine, re.IGNORECASE)
+
+        for (engine, name) in awstats_data.search_engines_hashid.items():
+            if not engine in self.search_engines: continue
+            self.search_engines[engine]['name'] = name
+
+        for (engine, knwown_url) in awstats_data.search_engines_knwown_url.items():
+            engine = engin[2:-2]
+            if not engine in self.search_engines: continue
+            print knwown_url
+            self.search_engines[engine]['known_url'] = re.compile(known_url + '(?P<key_phrase>.+)')
+
+
+        self.html_parser = HTMLParser.HTMLParser()
+
+        return True
+
+    def _extractKeyPhrase(self, key_phrase_re, parameters, key_phrases):
+        if not parameters or not key_phrase_re: return
+
+
+        for p in parameters.split('&'):
+            groups = key_phrase_re.match(p)
+            if groups:
+                print groups.groupddict()
+                key_phrase = self.html_parser.unescape(groups.groupddict()['key_phrase']).lower()
+                if not key_phrase in key_phrases.keys():
+                    key_phrases[key_phrase] = 1
+                else:
+                    key_phrases[key_phrase] += 1
+
+    def hook(self, iwla):
+        stats = iwla.getCurrentVisists()
+        referers = {}
+        robots_referers = {}
+        search_engine_referers = {}
+        key_phrases = {}
+
+        for (k, super_hit) in stats.items():
+            for r in super_hit['requests']:
+                if not r['http_referer']: continue
+                
+                uri = r['extract_referer']['extract_uri']
+                is_search_engine = False
+
+                if self.own_domain_re.match(uri): continue
+
+                for e in self.search_engines.values():
+                    if e['re'].match(uri):
+                        not_engine = e.get('not_search_engine', None)
+                        # Try not engine
+                        if not_engine and not_engine.match(uri): break
+                        is_search_engine = True
+                        uri = e['name']
+
+                        parameters = r['extract_referer'].get('extract_parameters', None)
+                        key_phrase_re = e.get('known_url', None)
+                        
+                        print parameters
+                        print key_phrase_re
+
+                        self._extractKeyPhrase(key_phrase_re, parameters, key_phrases)
+
+                        break
+
+                if is_search_engine:
+                    dictionary = search_engine_referers
+                elif super_hit['robot']:
+                    dictionary = robots_referers
+                    # print '%s => %s' % (uri, super_hit['remote_ip'])
+                else:
+                    dictionary = referers
+                if r['is_page']:
+                    key = 'pages'
+                else:
+                    key = 'hits'
+                if not uri in dictionary: dictionary[uri] = {'pages':0, 'hits':0}
+                dictionary[uri][key] += 1
+
+        top_referers = [(k, referers[k]['pages']) for k in referers.keys()]
+        top_referers = sorted(top_referers, key=lambda t: t[1], reverse=True)
+
+        top_robots_referers = [(k, robots_referers[k]['pages']) for k in robots_referers.keys()]
+        top_robots_referers = sorted(top_robots_referers, key=lambda t: t[1], reverse=True)
+
+        top_search_engine_referers = [(k, search_engine_referers[k]['pages']) for k in search_engine_referers.keys()]
+        top_search_engine_referers = sorted(top_search_engine_referers, key=lambda t: t[1], reverse=True)
+
+        top_key_phrases = key_phrases.items()
+        top_key_phrases = sorted(top_key_phrases, key=lambda t: t[1], reverse=True)
+
+        # Top referers in index
+        index = self.iwla.getDisplayIndex()
+
+        table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
+        table.appendRow(['<b>Search Engine</b>', '', ''])
+        for r,_ in top_search_engine_referers[:10]:
+            row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
+            table.appendRow(row)
+
+        table.appendRow(['<b>External URL</b>', '', ''])
+        for r,_ in top_referers[:10]:
+            row = [r, referers[r]['pages'], referers[r]['hits']]
+            table.appendRow(row)
+
+        table.appendRow(['<b>External URL (robot)</b>', '', ''])
+        for r,_ in top_robots_referers[:10]:
+            row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
+            table.appendRow(row)
+
+        index.appendBlock(table)
+
+        # All referers in a file
+        cur_time = self.iwla.getCurTime()
+        title = time.strftime('Connexion from - %B %Y', cur_time)
+
+        filename = 'referers_%d.html' % (cur_time.tm_mon)
+        path = '%d/%s' % (cur_time.tm_year, filename)
+
+        page = DisplayHTMLPage(title, path)
+        table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
+                        
+        table.appendRow(['<b>Search Engine</b>', '', ''])
+        for r,_ in top_search_engine_referers:
+            row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
+            table.appendRow(row)
+
+        table.appendRow(['<b>External URL</b>', '', ''])
+        for r,_ in top_referers:
+            row = [r, referers[r]['pages'], referers[r]['hits']]
+            table.appendRow(row)
+
+        table.appendRow(['<b>External URL (robot)</b>', '', ''])
+        for r,_ in top_robots_referers:
+            row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
+            table.appendRow(row)
+
+        page.appendBlock(table)
+
+        display = self.iwla.getDisplay()
+        display.addPage(page)
+
+        block = DisplayHTMLRawBlock()
+        block.setRawHTML('<a href=\'%s\'>All referers</a>' % (filename))
+        index.appendBlock(block)
+
+        # Top key phrases in index
+        table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
+        for phrase in top_key_phrases[:10]:
+            table.appendRow([phrase[0], phrase[1]])
+        index.appendBlock(table)
+
+        # All key phrases in a file
+        cur_time = self.iwla.getCurTime()
+        title = time.strftime('Key Phrases - %B %Y', cur_time)
+
+        filename = 'key_phrases_%d.html' % (cur_time.tm_mon)
+        path = '%d/%s' % (cur_time.tm_year, filename)
+
+        page = DisplayHTMLPage(title, path)
+        table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
+        for phrase in top_key_phrases:
+            table.appendRow([phrase[0], phrase[1]])
+        page.appendBlock(table)
+                        
+        display.addPage(page)

plugins/display/top_visitors.py

@@ -16,8 +16,13 @@ class IWLADisplayTopVisitors(IPlugin):
         index = iwla.getDisplayIndex()
         table = DisplayHTMLBlockTable('Top visitors', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
         for super_hit in stats['top_visitors']:
+            address = super_hit['remote_addr']
+            if self.iwla.getConfValue('display_visitor_ip', False) and\
+                    super_hit.get('dns_name_replaced', False):
+                address = '%s [%s]' % (address, super_hit['remote_ip'])
+
             row = [
-                super_hit['remote_addr'],
+                address,
                 super_hit['viewed_pages'],
                 super_hit['viewed_hits'],
                 bytesToStr(super_hit['bandwidth']),

plugins/post_analysis/reverse_dns.py

@@ -20,6 +20,7 @@ class IWLAPostAnalysisReverseDNS(IPlugin):
             try:
                 name, _, _ = socket.gethostbyaddr(k)
                 hit['remote_addr'] = name
+                hit['dns_name_replaced'] = True
             except:
                 pass
             finally:

plugins/post_analysis/top_visitors.py

@@ -9,7 +9,7 @@ class IWLAPostAnalysisTopVisitors(IPlugin):
     def hook(self, iwla):
         hits = iwla.getValidVisitors()
         stats = iwla.getMonthStats()
-        top_bandwidth = [(k,hits[k]['bandwidth']) for (k,v) in hits.items()]
+        top_bandwidth = [(k,hits[k]['bandwidth']) for k in hits.keys()]
         top_bandwidth = sorted(top_bandwidth, key=lambda t: t[1], reverse=True)
         stats['top_visitors'] = [hits[h[0]] for h in top_bandwidth[:10]]
 

plugins/pre_analysis/robots.py

@@ -3,7 +3,7 @@ import re
 from iwla import IWLA
 from iplugin import IPlugin
 
-from awstats_robots_data import awstats_robots
+import awstats_data
 
 class IWLAPreAnalysisRobots(IPlugin):
     def __init__(self, iwla):
@@ -11,9 +11,7 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.API_VERSION = 1
 
     def load(self):
-        global awstats_robots
-
-        self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
+        self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_data.robots)
     
         return True
 
@@ -32,13 +30,17 @@ class IWLAPreAnalysisRobots(IPlugin):
             if first_page['time_decoded'].tm_mday == super_hit['last_access'].tm_mday:
                 for r in self.awstats_robots:
                     if r.match(first_page['http_user_agent']):
-                        super_hit['robot'] = 1
-                        continue
+                        isRobot = True
+                        break
+
+                if isRobot:
+                    super_hit['robot'] = 1
+                    continue
 
 # 1) no pages view --> robot
-            if not super_hit['viewed_pages']:
-                super_hit['robot'] = 1
-                continue
+            # if not super_hit['viewed_pages']:
+            #     super_hit['robot'] = 1
+            #     continue
 
 # 2) pages without hit --> robot
             if not super_hit['viewed_hits']:
@@ -59,6 +61,7 @@ class IWLAPreAnalysisRobots(IPlugin):
                 super_hit['robot'] = 1
                 continue
 
-            if super_hit['viewed_hits'] and not referers:
+            if not super_hit['viewed_pages'] and \
+                    (super_hit['viewed_hits'] and not referers):
                 super_hit['robot'] = 1
                 continue