Externalize plugins

iwla.py

@@ -8,8 +8,6 @@ import imp
 import pickle
 import gzip
 
-from robots import awstats_robots;
-
 # Default configuration
 
 DB_ROOT = './output/'
@@ -21,6 +19,10 @@ log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local
 
 time_format = '%d/%b/%Y:%H:%M:%S +0100'
 
+pre_analysis_hooks = []
+post_analysis_hooks = []
+display_hooks = []
+
 from conf import *
 
 print '==> Start'
@@ -40,16 +42,36 @@ uri_re = re.compile(r'(?P<extract_uri>[^\?]*)[\?(?P<extract_parameters>.*)]?')
 pages_extensions = ['/', 'html', 'xhtml', 'py', 'pl', 'rb', 'php']
 viewed_http_codes = [200]
 
-HOOKS_ROOT = './hooks/'
-PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/*.py'
-POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/*.py'
-DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/*.py'
+HOOKS_ROOT = './plugins/'
+PRE_HOOK_DIRECTORY = HOOKS_ROOT + 'pre_analysis/'
+POST_HOOK_DIRECTORY = HOOKS_ROOT + 'post_analysis/'
+DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + 'display/'
 META_PATH = DB_ROOT + 'meta.db'
 DB_FILENAME = 'iwla.db'
 
-print '==> Generating robot dictionary'
+plugins = {PRE_HOOK_DIRECTORY : pre_analysis_hooks, POST_HOOK_DIRECTORY : post_analysis_hooks, DISPLAY_HOOK_DIRECTORY : display_hooks}
 
-awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
+ANALYSIS_CLASS = 'HTTP'
+API_VERSION = 1
+
+def preloadPlugins():
+    for root in plugins.keys():
+        for plugin_name in plugins[root]:
+            p = root + '/' + plugin_name
+            try:
+                mod = cache_plugins[p] = imp.load_source('hook', p)
+                infos = mod.get_plugins_infos()
+                if infos['class'] != ANALYSIS_CLASS or \
+                        API_VERSION < infos['min_version']  or\
+                        (infos['max_version'] != -1 and (API_VERSION > infos['max_version'])):
+                    del cache_plugins[p]
+                elif not mod.load():
+                    del cache_plugins[p]
+            except Exception as e:
+                print 'Error loading \'%s\' => %s' % (p, e)
+                return False
+    return True
+            
 
 def createEmptyVisits():
     visits = {'days_stats' : {}, 'month_stats' : {}, 'visits' : {}}
@@ -97,17 +119,11 @@ def deserialize(filename):
         return pickle.load(f)
     return None
 
-def callPlugins(path, *kwargs):
-    print '==> Call plugins (%s)' % path
-    plugins = glob.glob(path)
-    plugins.sort()
-    for p in plugins:
+def callPlugins(root, *kwargs):
+    print '==> Call plugins (%s)' % root
+    for p in plugins[root]:
         print '\t%s' % (p)
-        if not p in cache_plugins:
-            mod = imp.load_source('hook', p)
-            cache_plugins[p] = mod
-        else:
-            mod = cache_plugins[p]
+        mod = cache_plugins[root + '/' + p]
         mod.hook(*kwargs)
 
 def isPage(request):
@@ -164,16 +180,10 @@ def createUser(hit):
     super_hit['bandwith'] = 0;
     super_hit['last_access'] = meta_visit['last_time']
     super_hit['pages'] = [];
-    super_hit['robot'] = isRobot(hit);
+    super_hit['robot'] = False
     super_hit['hit_only'] = 0;
     appendHit(hit)
 
-def isRobot(hit):
-    for r in awstats_robots:
-        if r.match(hit['http_user_agent']):
-            return True
-    return False
-
 def decodeHTTPRequest(hit):
     if not 'request' in hit.keys(): return False
 
@@ -385,6 +395,8 @@ def newHit(hit):
 
     return True
 
+preloadPlugins()
+
 print '==> Analysing log'
 
 meta_visit = deserialize(META_PATH) or createEmptyMeta()