Externalize plugins

plugins/pre_analysis/H001_robot.py

@@ -1,3 +1,23 @@
+import re
+
+from awstats_robots_data import awstats_robots
+
+PLUGIN_CLASS = 'HTTP'
+API_VERSION = 1
+
+def get_plugins_infos():
+    infos = {'class' : PLUGIN_CLASS,
+             'min_version' : API_VERSION,
+             'max_version' : -1}
+    return infos
+
+def load():
+    global awstats_robots
+    print '==> Generating robot dictionary'
+
+    awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
+    
+    return True
 
 # Basic rule to detect robots
 
@@ -10,6 +30,11 @@ def hook(hits):
         isRobot = False
         referers = 0
 
+        for r in awstats_robots:
+            if r.match(super_hit['pages'][0]['http_user_agent']):
+                super_hit['robot'] = 1
+                continue
+
 # 1) no pages view --> robot
         if not super_hit['viewed_pages']:
             super_hit['robot'] = 1

plugins/pre_analysis/H002_soutade.py

@@ -3,6 +3,18 @@ import re
 # Remove logo from indefero
 logo_re = re.compile(r'^.+/logo/$')
 
+PLUGIN_CLASS = 'HTTP'
+API_VERSION = 1
+
+def get_plugins_infos():
+    infos = {'class' : PLUGIN_CLASS,
+             'min_version' : API_VERSION,
+             'max_version' : -1}
+    return infos
+
+def load():
+    return True
+
 # Basic rule to detect robots
 
 def hook(hits):
@@ -13,6 +25,7 @@ def hook(hits):
 
         for p in super_hit['pages']:
             if not p['is_page']: continue
+            if int(p['status']) != 200: continue
             if logo_re.match(p['extract_request']['extract_uri']):
                 p['is_page'] = False
                 super_hit['viewed_pages'] -= 1