2014-11-20 16:15:57 +01:00
|
|
|
import re
|
|
|
|
|
|
|
|
|
|
from awstats_robots_data import awstats_robots
|
|
|
|
|
|
|
|
|
|
PLUGIN_CLASS = 'HTTP'
|
|
|
|
|
API_VERSION = 1
|
|
|
|
|
|
|
|
|
|
def get_plugins_infos():
|
|
|
|
|
infos = {'class' : PLUGIN_CLASS,
|
|
|
|
|
'min_version' : API_VERSION,
|
|
|
|
|
'max_version' : -1}
|
|
|
|
|
return infos
|
|
|
|
|
|
|
|
|
|
def load():
|
|
|
|
|
global awstats_robots
|
|
|
|
|
print '==> Generating robot dictionary'
|
|
|
|
|
|
|
|
|
|
awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
|
|
|
|
|
|
|
|
|
|
return True
|
2014-11-19 08:01:12 +01:00
|
|
|
|
|
|
|
|
# Basic rule to detect robots
|
|
|
|
|
|
|
|
|
|
def hook(hits):
|
|
|
|
|
for k in hits.keys():
|
|
|
|
|
super_hit = hits[k]
|
|
|
|
|
|
|
|
|
|
if super_hit['robot']: continue
|
|
|
|
|
|
|
|
|
|
isRobot = False
|
|
|
|
|
referers = 0
|
|
|
|
|
|
2014-11-20 16:15:57 +01:00
|
|
|
for r in awstats_robots:
|
|
|
|
|
if r.match(super_hit['pages'][0]['http_user_agent']):
|
|
|
|
|
super_hit['robot'] = 1
|
|
|
|
|
continue
|
|
|
|
|
|
2014-11-19 08:01:12 +01:00
|
|
|
# 1) no pages view --> robot
|
|
|
|
|
if not super_hit['viewed_pages']:
|
|
|
|
|
super_hit['robot'] = 1
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
# 2) pages without hit --> robot
|
|
|
|
|
if not super_hit['viewed_hits']:
|
|
|
|
|
super_hit['robot'] = 1
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
for hit in super_hit['pages']:
|
|
|
|
|
# 3) /robots.txt read
|
|
|
|
|
if hit['extract_request']['http_uri'] == '/robots.txt':
|
|
|
|
|
isRobot = True
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
# 4) Any referer for hits
|
|
|
|
|
if not hit['is_page'] and hit['http_referer']:
|
|
|
|
|
referers += 1
|
|
|
|
|
|
|
|
|
|
if isRobot:
|
|
|
|
|
super_hit['robot'] = 1
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
|
|
if super_hit['viewed_hits'] and not referers:
|
|
|
|
|
super_hit['robot'] = 1
|
|
|
|
|
continue
|
