431 lines
14 KiB
Python
Executable File
431 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import os
|
|
import re
|
|
import time
|
|
import pickle
|
|
import gzip
|
|
import importlib
|
|
|
|
import default_conf as conf
|
|
import conf as _
|
|
conf.__dict__.update(_.__dict__)
|
|
del _
|
|
|
|
from iplugin import *
|
|
from display import *
|
|
|
|
class IWLA(object):
|
|
|
|
ANALYSIS_CLASS = 'HTTP'
|
|
API_VERSION = 1
|
|
|
|
def __init__(self):
|
|
print '==> Start'
|
|
|
|
self.meta_infos = {}
|
|
self.analyse_started = False
|
|
self.current_analysis = {}
|
|
self.cache_plugins = {}
|
|
self.display = DisplayHTMLBuild()
|
|
self.valid_visitors = None
|
|
|
|
self.log_format_extracted = re.sub(r'([^\$\w])', r'\\\g<1>', conf.log_format)
|
|
self.log_format_extracted = re.sub(r'\$(\w+)', '(?P<\g<1>>.+)', self.log_format_extracted)
|
|
self.http_request_extracted = re.compile(r'(?P<http_method>\S+) (?P<http_uri>\S+) (?P<http_version>\S+)')
|
|
self.log_re = re.compile(self.log_format_extracted)
|
|
self.uri_re = re.compile(r'(?P<extract_uri>[^\?]+)(\?(?P<extract_parameters>.+))?')
|
|
self.plugins = [(conf.PRE_HOOK_DIRECTORY , conf.pre_analysis_hooks),
|
|
(conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks),
|
|
(conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
|
|
|
|
def getConfValue(self, key, default=None):
|
|
if not key in dir(conf):
|
|
return default
|
|
else:
|
|
return conf.__dict__[key]
|
|
|
|
def _clearVisits(self):
|
|
self.current_analysis = {
|
|
'days_stats' : {},
|
|
'month_stats' : {},
|
|
'visits' : {}
|
|
}
|
|
self.valid_visitors = None
|
|
return self.current_analysis
|
|
|
|
def getDaysStats(self):
|
|
return self.current_analysis['days_stats']
|
|
|
|
def getMonthStats(self):
|
|
return self.current_analysis['month_stats']
|
|
|
|
def getCurrentVisists(self):
|
|
return self.current_analysis['visits']
|
|
|
|
def getValidVisitors(self):
|
|
return self.valid_visitors
|
|
|
|
def getDisplay(self):
|
|
return self.display
|
|
|
|
def getCurTime(self):
|
|
return self.meta_infos['last_time']
|
|
|
|
def getStartAnalysisTime(self):
|
|
return self.meta_infos['start_analysis_time']
|
|
|
|
def isValidForCurrentAnalysis(self, request):
|
|
cur_time = self.meta_infos['start_analysis_time']
|
|
return (time.mktime(cur_time) <= time.mktime(request['time_decoded']))
|
|
|
|
def _clearMeta(self):
|
|
self.meta_infos = {
|
|
'last_time' : None
|
|
}
|
|
return self.meta_infos
|
|
|
|
def _clearDisplay(self):
|
|
self.display = DisplayHTMLBuild()
|
|
return self.display
|
|
|
|
def getDBFilename(self, time):
|
|
return (conf.DB_ROOT + '%d/%d_%s') % (time.tm_year, time.tm_mon, conf.DB_FILENAME)
|
|
|
|
def _serialize(self, obj, filename):
|
|
base = os.path.dirname(filename)
|
|
if not os.path.exists(base):
|
|
os.makedirs(base)
|
|
|
|
# TODO : remove return
|
|
#return
|
|
|
|
with open(filename + '.tmp', 'wb+') as f:
|
|
pickle.dump(obj, f)
|
|
f.seek(0)
|
|
with gzip.open(filename, 'w') as fzip:
|
|
fzip.write(f.read())
|
|
os.remove(filename + '.tmp')
|
|
|
|
def _deserialize(self, filename):
|
|
if not os.path.exists(filename):
|
|
return None
|
|
|
|
with gzip.open(filename, 'r') as f:
|
|
return pickle.load(f)
|
|
return None
|
|
|
|
def _callPlugins(self, target_root, *args):
|
|
print '==> Call plugins (%s)' % target_root
|
|
for (root, plugins) in self.plugins:
|
|
if root != target_root: continue
|
|
for p in plugins:
|
|
mod = self.cache_plugins.get(root + '.' + p, None)
|
|
if mod:
|
|
print '\t%s' % (p)
|
|
mod.hook(*args)
|
|
|
|
def isPage(self, request):
|
|
for e in conf.pages_extensions:
|
|
if request.endswith(e):
|
|
return True
|
|
|
|
return False
|
|
|
|
def _appendHit(self, hit):
|
|
remote_addr = hit['remote_addr']
|
|
|
|
if not remote_addr in self.current_analysis['visits'].keys():
|
|
self._createVisitor(hit)
|
|
return
|
|
|
|
super_hit = self.current_analysis['visits'][remote_addr]
|
|
super_hit['requests'].append(hit)
|
|
super_hit['bandwidth'] += int(hit['body_bytes_sent'])
|
|
super_hit['last_access'] = self.meta_infos['last_time']
|
|
|
|
request = hit['extract_request']
|
|
|
|
uri = request.get('extract_uri', request['http_uri'])
|
|
|
|
hit['is_page'] = self.isPage(uri)
|
|
|
|
status = int(hit['status'])
|
|
if status not in conf.viewed_http_codes:
|
|
return
|
|
|
|
if super_hit['robot'] or\
|
|
not status in conf.viewed_http_codes:
|
|
page_key = 'not_viewed_pages'
|
|
hit_key = 'not_viewed_hits'
|
|
else:
|
|
page_key = 'viewed_pages'
|
|
hit_key = 'viewed_hits'
|
|
|
|
if hit['is_page']:
|
|
super_hit[page_key] += 1
|
|
else:
|
|
super_hit[hit_key] += 1
|
|
|
|
def _createVisitor(self, hit):
|
|
super_hit = self.current_analysis['visits'][hit['remote_addr']] = {}
|
|
super_hit['remote_addr'] = hit['remote_addr']
|
|
super_hit['remote_ip'] = hit['remote_addr']
|
|
super_hit['viewed_pages'] = 0
|
|
super_hit['viewed_hits'] = 0
|
|
super_hit['not_viewed_pages'] = 0
|
|
super_hit['not_viewed_hits'] = 0
|
|
super_hit['bandwidth'] = 0
|
|
super_hit['last_access'] = self.meta_infos['last_time']
|
|
super_hit['requests'] = []
|
|
super_hit['robot'] = False
|
|
super_hit['hit_only'] = 0
|
|
self._appendHit(hit)
|
|
|
|
def _decodeHTTPRequest(self, hit):
|
|
if not 'request' in hit.keys(): return False
|
|
|
|
groups = self.http_request_extracted.match(hit['request'])
|
|
|
|
if groups:
|
|
hit['extract_request'] = groups.groupdict()
|
|
uri_groups = self.uri_re.match(hit['extract_request']['http_uri'])
|
|
if uri_groups:
|
|
d = uri_groups.groupdict()
|
|
hit['extract_request']['extract_uri'] = d['extract_uri']
|
|
if 'extract_parameters' in d.keys():
|
|
hit['extract_request']['extract_parameters'] = d['extract_parameters']
|
|
else:
|
|
print "Bad request extraction " + hit['request']
|
|
return False
|
|
|
|
if hit['http_referer']:
|
|
referer_groups = self.uri_re.match(hit['http_referer'])
|
|
if referer_groups:
|
|
hit['extract_referer'] = referer_groups.groupdict()
|
|
return True
|
|
|
|
def _decodeTime(self, hit):
|
|
hit['time_decoded'] = time.strptime(hit['time_local'], conf.time_format)
|
|
return hit['time_decoded']
|
|
|
|
def getDisplayIndex(self):
|
|
cur_time = self.meta_infos['last_time']
|
|
filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
|
|
|
|
return self.display.getPage(filename)
|
|
|
|
def _generateDisplayDaysStat(self):
|
|
cur_time = self.meta_infos['last_time']
|
|
title = 'Stats %d/%d' % (cur_time.tm_mon, cur_time.tm_year)
|
|
filename = '%d/index_%d.html' % (cur_time.tm_year, cur_time.tm_mon)
|
|
print '==> Generate display (%s)' % (filename)
|
|
page = DisplayHTMLPage(title, filename)
|
|
|
|
days = DisplayHTMLBlockTable('By day', ['Day', 'Visits', 'Pages', 'Hits', 'Bandwidth', 'Not viewed Bandwidth'])
|
|
|
|
keys = self.current_analysis['days_stats'].keys()
|
|
keys.sort()
|
|
nb_visits = 0
|
|
for k in keys:
|
|
stats = self.current_analysis['days_stats'][k]
|
|
row = [k, stats['nb_visitors'], stats['viewed_pages'], stats['viewed_hits'],
|
|
bytesToStr(stats['viewed_bandwidth']), bytesToStr(stats['not_viewed_bandwidth'])]
|
|
days.appendRow(row)
|
|
nb_visits += stats['nb_visitors']
|
|
|
|
stats = self.current_analysis['month_stats']
|
|
|
|
nb_days = len(keys)
|
|
row = [0, nb_visits, stats['viewed_pages'], stats['viewed_hits'], stats['viewed_bandwidth'], stats['not_viewed_bandwidth']]
|
|
if nb_days:
|
|
average_row = map(lambda(v): int(v/nb_days), row)
|
|
else:
|
|
average_row = map(lambda(v): 0, row)
|
|
|
|
average_row[0] = 'Average'
|
|
average_row[4] = bytesToStr(average_row[4])
|
|
average_row[5] = bytesToStr(average_row[5])
|
|
days.appendRow(average_row)
|
|
|
|
row[0] = 'Total'
|
|
row[4] = bytesToStr(row[4])
|
|
row[5] = bytesToStr(row[5])
|
|
days.appendRow(row)
|
|
page.appendBlock(days)
|
|
self.display.addPage(page)
|
|
|
|
def _generateDisplay(self):
|
|
self._generateDisplayDaysStat()
|
|
self._callPlugins(conf.DISPLAY_HOOK_DIRECTORY)
|
|
self.display.build(conf.DISPLAY_ROOT)
|
|
|
|
def _generateStats(self, visits):
|
|
stats = {}
|
|
stats['viewed_bandwidth'] = 0
|
|
stats['not_viewed_bandwidth'] = 0
|
|
stats['viewed_pages'] = 0
|
|
stats['viewed_hits'] = 0
|
|
#stats['requests'] = set()
|
|
stats['nb_visitors'] = 0
|
|
|
|
for (k, super_hit) in visits.items():
|
|
if super_hit['robot']:
|
|
stats['not_viewed_bandwidth'] += super_hit['bandwidth']
|
|
continue
|
|
|
|
#print "[%s] =>\t%d/%d" % (k, super_hit['viewed_pages'], super_hit['viewed_hits'])
|
|
|
|
if conf.count_hit_only_visitors or\
|
|
super_hit['viewed_pages']:
|
|
stats['nb_visitors'] += 1
|
|
stats['viewed_bandwidth'] += super_hit['bandwidth']
|
|
stats['viewed_pages'] += super_hit['viewed_pages']
|
|
stats['viewed_hits'] += super_hit['viewed_hits']
|
|
|
|
# for p in super_hit['requests']:
|
|
# if not p['is_page']: continue
|
|
# req = p['extract_request']
|
|
# stats['requests'].add(req['extract_uri'])
|
|
|
|
return stats
|
|
|
|
def _generateMonthStats(self):
|
|
self._clearDisplay()
|
|
|
|
visits = self.current_analysis['visits']
|
|
|
|
stats = self._generateStats(visits)
|
|
|
|
cur_time = self.meta_infos['last_time']
|
|
print "== Stats for %d/%d ==" % (cur_time.tm_year, cur_time.tm_mon)
|
|
print stats
|
|
|
|
self.current_analysis['month_stats'] = stats
|
|
|
|
self.valid_visitors = {}
|
|
for (k,v) in visits.items():
|
|
if v['robot']: continue
|
|
if conf.count_hit_only_visitors and\
|
|
(not v['viewed_pages']):
|
|
continue
|
|
self.valid_visitors[k] = v
|
|
|
|
self._callPlugins(conf.POST_HOOK_DIRECTORY)
|
|
|
|
path = self.getDBFilename(cur_time)
|
|
if os.path.exists(path):
|
|
os.remove(path)
|
|
|
|
print "==> Serialize to %s" % path
|
|
|
|
self._serialize(self.current_analysis, path)
|
|
|
|
self._generateDisplay()
|
|
|
|
def _generateDayStats(self):
|
|
visits = self.current_analysis['visits']
|
|
|
|
self._callPlugins(conf.PRE_HOOK_DIRECTORY)
|
|
|
|
stats = self._generateStats(visits)
|
|
|
|
cur_time = self.meta_infos['last_time']
|
|
print "== Stats for %d/%d/%d ==" % (cur_time.tm_year, cur_time.tm_mon, cur_time.tm_mday)
|
|
|
|
if cur_time.tm_mday > 1:
|
|
last_day = cur_time.tm_mday - 1
|
|
while last_day:
|
|
if last_day in self.current_analysis['days_stats'].keys():
|
|
break
|
|
last_day -= 1
|
|
if last_day:
|
|
for k in stats.keys():
|
|
stats[k] -= self.current_analysis['days_stats'][last_day][k]
|
|
stats['nb_visitors'] = 0
|
|
for (k,v) in visits.items():
|
|
if v['robot']: continue
|
|
if conf.count_hit_only_visitors and\
|
|
(not v['viewed_pages']):
|
|
continue
|
|
if v['last_access'].tm_mday == cur_time.tm_mday:
|
|
stats['nb_visitors'] += 1
|
|
print stats
|
|
|
|
self.current_analysis['days_stats'][cur_time.tm_mday] = stats
|
|
|
|
def _newHit(self, hit):
|
|
t = self._decodeTime(hit)
|
|
|
|
cur_time = self.meta_infos['last_time']
|
|
|
|
if cur_time == None:
|
|
self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
|
|
self.analyse_started = True
|
|
else:
|
|
if not self.analyse_started:
|
|
if not self.isValidForCurrentAnalysis(hit):
|
|
return False
|
|
else:
|
|
self.analyse_started = True
|
|
if cur_time.tm_mon != t.tm_mon:
|
|
self._generateMonthStats()
|
|
self.current_analysis = self._deserialize(self.getDBFilename(t)) or self._clearVisits()
|
|
elif cur_time.tm_mday != t.tm_mday:
|
|
self._generateDayStats()
|
|
|
|
self.meta_infos['last_time'] = t
|
|
|
|
if not self.meta_infos['start_analysis_time']:
|
|
self.meta_infos['start_analysis_time'] = t
|
|
|
|
if not self._decodeHTTPRequest(hit): return False
|
|
|
|
for k in hit.keys():
|
|
if hit[k] == '-': hit[k] = ''
|
|
|
|
self._appendHit(hit)
|
|
|
|
return True
|
|
|
|
def start(self):
|
|
print '==> Load previous database'
|
|
|
|
self.meta_infos = self._deserialize(conf.META_PATH) or self._clearMeta()
|
|
if self.meta_infos['last_time']:
|
|
self.current_analysis = self._deserialize(self.getDBFilename(self.meta_infos['last_time'])) or self._clearVisits()
|
|
else:
|
|
self._clearVisits()
|
|
|
|
self.meta_infos['start_analysis_time'] = None
|
|
|
|
self.cache_plugins = preloadPlugins(self.plugins, self)
|
|
|
|
print '==> Analysing log'
|
|
|
|
with open(conf.analyzed_filename) as f:
|
|
for l in f:
|
|
# print "line " + l
|
|
|
|
groups = self.log_re.match(l)
|
|
|
|
if groups:
|
|
if not self._newHit(groups.groupdict()):
|
|
break
|
|
else:
|
|
print "No match for " + l
|
|
#break
|
|
|
|
if self.analyse_started:
|
|
self._generateDayStats()
|
|
self._generateMonthStats()
|
|
del self.meta_infos['start_analysis_time']
|
|
self._serialize(self.meta_infos, conf.META_PATH)
|
|
else:
|
|
print '==> Analyse not started : nothing to do'
|
|
self._generateMonthStats()
|
|
|
|
if __name__ == '__main__':
|
|
iwla = IWLA()
|
|
iwla.start()
|