soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Do a lot of things

Browse Source
This commit is contained in:
Grégory Soutadé
2014-11-26 16:17:16 +01:00
parent 6505ca3ee5
commit 81b3eee552
12 changed files with 333 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
plugins/display/all_visits.py
View File

@@ -14,7 +14,7 @@ class IWLADisplayAllVisits(IPlugin):
last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True)
cur_time = self.iwla.getCurTime()
title = time.strftime('All visits %B %Y', cur_time)
title = time.strftime('All visits - %B %Y', cur_time)
filename = 'all_visits_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
@@ -22,8 +22,13 @@ class IWLADisplayAllVisits(IPlugin):
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Last seen', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
for super_hit in last_access:
address = super_hit['remote_addr']
if self.iwla.getConfValue('display_visitor_ip', False) and\
super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip'])
row = [
super_hit['remote_addr'],
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),

199
plugins/display/referers.py Normal file
View File

@@ -0,0 +1,199 @@
import time
import re
import HTMLParser
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
class IWLADisplayReferers(IPlugin):
def __init__(self, iwla):
super(IWLADisplayReferers, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
domain_name = self.iwla.getConfValue('domain_name', '')
if not domain_name:
print 'domain_name required in conf'
return False
self.own_domain_re = re.compile('.*%s.*' % (domain_name))
self.search_engines = {}
for engine in awstats_data.search_engines:
self.search_engines[engine] = {
're' : re.compile(engine, re.IGNORECASE)
}
for (engine, not_engine) in awstats_data.not_search_engines_keys.items():
if not engine in self.search_engines: continue
self.search_engines[engine]['not_search_engine'] = \
re.compile(not_engine, re.IGNORECASE)
for (engine, name) in awstats_data.search_engines_hashid.items():
if not engine in self.search_engines: continue
self.search_engines[engine]['name'] = name
for (engine, knwown_url) in awstats_data.search_engines_knwown_url.items():
engine = engin[2:-2]
if not engine in self.search_engines: continue
print knwown_url
self.search_engines[engine]['known_url'] = re.compile(known_url + '(?P<key_phrase>.+)')
self.html_parser = HTMLParser.HTMLParser()
return True
def _extractKeyPhrase(self, key_phrase_re, parameters, key_phrases):
if not parameters or not key_phrase_re: return
for p in parameters.split('&'):
groups = key_phrase_re.match(p)
if groups:
print groups.groupddict()
key_phrase = self.html_parser.unescape(groups.groupddict()['key_phrase']).lower()
if not key_phrase in key_phrases.keys():
key_phrases[key_phrase] = 1
else:
key_phrases[key_phrase] += 1
def hook(self, iwla):
stats = iwla.getCurrentVisists()
referers = {}
robots_referers = {}
search_engine_referers = {}
key_phrases = {}
for (k, super_hit) in stats.items():
for r in super_hit['requests']:
if not r['http_referer']: continue
uri = r['extract_referer']['extract_uri']
is_search_engine = False
if self.own_domain_re.match(uri): continue
for e in self.search_engines.values():
if e['re'].match(uri):
not_engine = e.get('not_search_engine', None)
# Try not engine
if not_engine and not_engine.match(uri): break
is_search_engine = True
uri = e['name']
parameters = r['extract_referer'].get('extract_parameters', None)
key_phrase_re = e.get('known_url', None)
print parameters
print key_phrase_re
self._extractKeyPhrase(key_phrase_re, parameters, key_phrases)
break
if is_search_engine:
dictionary = search_engine_referers
elif super_hit['robot']:
dictionary = robots_referers
# print '%s => %s' % (uri, super_hit['remote_ip'])
else:
dictionary = referers
if r['is_page']:
key = 'pages'
else:
key = 'hits'
if not uri in dictionary: dictionary[uri] = {'pages':0, 'hits':0}
dictionary[uri][key] += 1
top_referers = [(k, referers[k]['pages']) for k in referers.keys()]
top_referers = sorted(top_referers, key=lambda t: t[1], reverse=True)
top_robots_referers = [(k, robots_referers[k]['pages']) for k in robots_referers.keys()]
top_robots_referers = sorted(top_robots_referers, key=lambda t: t[1], reverse=True)
top_search_engine_referers = [(k, search_engine_referers[k]['pages']) for k in search_engine_referers.keys()]
top_search_engine_referers = sorted(top_search_engine_referers, key=lambda t: t[1], reverse=True)
top_key_phrases = key_phrases.items()
top_key_phrases = sorted(top_key_phrases, key=lambda t: t[1], reverse=True)
# Top referers in index
index = self.iwla.getDisplayIndex()
table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
table.appendRow(['<b>Search Engine</b>', '', ''])
for r,_ in top_search_engine_referers[:10]:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers[:10]:
row = [r, referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers[:10]:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
index.appendBlock(table)
# All referers in a file
cur_time = self.iwla.getCurTime()
title = time.strftime('Connexion from - %B %Y', cur_time)
filename = 'referers_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Connexion from', ['Origin', 'Pages', 'Hits'])
table.appendRow(['<b>Search Engine</b>', '', ''])
for r,_ in top_search_engine_referers:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL</b>', '', ''])
for r,_ in top_referers:
row = [r, referers[r]['pages'], referers[r]['hits']]
table.appendRow(row)
table.appendRow(['<b>External URL (robot)</b>', '', ''])
for r,_ in top_robots_referers:
row = [r, robots_referers[r]['pages'], robots_referers[r]['hits']]
table.appendRow(row)
page.appendBlock(table)
display = self.iwla.getDisplay()
display.addPage(page)
block = DisplayHTMLRawBlock()
block.setRawHTML('<a href=\'%s\'>All referers</a>' % (filename))
index.appendBlock(block)
# Top key phrases in index
table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
for phrase in top_key_phrases[:10]:
table.appendRow([phrase[0], phrase[1]])
index.appendBlock(table)
# All key phrases in a file
cur_time = self.iwla.getCurTime()
title = time.strftime('Key Phrases - %B %Y', cur_time)
filename = 'key_phrases_%d.html' % (cur_time.tm_mon)
path = '%d/%s' % (cur_time.tm_year, filename)
page = DisplayHTMLPage(title, path)
table = DisplayHTMLBlockTable('Top key phrases', ['Key phrase', 'Search'])
for phrase in top_key_phrases:
table.appendRow([phrase[0], phrase[1]])
page.appendBlock(table)
display.addPage(page)

7
plugins/display/top_visitors.py
View File

@@ -16,8 +16,13 @@ class IWLADisplayTopVisitors(IPlugin):
index = iwla.getDisplayIndex()
table = DisplayHTMLBlockTable('Top visitors', ['Host', 'Pages', 'Hits', 'Bandwidth', 'Last seen'])
for super_hit in stats['top_visitors']:
address = super_hit['remote_addr']
if self.iwla.getConfValue('display_visitor_ip', False) and\
super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip'])
row = [
super_hit['remote_addr'],
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),

1
plugins/post_analysis/reverse_dns.py
View File

@@ -20,6 +20,7 @@ class IWLAPostAnalysisReverseDNS(IPlugin):
try:
name, _, _ = socket.gethostbyaddr(k)
hit['remote_addr'] = name
hit['dns_name_replaced'] = True
except:
pass
finally:

2
plugins/post_analysis/top_visitors.py
View File

@@ -9,7 +9,7 @@ class IWLAPostAnalysisTopVisitors(IPlugin):
def hook(self, iwla):
hits = iwla.getValidVisitors()
stats = iwla.getMonthStats()
top_bandwidth = [(k,hits[k]['bandwidth']) for (k,v) in hits.items()]
top_bandwidth = [(k,hits[k]['bandwidth']) for k in hits.keys()]
top_bandwidth = sorted(top_bandwidth, key=lambda t: t[1], reverse=True)
stats['top_visitors'] = [hits[h[0]] for h in top_bandwidth[:10]]

23
plugins/pre_analysis/robots.py
View File

@@ -3,7 +3,7 @@ import re
from iwla import IWLA
from iplugin import IPlugin
from awstats_robots_data import awstats_robots
import awstats_data
class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):
@@ -11,9 +11,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.API_VERSION = 1
def load(self):
global awstats_robots
self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_robots)
self.awstats_robots = map(lambda (x) : re.compile(x, re.IGNORECASE), awstats_data.robots)
return True
@@ -32,13 +30,17 @@ class IWLAPreAnalysisRobots(IPlugin):
if first_page['time_decoded'].tm_mday == super_hit['last_access'].tm_mday:
for r in self.awstats_robots:
if r.match(first_page['http_user_agent']):
super_hit['robot'] = 1
continue
isRobot = True
break
if isRobot:
super_hit['robot'] = 1
continue
# 1) no pages view --> robot
if not super_hit['viewed_pages']:
super_hit['robot'] = 1
continue
# if not super_hit['viewed_pages']:
# super_hit['robot'] = 1
# continue
# 2) pages without hit --> robot
if not super_hit['viewed_hits']:
@@ -59,6 +61,7 @@ class IWLAPreAnalysisRobots(IPlugin):
super_hit['robot'] = 1
continue
if super_hit['viewed_hits'] and not referers:
if not super_hit['viewed_pages'] and \
(super_hit['viewed_hits'] and not referers):
super_hit['robot'] = 1
continue