soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Do merge

Browse Source
This commit is contained in:
Gregory Soutade 2016-02-06 14:45:09 +01:00
parent 992c3cee93
commit 12cc80208d
16 changed files with 356 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf.py
View File

@ -10,8 +10,8 @@ display_visitor_ip = True
# Hooks used # Hooks used
pre_analysis_hooks = ['page_to_hit', 'robots'] pre_analysis_hooks = ['page_to_hit', 'robots']
post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns'] post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo']
display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff'] display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff']
# Reverse DNS timeout # Reverse DNS timeout
reverse_dns_timeout = 0.2 reverse_dns_timeout = 0.2

45
display.py
View File

@ -48,10 +48,18 @@ class DisplayHTMLRaw(object):
def _build(self, f, html): def _build(self, f, html):
if html: f.write(html) if html: f.write(html)
def build(self, f): def build(self, f, filters=None):
if filters: self.filter(filters)
self._buildHTML() self._buildHTML()
self._build(f, self.html) self._build(f, self.html)
def _filter(self, function, **kwargs):
pass
def filter(self, filters):
for (args, function) in filters:
self._filter(function, **args)
def getTitle(self): def getTitle(self):
return '' return ''
@ -189,6 +197,18 @@ class DisplayHTMLBlockTable(DisplayHTMLBlock):
val = r[column] and int(r[column]) or 0 val = r[column] and int(r[column]) or 0
self.setCellValue(index, column_insertion, '%.1f%%' % (float(val*100)/float(total))) self.setCellValue(index, column_insertion, '%.1f%%' % (float(val*100)/float(total)))
def _filter(self, function, column, args):
target_col = None
for col in range(0, len(self.cols)):
if self.cols[col] == column:
target_col = col
break
if target_col is None: return
for row in self.rows:
res = function(row[target_col], **args)
if res:
row[target_col] = res
def _buildHTML(self): def _buildHTML(self):
style = u'' style = u''
if self.table_css: style = u' class="%s"' % (self.table_css) if self.table_css: style = u' class="%s"' % (self.table_css)
@ -316,10 +336,13 @@ class DisplayHTMLPage(object):
return b return b
return None return None
def getAllBlocks(self):
return self.blocks
def appendBlock(self, block): def appendBlock(self, block):
self.blocks.append(block) self.blocks.append(block)
def build(self, root, displayVersion=True): def build(self, root, displayVersion=True, filters=None):
filename = os.path.join(root, self.filename) filename = os.path.join(root, self.filename)
base = os.path.dirname(filename) base = os.path.dirname(filename)
@ -339,7 +362,7 @@ class DisplayHTMLPage(object):
f.write(u'<title>%s</title>' % (self.title)) f.write(u'<title>%s</title>' % (self.title))
f.write(u'</head><body>') f.write(u'</head><body>')
for block in self.blocks: for block in self.blocks:
block.build(f) block.build(f, filters=filters)
if displayVersion: if displayVersion:
f.write(u'<center>Generated by <a href="%s">IWLA %s</a></center>' % f.write(u'<center>Generated by <a href="%s">IWLA %s</a></center>' %
("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion())) ("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion()))
@ -349,8 +372,12 @@ class DisplayHTMLPage(object):
class DisplayHTMLBuild(object): class DisplayHTMLBuild(object):
def __init__(self, iwla): def __init__(self, iwla):
self.pages = []
self.iwla = iwla self.iwla = iwla
self.filters = []
self.clear()
def clear(self):
self.pages = []
def createPage(self, *args): def createPage(self, *args):
return DisplayHTMLPage(self.iwla, *args) return DisplayHTMLPage(self.iwla, *args)
@ -364,6 +391,9 @@ class DisplayHTMLBuild(object):
return page return page
return None return None
def getAllPages(self):
return self.pages
def addPage(self, page): def addPage(self, page):
self.pages.append(page) self.pages.append(page)
@ -378,7 +408,11 @@ class DisplayHTMLBuild(object):
os.symlink(target, link_name) os.symlink(target, link_name)
for page in self.pages: for page in self.pages:
page.build(root) page.build(root, filters=self.filters)
def addColumnFilter(self, column, function, args):
self.filters.append(({'column':column, 'args':args}, function))
# #
# Global functions # Global functions
@ -417,4 +451,3 @@ def createCurTitle(iwla, title):
if domain_name: if domain_name:
title += u' - %s' % (domain_name) title += u' - %s' % (domain_name)
return title return title

4
iwla.py
View File

@ -188,7 +188,7 @@ class IWLA(object):
def getMonthStats(self): def getMonthStats(self):
return self.current_analysis['month_stats'] return self.current_analysis['month_stats']
def getCurrentVisists(self): def getCurrentVisits(self):
return self.current_analysis['visits'] return self.current_analysis['visits']
def getValidVisitors(self): def getValidVisitors(self):
@ -230,7 +230,7 @@ class IWLA(object):
return self.meta_infos return self.meta_infos
def _clearDisplay(self): def _clearDisplay(self):
self.display = DisplayHTMLBuild(self) self.display.clear()
return self.display return self.display
def getDBFilename(self, time): def getDBFilename(self, time):

2
plugins/display/feeds.py
View File

@ -62,7 +62,7 @@ class IWLADisplayFeeds(IPlugin):
from plugins.post_analysis.feeds import IWLAPostAnalysisFeeds from plugins.post_analysis.feeds import IWLAPostAnalysisFeeds
display = self.iwla.getDisplay() display = self.iwla.getDisplay()
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
nb_feeds_parsers = 0 nb_feeds_parsers = 0
# All in a page # All in a page

120
plugins/display/ip_to_geo.py Normal file
View File

@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
import re
from iwla import IWLA
from iplugin import IPlugin
from display import *
import awstats_data
"""
Display hook
Add geo statistics
Plugin requirements :
post_analysis/ip_to_geo
Conf values needed :
create_geo_page*
Output files :
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopGeo(IPlugin):
def __init__(self, iwla):
super(IWLADisplayTopGeo, self).__init__(iwla)
self.API_VERSION = 1
self.requires = ['IWLAPostAnalysisIPToGeo']
def load(self):
self.icon_path = self.iwla.getConfValue('icon_path', '/')
self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True)
display = self.iwla.getDisplay()
display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self})
return True
@staticmethod # Needed to have unbound methd
def FlagFilter(host, self):
cc = None
host_name = host.split(' ')[0] # hostname or ip
if host_name in self.valid_visitors.keys():
cc = self.valid_visitors[host_name]['country_code']
else:
for visitor in self.valid_visitors.values():
if visitor['remote_addr'] == host_name:
cc = visitor['country_code']
break
if not cc or cc == 'ip': return None
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
return '%s %s' % (icon ,host)
def hook(self):
display = self.iwla.getDisplay()
geo = self.iwla.getMonthStats()['geo']
geo = sorted(geo.items(), key=lambda t: t[1], reverse=True)
self.valid_visitors = self.iwla.getValidVisitors()
# All in a page
if self.create_geo_page:
title = createCurTitle(self.iwla, u'All Coutries')
filename = 'geo.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (cc, visitors) in geo:
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
table.appendRow([icon, cc, visitors])
table.computeRatio(2)
page.appendBlock(table)
display.addPage(page)
# Countries in index
title = self.iwla._(u'Countries')
if self.create_geo_page:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
title = '%s - %s' % (title, link)
index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')])
table.setColsCSSClass(['', '', 'iwla_hit'])
for (cc, visitors) in geo[:10]:
icon = '<img src="/%s/flags/%s.png"/>' % (self.icon_path, cc)
table.appendRow([icon, cc, visitors])
table.computeRatio(2)
index.appendBlock(table)

4
plugins/display/track_users.py
View File

@ -64,7 +64,7 @@ class IWLADisplayTrackUsers(IPlugin):
def hook(self): def hook(self):
display = self.iwla.getDisplay() display = self.iwla.getDisplay()
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
stats = {} stats = {}
# All in a page # All in a page
@ -110,7 +110,7 @@ class IWLADisplayTrackUsers(IPlugin):
index = self.iwla.getDisplayIndex() index = self.iwla.getDisplayIndex()
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'IP'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')]) table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Host'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', '', 'iwla_page', 'iwla_hit']) table.setColsCSSClass(['', '', 'iwla_page', 'iwla_hit'])
for ip in self.tracked_ip: for ip in self.tracked_ip:
if not ip in hits.keys(): continue if not ip in hits.keys(): continue

2
plugins/post_analysis/feeds.py
View File

@ -85,7 +85,7 @@ class IWLAPostAnalysisFeeds(IPlugin):
hit['feed_parser'] = isFeedParser hit['feed_parser'] = isFeedParser
def hook(self): def hook(self):
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
one_hit_only = {} one_hit_only = {}
for hit in hits.values(): for hit in hits.values():
isFeedParser = hit.get('feed_parser', None) isFeedParser = hit.get('feed_parser', None)

2
plugins/post_analysis/hours_stats.py
View File

@ -62,7 +62,7 @@ class IWLAPostAnalysisHoursStats(IPlugin):
self.API_VERSION = 1 self.API_VERSION = 1
def hook(self): def hook(self):
stats = self.iwla.getCurrentVisists() stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats() month_stats = self.iwla.getMonthStats()
hours_stats = month_stats.get('hours_stats', {}) hours_stats = month_stats.get('hours_stats', {})

92
plugins/post_analysis/ip_to_geo.py Normal file
View File

@ -0,0 +1,92 @@
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2016
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from iptogeo import IPToGeo
"""
Post analysis hook
Get country code from IP address
Plugin requirements :
None
Conf values needed :
iptogeo_remote_addr*
iptogeo_remote_port*
Output files :
None
Statistics creation :
geo =>
country_code => count
None
Statistics update :
valid_visitors:
country_code
Statistics deletion :
None
"""
class IWLAPostAnalysisIPToGeo(IPlugin):
def __init__(self, iwla):
super(IWLAPostAnalysisIPToGeo, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
remote_addr = self.iwla.getConfValue('iptogeo_remote_addr',
None)
remote_port = self.iwla.getConfValue('iptogeo_remote_port',
None)
args = {}
if remote_addr: args['remote_addr'] = remote_addr
if remote_port: args['remote_port'] = remote_port
self.iptogeo = IPToGeo(**args)
return True
def hook(self):
visitors = self.iwla.getValidVisitors()
month_stats = self.iwla.getMonthStats()
geo = month_stats.get('geo', {})
for (ip, visitor) in visitors.items():
if visitor.get('country_code', False): continue
try:
(_, cc) = self.iptogeo.ip_to_geo(ip)
cc = cc and cc or 'ip'
visitor['country_code'] = cc
if cc in geo.keys():
geo[cc] += 1
else:
geo[cc] = 1
except Exception, e:
print e
month_stats['geo'] = geo

89
plugins/post_analysis/iptogeo.py Normal file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import socket
import struct
class IPToGeoException(Exception):
pass
class IPToGeo(object):
MAGIC = 0x179E08EF
VERSION = 1
REQ = 1
RESP = 0
IPV4 = 4
IPV6 = 16
IP_NOT_FOUND = 6
PACKET_SIZE = 32
ERRORS = {1 : 'Bad magic',
2 : 'Bad version',
3 : 'Bad request field' ,
4 : 'Bad IP version',
5 : 'Unsupported IP version',
6 : 'IP not found'}
def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None):
self._remote_addr = remote_addr
self._remote_port = remote_port
self._timeout = timeout
self._create_socket()
def _create_socket(self):
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if not self._timeout is None:
self._socket.settimeout(self._timeout)
self._socket.connect((self._remote_addr, self._remote_port))
def _create_request(self, ip):
packet = ''
packet += struct.pack('<IBBBBI', IPToGeo.MAGIC, IPToGeo.VERSION, IPToGeo.REQ,
0, #err
IPToGeo.IPV4, # ip type
0) # flags
packet += struct.pack('<BBBB', ip[0], ip[1], ip[2], ip[3]) # ipv4
packet += struct.pack('<III', 0, 0, 0) # ipv6
packet += struct.pack('<I', 0) # country code
return packet
def _check_request(self, packet):
(magic, version, req, err, ip_type, flags, ipv4, ipv6b, ipv6c, ipv6d) = struct.unpack_from('<IBBBBIIIII', packet, 0)
if magic != IPToGeo.MAGIC:
raise IPToGeoException('Invalid magic %08x' % (magic))
if err == IPToGeo.IP_NOT_FOUND: return (ipv4, None) # IP not found
if err != 0:
raise IPToGeoException(IPToGeo.ERRORS[err])
(cc0, cc1, cc2, cc3) = struct.unpack_from('BBBB', packet, 7*4)
return (ipv4, '%c%c%c%c' % (cc0, cc1, cc2, cc3))
def ip_to_geo(self, ip):
splitted_ip = [int(a) for a in ip.split('.')]
packet = self._create_request(splitted_ip)
try:
self._socket.send(packet)
except IOError, e:
# Give another chance (we may have been disconnected due to timeout)
self._create_socket()
self._socket.send(packet)
packet = self._socket.recv(IPToGeo.PACKET_SIZE)
if not packet:
raise IPToGeoException('Error, empty packet')
(ip, country_code) = self._check_request(packet)
if country_code:
# convert to string
country_code = '%c%c' % (country_code[0], country_code[1])
return (ip, country_code)
def close(self):
self._socket.close()

2
plugins/post_analysis/referers.py
View File

@ -120,7 +120,7 @@ class IWLAPostAnalysisReferers(IPlugin):
break break
def hook(self): def hook(self):
stats = self.iwla.getCurrentVisists() stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats() month_stats = self.iwla.getMonthStats()
referers = month_stats.get('referers', {}) referers = month_stats.get('referers', {})

2
plugins/post_analysis/reverse_dns.py
View File

@ -64,7 +64,7 @@ class IWLAPostAnalysisReverseDNS(IPlugin):
return True return True
def hook(self): def hook(self):
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
for (k, hit) in hits.items(): for (k, hit) in hits.items():
if hit.get('dns_analysed', False): continue if hit.get('dns_analysed', False): continue
if not hit['feed_parser'] and\ if not hit['feed_parser'] and\

2
plugins/post_analysis/top_hits.py
View File

@ -53,7 +53,7 @@ class IWLAPostAnalysisTopHits(IPlugin):
self.API_VERSION = 1 self.API_VERSION = 1
def hook(self): def hook(self):
stats = self.iwla.getCurrentVisists() stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats() month_stats = self.iwla.getMonthStats()
top_hits = month_stats.get('top_hits', {}) top_hits = month_stats.get('top_hits', {})

2
plugins/post_analysis/top_pages.py
View File

@ -59,7 +59,7 @@ class IWLAPostAnalysisTopPages(IPlugin):
return True return True
def hook(self): def hook(self):
stats = self.iwla.getCurrentVisists() stats = self.iwla.getCurrentVisits()
month_stats = self.iwla.getMonthStats() month_stats = self.iwla.getMonthStats()
top_pages = month_stats.get('top_pages', {}) top_pages = month_stats.get('top_pages', {})

2
plugins/pre_analysis/page_to_hit.py
View File

@ -69,7 +69,7 @@ class IWLAPreAnalysisPageToHit(IPlugin):
return True return True
def hook(self): def hook(self):
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items(): for (k, super_hit) in hits.items():
if super_hit['robot']: continue if super_hit['robot']: continue

2
plugins/pre_analysis/robots.py
View File

@ -76,7 +76,7 @@ class IWLAPreAnalysisRobots(IPlugin):
# Basic rule to detect robots # Basic rule to detect robots
def hook(self): def hook(self):
hits = self.iwla.getCurrentVisists() hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items(): for (k, super_hit) in hits.items():
if super_hit['robot']: if super_hit['robot']:
self.logger.debug('%s is a robot' % (k)) self.logger.debug('%s is a robot' % (k))