diff --git a/conf.py b/conf.py index e06aa5b..330196b 100644 --- a/conf.py +++ b/conf.py @@ -10,8 +10,8 @@ display_visitor_ip = True # Hooks used pre_analysis_hooks = ['page_to_hit', 'robots'] -post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns'] -display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff'] +post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo'] +display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff'] # Reverse DNS timeout reverse_dns_timeout = 0.2 diff --git a/display.py b/display.py index 6be75df..b5a5bc0 100644 --- a/display.py +++ b/display.py @@ -48,10 +48,18 @@ class DisplayHTMLRaw(object): def _build(self, f, html): if html: f.write(html) - def build(self, f): + def build(self, f, filters=None): + if filters: self.filter(filters) self._buildHTML() self._build(f, self.html) + def _filter(self, function, **kwargs): + pass + + def filter(self, filters): + for (args, function) in filters: + self._filter(function, **args) + def getTitle(self): return '' @@ -189,6 +197,18 @@ class DisplayHTMLBlockTable(DisplayHTMLBlock): val = r[column] and int(r[column]) or 0 self.setCellValue(index, column_insertion, '%.1f%%' % (float(val*100)/float(total))) + def _filter(self, function, column, args): + target_col = None + for col in range(0, len(self.cols)): + if self.cols[col] == column: + target_col = col + break + if target_col is None: return + for row in self.rows: + res = function(row[target_col], **args) + if res: + row[target_col] = res + def _buildHTML(self): style = u'' if self.table_css: style = u' class="%s"' % (self.table_css) @@ -315,11 +335,14 @@ class DisplayHTMLPage(object): if title == b.getTitle(): return b return None - + + def getAllBlocks(self): + return self.blocks + def appendBlock(self, block): self.blocks.append(block) - def build(self, root, displayVersion=True): + def build(self, root, displayVersion=True, filters=None): filename = os.path.join(root, self.filename) base = os.path.dirname(filename) @@ -339,7 +362,7 @@ class DisplayHTMLPage(object): f.write(u'%s' % (self.title)) f.write(u'') for block in self.blocks: - block.build(f) + block.build(f, filters=filters) if displayVersion: f.write(u'
Generated by IWLA %s
' % ("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion())) @@ -349,8 +372,12 @@ class DisplayHTMLPage(object): class DisplayHTMLBuild(object): def __init__(self, iwla): - self.pages = [] self.iwla = iwla + self.filters = [] + self.clear() + + def clear(self): + self.pages = [] def createPage(self, *args): return DisplayHTMLPage(self.iwla, *args) @@ -364,6 +391,9 @@ class DisplayHTMLBuild(object): return page return None + def getAllPages(self): + return self.pages + def addPage(self, page): self.pages.append(page) @@ -378,7 +408,11 @@ class DisplayHTMLBuild(object): os.symlink(target, link_name) for page in self.pages: - page.build(root) + page.build(root, filters=self.filters) + + def addColumnFilter(self, column, function, args): + self.filters.append(({'column':column, 'args':args}, function)) + # # Global functions @@ -417,4 +451,3 @@ def createCurTitle(iwla, title): if domain_name: title += u' - %s' % (domain_name) return title - diff --git a/iwla.py b/iwla.py index 1e24f2e..480d2f1 100755 --- a/iwla.py +++ b/iwla.py @@ -188,7 +188,7 @@ class IWLA(object): def getMonthStats(self): return self.current_analysis['month_stats'] - def getCurrentVisists(self): + def getCurrentVisits(self): return self.current_analysis['visits'] def getValidVisitors(self): @@ -230,7 +230,7 @@ class IWLA(object): return self.meta_infos def _clearDisplay(self): - self.display = DisplayHTMLBuild(self) + self.display.clear() return self.display def getDBFilename(self, time): diff --git a/plugins/display/feeds.py b/plugins/display/feeds.py index bcd7194..ca5de4e 100644 --- a/plugins/display/feeds.py +++ b/plugins/display/feeds.py @@ -62,7 +62,7 @@ class IWLADisplayFeeds(IPlugin): from plugins.post_analysis.feeds import IWLAPostAnalysisFeeds display = self.iwla.getDisplay() - hits = self.iwla.getCurrentVisists() + hits = self.iwla.getCurrentVisits() nb_feeds_parsers = 0 # All in a page diff --git a/plugins/display/ip_to_geo.py b/plugins/display/ip_to_geo.py new file mode 100644 index 0000000..c0127ac --- /dev/null +++ b/plugins/display/ip_to_geo.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2015 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +import re + +from iwla import IWLA +from iplugin import IPlugin +from display import * + +import awstats_data + +""" +Display hook + +Add geo statistics + +Plugin requirements : + post_analysis/ip_to_geo + +Conf values needed : + create_geo_page* + +Output files : + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" + +class IWLADisplayTopGeo(IPlugin): + def __init__(self, iwla): + super(IWLADisplayTopGeo, self).__init__(iwla) + self.API_VERSION = 1 + self.requires = ['IWLAPostAnalysisIPToGeo'] + + def load(self): + self.icon_path = self.iwla.getConfValue('icon_path', '/') + self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True) + + display = self.iwla.getDisplay() + display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self}) + + return True + + @staticmethod # Needed to have unbound methd + def FlagFilter(host, self): + cc = None + host_name = host.split(' ')[0] # hostname or ip + if host_name in self.valid_visitors.keys(): + cc = self.valid_visitors[host_name]['country_code'] + else: + for visitor in self.valid_visitors.values(): + if visitor['remote_addr'] == host_name: + cc = visitor['country_code'] + break + if not cc or cc == 'ip': return None + icon = '' % (self.icon_path, cc) + return '%s %s' % (icon ,host) + + def hook(self): + display = self.iwla.getDisplay() + geo = self.iwla.getMonthStats()['geo'] + geo = sorted(geo.items(), key=lambda t: t[1], reverse=True) + self.valid_visitors = self.iwla.getValidVisitors() + + # All in a page + if self.create_geo_page: + title = createCurTitle(self.iwla, u'All Coutries') + filename = 'geo.html' + path = self.iwla.getCurDisplayPath(filename) + + page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) + table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (cc, visitors) in geo: + icon = '' % (self.icon_path, cc) + table.appendRow([icon, cc, visitors]) + table.computeRatio(2) + page.appendBlock(table) + + display.addPage(page) + + # Countries in index + title = self.iwla._(u'Countries') + if self.create_geo_page: + link = '%s' % (filename, self.iwla._(u'Details')) + title = '%s - %s' % (title, link) + + index = self.iwla.getDisplayIndex() + + table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (cc, visitors) in geo[:10]: + icon = '' % (self.icon_path, cc) + table.appendRow([icon, cc, visitors]) + table.computeRatio(2) + index.appendBlock(table) diff --git a/plugins/display/track_users.py b/plugins/display/track_users.py index 53b7b9d..c653e4a 100644 --- a/plugins/display/track_users.py +++ b/plugins/display/track_users.py @@ -64,7 +64,7 @@ class IWLADisplayTrackUsers(IPlugin): def hook(self): display = self.iwla.getDisplay() - hits = self.iwla.getCurrentVisists() + hits = self.iwla.getCurrentVisits() stats = {} # All in a page @@ -110,7 +110,7 @@ class IWLADisplayTrackUsers(IPlugin): index = self.iwla.getDisplayIndex() - table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'IP'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')]) + table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Host'), self.iwla._(u'Last Access'), self.iwla._(u'Pages'), self.iwla._(u'Hits')]) table.setColsCSSClass(['', '', 'iwla_page', 'iwla_hit']) for ip in self.tracked_ip: if not ip in hits.keys(): continue diff --git a/plugins/post_analysis/feeds.py b/plugins/post_analysis/feeds.py index 8476881..570649f 100644 --- a/plugins/post_analysis/feeds.py +++ b/plugins/post_analysis/feeds.py @@ -85,7 +85,7 @@ class IWLAPostAnalysisFeeds(IPlugin): hit['feed_parser'] = isFeedParser def hook(self): - hits = self.iwla.getCurrentVisists() + hits = self.iwla.getCurrentVisits() one_hit_only = {} for hit in hits.values(): isFeedParser = hit.get('feed_parser', None) diff --git a/plugins/post_analysis/hours_stats.py b/plugins/post_analysis/hours_stats.py index a45dbdd..4ec9014 100644 --- a/plugins/post_analysis/hours_stats.py +++ b/plugins/post_analysis/hours_stats.py @@ -62,7 +62,7 @@ class IWLAPostAnalysisHoursStats(IPlugin): self.API_VERSION = 1 def hook(self): - stats = self.iwla.getCurrentVisists() + stats = self.iwla.getCurrentVisits() month_stats = self.iwla.getMonthStats() hours_stats = month_stats.get('hours_stats', {}) diff --git a/plugins/post_analysis/ip_to_geo.py b/plugins/post_analysis/ip_to_geo.py new file mode 100644 index 0000000..a15e016 --- /dev/null +++ b/plugins/post_analysis/ip_to_geo.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2016 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +from iwla import IWLA +from iplugin import IPlugin + +from iptogeo import IPToGeo + +""" +Post analysis hook + +Get country code from IP address + +Plugin requirements : + None + +Conf values needed : + iptogeo_remote_addr* + iptogeo_remote_port* + +Output files : + None + +Statistics creation : + geo => + country_code => count + None + +Statistics update : +valid_visitors: + country_code + +Statistics deletion : + None +""" + +class IWLAPostAnalysisIPToGeo(IPlugin): + + def __init__(self, iwla): + super(IWLAPostAnalysisIPToGeo, self).__init__(iwla) + self.API_VERSION = 1 + + def load(self): + remote_addr = self.iwla.getConfValue('iptogeo_remote_addr', + None) + remote_port = self.iwla.getConfValue('iptogeo_remote_port', + None) + + args = {} + if remote_addr: args['remote_addr'] = remote_addr + if remote_port: args['remote_port'] = remote_port + self.iptogeo = IPToGeo(**args) + + return True + + def hook(self): + visitors = self.iwla.getValidVisitors() + month_stats = self.iwla.getMonthStats() + + geo = month_stats.get('geo', {}) + + for (ip, visitor) in visitors.items(): + if visitor.get('country_code', False): continue + try: + (_, cc) = self.iptogeo.ip_to_geo(ip) + cc = cc and cc or 'ip' + visitor['country_code'] = cc + if cc in geo.keys(): + geo[cc] += 1 + else: + geo[cc] = 1 + except Exception, e: + print e + + month_stats['geo'] = geo diff --git a/plugins/post_analysis/iptogeo.py b/plugins/post_analysis/iptogeo.py new file mode 100644 index 0000000..f887346 --- /dev/null +++ b/plugins/post_analysis/iptogeo.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import socket +import struct + +class IPToGeoException(Exception): + pass + +class IPToGeo(object): + + MAGIC = 0x179E08EF + VERSION = 1 + REQ = 1 + RESP = 0 + IPV4 = 4 + IPV6 = 16 + + IP_NOT_FOUND = 6 + + PACKET_SIZE = 32 + + ERRORS = {1 : 'Bad magic', + 2 : 'Bad version', + 3 : 'Bad request field' , + 4 : 'Bad IP version', + 5 : 'Unsupported IP version', + 6 : 'IP not found'} + + def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None): + self._remote_addr = remote_addr + self._remote_port = remote_port + self._timeout = timeout + + self._create_socket() + + def _create_socket(self): + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not self._timeout is None: + self._socket.settimeout(self._timeout) + self._socket.connect((self._remote_addr, self._remote_port)) + + def _create_request(self, ip): + packet = '' + packet += struct.pack('