diff --git a/conf.py b/conf.py index e06aa5b..330196b 100644 --- a/conf.py +++ b/conf.py @@ -10,8 +10,8 @@ display_visitor_ip = True # Hooks used pre_analysis_hooks = ['page_to_hit', 'robots'] -post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns'] -display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff'] +post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'reverse_dns', 'ip_to_geo'] +display_hooks = ['track_users', 'top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'ip_to_geo', 'operating_systems', 'browsers', 'feeds', 'hours_stats', 'top_downloads_diff'] # Reverse DNS timeout reverse_dns_timeout = 0.2 diff --git a/plugins/display/ip_to_geo.py b/plugins/display/ip_to_geo.py new file mode 100644 index 0000000..c0127ac --- /dev/null +++ b/plugins/display/ip_to_geo.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2015 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +import re + +from iwla import IWLA +from iplugin import IPlugin +from display import * + +import awstats_data + +""" +Display hook + +Add geo statistics + +Plugin requirements : + post_analysis/ip_to_geo + +Conf values needed : + create_geo_page* + +Output files : + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" + +class IWLADisplayTopGeo(IPlugin): + def __init__(self, iwla): + super(IWLADisplayTopGeo, self).__init__(iwla) + self.API_VERSION = 1 + self.requires = ['IWLAPostAnalysisIPToGeo'] + + def load(self): + self.icon_path = self.iwla.getConfValue('icon_path', '/') + self.create_geo_page = self.iwla.getConfValue('create_geo_page_page', True) + + display = self.iwla.getDisplay() + display.addColumnFilter(self.iwla._(u'Host'), self.FlagFilter, {'self':self}) + + return True + + @staticmethod # Needed to have unbound methd + def FlagFilter(host, self): + cc = None + host_name = host.split(' ')[0] # hostname or ip + if host_name in self.valid_visitors.keys(): + cc = self.valid_visitors[host_name]['country_code'] + else: + for visitor in self.valid_visitors.values(): + if visitor['remote_addr'] == host_name: + cc = visitor['country_code'] + break + if not cc or cc == 'ip': return None + icon = '' % (self.icon_path, cc) + return '%s %s' % (icon ,host) + + def hook(self): + display = self.iwla.getDisplay() + geo = self.iwla.getMonthStats()['geo'] + geo = sorted(geo.items(), key=lambda t: t[1], reverse=True) + self.valid_visitors = self.iwla.getValidVisitors() + + # All in a page + if self.create_geo_page: + title = createCurTitle(self.iwla, u'All Coutries') + filename = 'geo.html' + path = self.iwla.getCurDisplayPath(filename) + + page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) + table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Countries'), ['', self.iwla._(u'Country'), self.iwla._(u'Visitors')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (cc, visitors) in geo: + icon = '' % (self.icon_path, cc) + table.appendRow([icon, cc, visitors]) + table.computeRatio(2) + page.appendBlock(table) + + display.addPage(page) + + # Countries in index + title = self.iwla._(u'Countries') + if self.create_geo_page: + link = '%s' % (filename, self.iwla._(u'Details')) + title = '%s - %s' % (title, link) + + index = self.iwla.getDisplayIndex() + + table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Countries'), self.iwla._(u'Visitors')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (cc, visitors) in geo[:10]: + icon = '' % (self.icon_path, cc) + table.appendRow([icon, cc, visitors]) + table.computeRatio(2) + index.appendBlock(table) diff --git a/plugins/post_analysis/ip_to_geo.py b/plugins/post_analysis/ip_to_geo.py new file mode 100644 index 0000000..a15e016 --- /dev/null +++ b/plugins/post_analysis/ip_to_geo.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2016 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +from iwla import IWLA +from iplugin import IPlugin + +from iptogeo import IPToGeo + +""" +Post analysis hook + +Get country code from IP address + +Plugin requirements : + None + +Conf values needed : + iptogeo_remote_addr* + iptogeo_remote_port* + +Output files : + None + +Statistics creation : + geo => + country_code => count + None + +Statistics update : +valid_visitors: + country_code + +Statistics deletion : + None +""" + +class IWLAPostAnalysisIPToGeo(IPlugin): + + def __init__(self, iwla): + super(IWLAPostAnalysisIPToGeo, self).__init__(iwla) + self.API_VERSION = 1 + + def load(self): + remote_addr = self.iwla.getConfValue('iptogeo_remote_addr', + None) + remote_port = self.iwla.getConfValue('iptogeo_remote_port', + None) + + args = {} + if remote_addr: args['remote_addr'] = remote_addr + if remote_port: args['remote_port'] = remote_port + self.iptogeo = IPToGeo(**args) + + return True + + def hook(self): + visitors = self.iwla.getValidVisitors() + month_stats = self.iwla.getMonthStats() + + geo = month_stats.get('geo', {}) + + for (ip, visitor) in visitors.items(): + if visitor.get('country_code', False): continue + try: + (_, cc) = self.iptogeo.ip_to_geo(ip) + cc = cc and cc or 'ip' + visitor['country_code'] = cc + if cc in geo.keys(): + geo[cc] += 1 + else: + geo[cc] = 1 + except Exception, e: + print e + + month_stats['geo'] = geo diff --git a/plugins/post_analysis/iptogeo.py b/plugins/post_analysis/iptogeo.py new file mode 100644 index 0000000..f887346 --- /dev/null +++ b/plugins/post_analysis/iptogeo.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import socket +import struct + +class IPToGeoException(Exception): + pass + +class IPToGeo(object): + + MAGIC = 0x179E08EF + VERSION = 1 + REQ = 1 + RESP = 0 + IPV4 = 4 + IPV6 = 16 + + IP_NOT_FOUND = 6 + + PACKET_SIZE = 32 + + ERRORS = {1 : 'Bad magic', + 2 : 'Bad version', + 3 : 'Bad request field' , + 4 : 'Bad IP version', + 5 : 'Unsupported IP version', + 6 : 'IP not found'} + + def __init__(self, remote_addr='127.0.0.1', remote_port=53333, timeout=None): + self._remote_addr = remote_addr + self._remote_port = remote_port + self._timeout = timeout + + self._create_socket() + + def _create_socket(self): + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if not self._timeout is None: + self._socket.settimeout(self._timeout) + self._socket.connect((self._remote_addr, self._remote_port)) + + def _create_request(self, ip): + packet = '' + packet += struct.pack('