From a35d462cb773a135cd09e6e9e12bcac1a041c7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 11:34:25 +0100 Subject: [PATCH 1/7] Replace # for module description by """ (help auto extraction) --- default_conf.py | 6 +- iwla.py | 165 ++++++++++++----------- plugins/display/all_visits.py | 48 +++---- plugins/display/referers.py | 56 ++++---- plugins/display/top_downloads.py | 50 +++---- plugins/display/top_hits.py | 50 +++---- plugins/display/top_pages.py | 50 +++---- plugins/display/top_visitors.py | 46 +++---- plugins/post_analysis/referers.py | 68 +++++----- plugins/post_analysis/reverse_dns.py | 52 +++---- plugins/post_analysis/top_downloads.py | 50 +++---- plugins/post_analysis/top_hits.py | 50 +++---- plugins/post_analysis/top_pages.py | 50 +++---- plugins/pre_analysis/page_to_hit.py | 50 +++---- plugins/pre_analysis/robots.py | 52 +++---- iwla_convert.pl => tools/iwla_convert.pl | 0 16 files changed, 422 insertions(+), 421 deletions(-) rename iwla_convert.pl => tools/iwla_convert.pl (100%) diff --git a/default_conf.py b/default_conf.py index c28a4c6..e1b94b7 100644 --- a/default_conf.py +++ b/default_conf.py @@ -2,9 +2,9 @@ import os # Default configuration -# Default directory where to store database information +# Default database directory DB_ROOT = './output_db' -# Default directory where to create html files +# Default HTML output directory DISPLAY_ROOT = './output' # Hooks directories (don't edit) @@ -49,7 +49,7 @@ icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon') # CSS path (you can add yours) css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')] -# Extensions to compress in gzip during display build +# Extensions to compress in gzip during display build compress_output_files = [] # Path to locales files diff --git a/iwla.py b/iwla.py index 609108d..0bcc3fc 100755 --- a/iwla.py +++ b/iwla.py @@ -41,88 +41,89 @@ del _ from iplugin import * from display import * -# -# Main class IWLA -# Parse Log, compute them, call plugins and produce output -# For now, only HTTP log are valid -# -# Plugin requirements : -# None -# -# Conf values needed : -# analyzed_filename -# domain_name -# locales_path -# compress_output_files* -# -# Output files : -# DB_ROOT/meta.db -# DB_ROOT/year/month/iwla.db -# OUTPUT_ROOT/index.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# -# meta : -# last_time -# start_analysis_time -# stats => -# year => -# month => -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# nb_visitors -# -# month_stats : -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# -# days_stats : -# day => -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# nb_visitors -# -# visits : -# remote_addr => -# remote_addr -# remote_ip -# viewed_pages -# viewed_hits -# not_viewed_pages -# not_viewed_hits -# bandwidth -# last_access -# requests => -# [fields_from_format_log] -# extract_request => -# extract_uri -# extract_parameters* -# extract_referer* => -# extract_uri -# extract_parameters* -# robot -# hit_only -# is_page -# -# valid_visitors: -# month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors) -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Main class IWLA +Parse Log, compute them, call plugins and produce output +For now, only HTTP log are valid + +Plugin requirements : + None + +Conf values needed : + analyzed_filename + domain_name + locales_path + compress_output_files* + +Output files : + DB_ROOT/meta.db + DB_ROOT/year/month/iwla.db + OUTPUT_ROOT/index.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + +meta : + last_time + start_analysis_time + stats => + year => + month => + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + nb_visitors + +month_stats : + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + +days_stats : + day => + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + nb_visitors + +visits : + remote_addr => + remote_addr + remote_ip + viewed_pages + viewed_hits + not_viewed_pages + not_viewed_hits + bandwidth + last_access + requests => + [fields_from_format_log] + extract_request => + extract_uri + extract_parameters* + extract_referer* => + extract_uri + extract_parameters* + robot + hit_only + is_page + +valid_visitors: + month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors) + +Statistics update : + None + +Statistics deletion : + None +""" + class IWLA(object): diff --git a/plugins/display/all_visits.py b/plugins/display/all_visits.py index 442f4e2..4f4dd52 100644 --- a/plugins/display/all_visits.py +++ b/plugins/display/all_visits.py @@ -24,30 +24,30 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create All visits page -# -# Plugin requirements : -# None -# -# Conf values needed : -# display_visitor_ip* -# -# Output files : -# OUTPUT_ROOT/year/month/all_visits.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create All visits page + +Plugin requirements : + None + +Conf values needed : + display_visitor_ip* + +Output files : + OUTPUT_ROOT/year/month/all_visits.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayAllVisits(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/referers.py b/plugins/display/referers.py index cbdbeea..bdd04a5 100644 --- a/plugins/display/referers.py +++ b/plugins/display/referers.py @@ -22,34 +22,34 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create Referers page -# -# Plugin requirements : -# post_analysis/referers -# -# Conf values needed : -# max_referers_displayed* -# create_all_referers_page* -# max_key_phrases_displayed* -# create_all_key_phrases_page* -# -# Output files : -# OUTPUT_ROOT/year/month/referers.html -# OUTPUT_ROOT/year/month/key_phrases.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create Referers page + +Plugin requirements : + post_analysis/referers + +Conf values needed : + max_referers_displayed* + create_all_referers_page* + max_key_phrases_displayed* + create_all_key_phrases_page* + +Output files : + OUTPUT_ROOT/year/month/referers.html + OUTPUT_ROOT/year/month/key_phrases.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayReferers(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_downloads.py b/plugins/display/top_downloads.py index b97c6a7..f5d4a60 100644 --- a/plugins/display/top_downloads.py +++ b/plugins/display/top_downloads.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP downloads page -# -# Plugin requirements : -# post_analysis/top_downloads -# -# Conf values needed : -# max_downloads_displayed* -# create_all_downloads_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_downloads.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP downloads page + +Plugin requirements : + post_analysis/top_downloads + +Conf values needed : + max_downloads_displayed* + create_all_downloads_page* + +Output files : + OUTPUT_ROOT/year/month/top_downloads.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopDownloads(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_hits.py b/plugins/display/top_hits.py index f080004..412b3fe 100644 --- a/plugins/display/top_hits.py +++ b/plugins/display/top_hits.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP hits page -# -# Plugin requirements : -# post_analysis/top_hits -# -# Conf values needed : -# max_hits_displayed* -# create_all_hits_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_hits.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP hits page + +Plugin requirements : + post_analysis/top_hits + +Conf values needed : + max_hits_displayed* + create_all_hits_page* + +Output files : + OUTPUT_ROOT/year/month/top_hits.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopHits(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_pages.py b/plugins/display/top_pages.py index 53378ce..b44bf81 100644 --- a/plugins/display/top_pages.py +++ b/plugins/display/top_pages.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP pages page -# -# Plugin requirements : -# post_analysis/top_pages -# -# Conf values needed : -# max_pages_displayed* -# create_all_pages_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_pages.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP pages page + +Plugin requirements : + post_analysis/top_pages + +Conf values needed : + max_pages_displayed* + create_all_pages_page* + +Output files : + OUTPUT_ROOT/year/month/top_pages.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopPages(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_visitors.py b/plugins/display/top_visitors.py index a48255b..b7d3b66 100644 --- a/plugins/display/top_visitors.py +++ b/plugins/display/top_visitors.py @@ -24,29 +24,29 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP visitors block -# -# Plugin requirements : -# None -# -# Conf values needed : -# display_visitor_ip* -# -# Output files : -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP visitors block + +Plugin requirements : + None + +Conf values needed : + display_visitor_ip* + +Output files : + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopVisitors(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/referers.py b/plugins/post_analysis/referers.py index 4a44b24..64ec66f 100644 --- a/plugins/post_analysis/referers.py +++ b/plugins/post_analysis/referers.py @@ -26,40 +26,40 @@ from iplugin import IPlugin import awstats_data -# -# Post analysis hook -# -# Extract referers and key phrases from requests -# -# Plugin requirements : -# None -# -# Conf values needed : -# domain_name -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats : -# referers => -# pages -# hits -# robots_referers => -# pages -# hits -# search_engine_referers => -# pages -# hits -# key_phrases => -# phrase -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Extract referers and key phrases from requests + +Plugin requirements : + None + +Conf values needed : + domain_name + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats : + referers => + pages + hits + robots_referers => + pages + hits + search_engine_referers => + pages + hits + key_phrases => + phrase + +Statistics deletion : + None +""" class IWLAPostAnalysisReferers(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/reverse_dns.py b/plugins/post_analysis/reverse_dns.py index 6def2ff..c63965f 100644 --- a/plugins/post_analysis/reverse_dns.py +++ b/plugins/post_analysis/reverse_dns.py @@ -23,32 +23,32 @@ import socket from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Replace IP by reverse DNS names -# -# Plugin requirements : -# None -# -# Conf values needed : -# reverse_dns_timeout* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# valid_visitors: -# remote_addr -# dns_name_replaced -# dns_analyzed -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Replace IP by reverse DNS names + +Plugin requirements : + None + +Conf values needed : + reverse_dns_timeout* + +Output files : + None + +Statistics creation : + None + +Statistics update : +valid_visitors: + remote_addr + dns_name_replaced + dns_analyzed + +Statistics deletion : + None +""" class IWLAPostAnalysisReverseDNS(IPlugin): DEFAULT_DNS_TIMEOUT = 0.5 diff --git a/plugins/post_analysis/top_downloads.py b/plugins/post_analysis/top_downloads.py index 31bb112..7b28c33 100644 --- a/plugins/post_analysis/top_downloads.py +++ b/plugins/post_analysis/top_downloads.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP downloads -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_downloads => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP downloads + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_downloads => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopDownloads(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/top_hits.py b/plugins/post_analysis/top_hits.py index 2f056b2..64446c7 100644 --- a/plugins/post_analysis/top_hits.py +++ b/plugins/post_analysis/top_hits.py @@ -21,31 +21,31 @@ from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP hits -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_hits => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP hits + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_hits => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopHits(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/top_pages.py b/plugins/post_analysis/top_pages.py index e47413e..a5d086c 100644 --- a/plugins/post_analysis/top_pages.py +++ b/plugins/post_analysis/top_pages.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP pages -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_pages => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP pages + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_pages => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopPages(IPlugin): def __init__(self, iwla): diff --git a/plugins/pre_analysis/page_to_hit.py b/plugins/pre_analysis/page_to_hit.py index 9093c66..a3919c4 100644 --- a/plugins/pre_analysis/page_to_hit.py +++ b/plugins/pre_analysis/page_to_hit.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Pre analysis hook -# Change page into hit and hit into page into statistics -# -# Plugin requirements : -# None -# -# Conf values needed : -# page_to_hit_conf* -# hit_to_page_conf* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# visits : -# remote_addr => -# is_page -# -# Statistics deletion : -# None -# +""" +Pre analysis hook +Change page into hit and hit into page into statistics + +Plugin requirements : + None + +Conf values needed : + page_to_hit_conf* + hit_to_page_conf* + +Output files : + None + +Statistics creation : + None + +Statistics update : +visits : + remote_addr => + is_page + +Statistics deletion : + None +""" class IWLAPreAnalysisPageToHit(IPlugin): diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index a311588..662ce57 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -25,32 +25,32 @@ from iplugin import IPlugin import awstats_data -# -# Pre analysis hook -# -# Filter robots -# -# Plugin requirements : -# None -# -# Conf values needed : -# page_to_hit_conf* -# hit_to_page_conf* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# visits : -# remote_addr => -# robot -# -# Statistics deletion : -# None -# +""" +Pre analysis hook + +Filter robots + +Plugin requirements : + None + +Conf values needed : + page_to_hit_conf* + hit_to_page_conf* + +Output files : + None + +Statistics creation : + None + +Statistics update : +visits : + remote_addr => + robot + +Statistics deletion : + None +""" class IWLAPreAnalysisRobots(IPlugin): def __init__(self, iwla): diff --git a/iwla_convert.pl b/tools/iwla_convert.pl similarity index 100% rename from iwla_convert.pl rename to tools/iwla_convert.pl From a37b661c1fdfd7a970b8b7f6861fa220657c4839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 11:35:00 +0100 Subject: [PATCH 2/7] Add tools to automatically extract documentation --- tools/extract_doc.py | 29 +++++++++++++++++++++++++++++ tools/extract_docs.sh | 16 ++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 tools/extract_doc.py create mode 100755 tools/extract_docs.sh diff --git a/tools/extract_doc.py b/tools/extract_doc.py new file mode 100755 index 0000000..0f5fbc3 --- /dev/null +++ b/tools/extract_doc.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import sys + +filename = sys.argv[1] + +if filename.endswith('__init__.py'): + sys.exit(0) + +package_name = filename.replace('/', '.').replace('.py', '') +sys.stdout.write('**%s**' % (package_name)) +sys.stdout.write('\n\n') +# sys.stdout.write('-' * len(package_name)) +# sys.stdout.write('\n\n') + +sys.stderr.write('\tExtract doc from %s\n' % (filename)) + +with open(filename) as infile: + copy = False + for line in infile: + if line.strip() in ['"""', "'''"]: + if not copy: + copy = True + else: + break + elif copy: + sys.stdout.write(line) + +sys.stdout.write('\n\n') diff --git a/tools/extract_docs.sh b/tools/extract_docs.sh new file mode 100755 index 0000000..e556330 --- /dev/null +++ b/tools/extract_docs.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +MODULES_TARGET="docs/modules.md" +MAIN_MD="docs/main.md" +TARGET_MD="docs/index.md" + +rm -f "${MODULES_TARGET}" + +echo "Generate doc from iwla.py" +python tools/extract_doc.py iwla.py > "${MODULES_TARGET}" + +echo "Generate plugins documentation" +find plugins -name '*.py' -exec python tools/extract_doc.py \{\} \; >> "${MODULES_TARGET}" + +echo "Generate ${TARGET_MD}" +cat "${MAIN_MD}" "${MODULES_TARGET}" > "${TARGET_MD}" From f92cec4428b15dec25af1058f3df40ee1dc0a672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 11:35:27 +0100 Subject: [PATCH 3/7] Add Markdown documentation --- docs/main.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 docs/main.md diff --git a/docs/main.md b/docs/main.md new file mode 100644 index 0000000..389e653 --- /dev/null +++ b/docs/main.md @@ -0,0 +1,74 @@ +iwla +==== + +Introduction +------------ + +iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www.awstats.org). The main problem with awstats is that it's a very monolothic project with everything in one big PERL file. In opposite, iwla has be though to be very modulor : a small core analysis and a lot of filters. It can be viewed as UNIX pipes. Philosophy of iwla is : add, update, delete ! That's the job of each filters : modify statistics until final result. + +Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option). + +Usage +----- + +./iwla [-c|--clean-output] [-i|--stdin] [-f FILE|--file FILE] [-d LOGLEVEL|--log-level LOGLEVEL] + + -c : Clean output (database and HTML) before starting + -i : Read data from stdin instead of conf.analyzed_filename + -f : Read data from FILE instead of conf.analyzed_filename + -d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + +Basic usage +----------- + +In addition to command line, iwla read parameters in _ default_conf.py _. User can override default values using _conf.py_ file. Each module requires its own parameters. +Main valued to edit are : + analyzed_filename : web server log + domaine_name : domain name to filter + pre_analysis_hooks + post_analysis_hooks + display_hooks + locale + +You can then launch iwla. Output HTML files are created in _output_ directory by default. To quickly see it go in output and type + python -m SimpleHTTPServer 8000 +Open your favorite web browser at _http://localhost:8000_. Enjoy ! + +Interesting default configuration values +---------------------------------------- + + DB_ROOT : Default database directory + DISPLAY_ROOT : Default HTML output directory + log_format : Web server log format (nginx style). Default is what apache log + time_format : Time format used in log format + pages_extensions : Extensions that are considered as a HTML page (or result) + viewed_http_codes : HTTP codes that are cosidered OK + count_hit_only_visitors : If False, doesn't cout visitors that doesn't GET a page but resources only (images, rss...) + multimedia_files : Multimedia extensions (not accounted as downloaded files) + css_path : CSS path (you can add yours) + compress_output_files : Extensions to compress in gzip during display build + +Plugins +------- + +As previously described, plugins acts like UNIX pipes : final statistics are constantly updated by each plugin to produce final result. We have three type of plugins : + Pre analysis plugins : Called before generating days statistics. They are in charge to filter robots, crawlers, bad pages... + Post analysis plugins : Called after basic statistics computation. They are in charge to enlight them with each own algorithms + Display plugins : They are in charge to produce HTML files from statistics. + +To use plugins, just insert their name in pre_analysis_hooks, post_analysis_hooks and display_hooks list. + +Create a Plugins +---------------- + +To create a new plugin, it's necessary to create a derived class of IPlugin (_iplugin.py) in the right directory (_plugins/xxx/your_plugin.py_). + +Plugins can defines required configuration values (self.conf_requires) that must be set in conf.py (or can be optional). They can also defines required plugins (self.requires). + +For display plugins, a lot of code has been wrote in _display.py_ that simplify the creation on HTML blocks, tables and graphs. + +Modules +------- + +Optional configuration values ends with *. + From d6bc6ea19285fdb6d4cbb9ed9698d861ccd3fb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 11:35:49 +0100 Subject: [PATCH 4/7] Update TODO --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index 84bc632..c3e15d5 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,3 @@ -doc auto generation doc enhancement Limit hits/pages/downloads by rate Automatic tests From 54b7b59899565938c72e772dac37858f87acc3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 17:21:45 +0100 Subject: [PATCH 5/7] Update documentation --- default_conf.py | 6 ++-- docs/main.md | 78 +++++++++++++++++++++++++++----------------- tools/extract_doc.py | 9 ++--- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/default_conf.py b/default_conf.py index e1b94b7..3c42a1e 100644 --- a/default_conf.py +++ b/default_conf.py @@ -17,7 +17,7 @@ META_PATH = os.path.join(DB_ROOT, 'meta.db') # Database filename per month DB_FILENAME = 'iwla.db' -# Web server log format (nginx style). Default is what apache log +# Web server log format (nginx style). Default is apache log format log_format = '$server_name:$server_port $remote_addr - $remote_user [$time_local] ' +\ '"$request" $status $body_bytes_sent ' +\ '"$http_referer" "$http_user_agent"' @@ -30,7 +30,7 @@ pre_analysis_hooks = [] post_analysis_hooks = [] display_hooks = [] -# Extensions that are considered as a HTML page (or result) +# Extensions that are considered as a HTML page (or result) in opposite to hits pages_extensions = ['/', 'htm', 'html', 'xhtml', 'py', 'pl', 'rb', 'php'] # HTTP codes that are cosidered OK viewed_http_codes = [200, 304] @@ -49,7 +49,7 @@ icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon') # CSS path (you can add yours) css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')] -# Extensions to compress in gzip during display build +# Files extensions to compress in gzip during display build compress_output_files = [] # Path to locales files diff --git a/docs/main.md b/docs/main.md index 389e653..8fe2240 100644 --- a/docs/main.md +++ b/docs/main.md @@ -4,14 +4,14 @@ iwla Introduction ------------ -iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www.awstats.org). The main problem with awstats is that it's a very monolothic project with everything in one big PERL file. In opposite, iwla has be though to be very modulor : a small core analysis and a lot of filters. It can be viewed as UNIX pipes. Philosophy of iwla is : add, update, delete ! That's the job of each filters : modify statistics until final result. +iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www.awstats.org). The main problem with awstats is that it's a very monolothic project with everything in one big PERL file. In opposite, iwla has be though to be very modular : a small core analysis and a lot of filters. It can be viewed as UNIX pipes. Philosophy of iwla is : add, update, delete ! That's the job of each filters : modify statistics until final result. Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option). Usage ----- -./iwla [-c|--clean-output] [-i|--stdin] [-f FILE|--file FILE] [-d LOGLEVEL|--log-level LOGLEVEL] + ./iwla [-c|--clean-output] [-i|--stdin] [-f FILE|--file FILE] [-d LOGLEVEL|--log-level LOGLEVEL] -c : Clean output (database and HTML) before starting -i : Read data from stdin instead of conf.analyzed_filename @@ -21,54 +21,72 @@ Usage Basic usage ----------- -In addition to command line, iwla read parameters in _ default_conf.py _. User can override default values using _conf.py_ file. Each module requires its own parameters. -Main valued to edit are : - analyzed_filename : web server log - domaine_name : domain name to filter - pre_analysis_hooks - post_analysis_hooks - display_hooks - locale +In addition to command line, iwla read parameters in default_conf.py. User can override default values using _conf.py_ file. Each module requires its own parameters. + +Main values to edit are : + + * **analyzed_filename** : web server log + * **domaine_name** : domain name to filter + * **pre_analysis_hooks** : List of pre analysis hooks + * **post_analysis_hooks** : List of post analysis hooks + * **display_hooks** : List of display hooks + * **locale** : Displayed locale (_en_ or _fr_) + +Then, you can then iwla. Output HTML files are created in _output_ directory by default. To quickly see it go in output and type -You can then launch iwla. Output HTML files are created in _output_ directory by default. To quickly see it go in output and type python -m SimpleHTTPServer 8000 + Open your favorite web browser at _http://localhost:8000_. Enjoy ! +**Warning** : The order is hooks list is important : Some plugins may requires others plugins, and the order of display_hooks is the order of displayed blocks in final result. + + Interesting default configuration values ---------------------------------------- - DB_ROOT : Default database directory - DISPLAY_ROOT : Default HTML output directory - log_format : Web server log format (nginx style). Default is what apache log - time_format : Time format used in log format - pages_extensions : Extensions that are considered as a HTML page (or result) - viewed_http_codes : HTTP codes that are cosidered OK - count_hit_only_visitors : If False, doesn't cout visitors that doesn't GET a page but resources only (images, rss...) - multimedia_files : Multimedia extensions (not accounted as downloaded files) - css_path : CSS path (you can add yours) - compress_output_files : Extensions to compress in gzip during display build + * **DB_ROOT** : Default database directory (default ./output_db) + * **DISPLAY_ROOT** : Default HTML output directory (default ./output) + * **log_format** : Web server log format (nginx style). Default is apache log format + * **time_format** : Time format used in log format + * **pages_extensions** : Extensions that are considered as a HTML page (or result) in opposit to hits + * **viewed_http_codes** : HTTP codes that are cosidered OK (200, 304) + * **count_hit_only_visitors** : If False, doesn't cout visitors that doesn't GET a page but resources only (images, rss...) + * **multimedia_files** : Multimedia extensions (not accounted as downloaded files) + * **css_path** : CSS path (you can add yours) + * **compress_output_files** : Files extensions to compress in gzip during display build Plugins ------- -As previously described, plugins acts like UNIX pipes : final statistics are constantly updated by each plugin to produce final result. We have three type of plugins : - Pre analysis plugins : Called before generating days statistics. They are in charge to filter robots, crawlers, bad pages... - Post analysis plugins : Called after basic statistics computation. They are in charge to enlight them with each own algorithms - Display plugins : They are in charge to produce HTML files from statistics. +As previously described, plugins acts like UNIX pipes : statistics are constantly updated by each plugin to produce final result. We have three type of plugins : -To use plugins, just insert their name in pre_analysis_hooks, post_analysis_hooks and display_hooks list. + * **Pre analysis plugins** : Called before generating days statistics. They are in charge to filter robots, crawlers, bad pages... + * **Post analysis plugins** : Called after basic statistics computation. They are in charge to enlight them with their own algorithms + * **Display plugins** : They are in charge to produce HTML files from statistics. + +To use plugins, just insert their name in _pre_analysis_hooks_, _post_analysis_hooks_ and _display_hooks_ lists in conf.py. + +Statistics are stored in dictionaries : + + * **month_stats** : Statistics of current analysed month + * **valid_visitor** : A subset of month_stats without robots + * **days_stats** : Statistics of current analysed day + * **visits** : All visitors with all of its requests + * **meta** : Final result of month statistics (by year) Create a Plugins ---------------- -To create a new plugin, it's necessary to create a derived class of IPlugin (_iplugin.py) in the right directory (_plugins/xxx/your_plugin.py_). +To create a new plugin, it's necessary to create a derived class of IPlugin (_iplugin.py) in the right directory (_plugins/xxx/yourPlugin.py_). Plugins can defines required configuration values (self.conf_requires) that must be set in conf.py (or can be optional). They can also defines required plugins (self.requires). -For display plugins, a lot of code has been wrote in _display.py_ that simplify the creation on HTML blocks, tables and graphs. +The two functions to overload are _load(self)_ that must returns True or False if all is good (or not). It's called after _init_. The second is _hook(self)_ that is the body of plugins. -Modules -------- +For display plugins, a lot of code has been wrote in _display.py_ that simplify the creation on HTML blocks, tables and bar graphs. + +Plugins +======= Optional configuration values ends with *. diff --git a/tools/extract_doc.py b/tools/extract_doc.py index 0f5fbc3..eee534b 100755 --- a/tools/extract_doc.py +++ b/tools/extract_doc.py @@ -8,10 +8,11 @@ if filename.endswith('__init__.py'): sys.exit(0) package_name = filename.replace('/', '.').replace('.py', '') -sys.stdout.write('**%s**' % (package_name)) -sys.stdout.write('\n\n') -# sys.stdout.write('-' * len(package_name)) +sys.stdout.write('%s' % (package_name)) +sys.stdout.write('\n') # sys.stdout.write('\n\n') +sys.stdout.write('-' * len(package_name)) +sys.stdout.write('\n\n') sys.stderr.write('\tExtract doc from %s\n' % (filename)) @@ -24,6 +25,6 @@ with open(filename) as infile: else: break elif copy: - sys.stdout.write(line) + sys.stdout.write(' %s' % (line)) sys.stdout.write('\n\n') From d47a4609d87cd2b71e3b1b34ac80dbd04e35b6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 17:50:34 +0100 Subject: [PATCH 6/7] Update locales --- locales/fr_FR/LC_MESSAGES/iwla.mo | Bin 2596 -> 2934 bytes locales/fr_FR/LC_MESSAGES/iwla.pot | 311 ++++++++++++++++------------- 2 files changed, 175 insertions(+), 136 deletions(-) diff --git a/locales/fr_FR/LC_MESSAGES/iwla.mo b/locales/fr_FR/LC_MESSAGES/iwla.mo index 933477e0bbe4248cb5cde51610aeb02461f0fee0..360249b9be26cf63291f68baccb3bc9bf31d6867 100644 GIT binary patch literal 2934 zcmZ{kO^g&p6vqn%#SxKD6%{EJ5X4?)_-2+5cXxIL++~;9fhaN2-kI9jhMw-Gx_Wje z#t;uA9yAdX4n_@LNZf<*LOei1Ool|G2M-)HhL~_sPF_s(Ac_9}-L*S|(MrGm)vKz0 z^O067U1?VelfD1HS<8 z1+Rd3#5Jp52k(Ra4R}BJJxDu0*zsH7Qs}oqe*dTCUm*AW8{~J3VTAfJko&IzSAiQr ze%EW~4_f^Y$WVqYkJ<4PAnl#B`h=aIv^)(mo(QC$Z-Nhk=Rodr9)wHcg4I8=`bCib zT(aX=K>B;l>R*H0_d5`exMBGdNPD-e{wv6H{|)4RcR>2VkLgzqUDVT>k>d zbw!ZpQv#WvV^()T?h}BtYe4!p17ZvDHrQBz{DO}|{{xe>yBtQBf$PAPU>sbhM`0vJhy2%c11|NfLh3v6oJTErp`5DLtDqW3rz_uOI1Hn;=1CTs~xp)G? zvwR-HwjNS|?1u2nSs&{lY3uWSSqi5EKXhxcpmxYJS!c|xtE^X6lS(U{DW_FSMpN32 z?PSx7(=Q?|`XcT5qA#galGup8rqZw?`rV*5?bXbb=x@oI+Y$p|5U3e14CJH^8={1v zlB!ZvV?t?kjO+Qa7z~VdgQ^mPGe+sa_2u#M@Lh`Bq{E5OY!*Xmf}#^ww*-Qq#Ntp0 zH;%XgC7n5v_yW?xD^TL)`C!Db8ygub6=c06s<}pq5m!^HQ+T(;NEjf7qoI*aPfe?u z>;!f+Y+|CBxj$Mp0unNmQSH^eK#ZCxo?v#U<0>_oK@J93pv1U}C?+#8Zd~KV#;eAn z)0LB<7UM~y!5AxH)Rj7FnQ`{!Ge)I5W7N}(g>a@48rK)c%ZJ4YCXqvF2X5-{l&Tu% zaLqXZw~?r#^rn3`H#S;v2DE~2Vc?W-LPfbPU)bRkwmF5pvaq|jWBZnY!3x$nWyOvD ze_6he&*ymRP6auReHRWC<#?DFx7MDO`{J~xUaU886?&>+;{a9%^#m8nsVKK$J(?RU z;wrs9RL+eY9vSR7UFgZ@aM_FsuuUt1yGG5Jt~L9eV%xjI&6|MAL@v!}C;?%>8d zgF!W{c|pA>_e^*u_mT>5osFtF{jH*mT6JY;tS4&8LeI|JD~?^jFvdbv^9;s!^%U}$ z%5(?O>8cvYwk^tRU*sW)rLSUX+OvLpmR{9W11>W~c4~^Z8JfW|E{)Ra>>3EudxoOy z8nbmOtFYQVp56e~Zxcn^Op7*bUq!v9?~$;jP@ZYkDr^+TE)Eb^SIblIzpHp_DoVUq z9N}mQj;Al&U#-~dlyG|%o1tozneHokE5=Pl+RIW$y zeTC9aP+Mc^?@#FbTDSyfTs5ZE#*(Xytbo*(O~Z5Ws<=rajE?^a~QlPtGq o@1N3lHQrx*@l_!p}S^w zC*hz+6AvELgBQUVu&sQD0bg-`~dQ(k9f(xpStmH9KVHfj_)0Rf;{TH(|>~!w?CoS z{Ts@@SD>7GC5z>p8=%bF1jV22P|iC5C9XqGe-W}&y$r=(1xkG8A&)x6tIpjm!j0&E zU|j63VA49c7TyIlycZ6@_3#iBd&i;JFGKMwfa2#V$fI82C3aqe67P2$--SHteaDZX z#Q6&-`<;U_|7R%soOk*KDEs~H#xFWva^EjQiNilo?DY^F*?%3p2X2O9cMlZ1d!0T6 z#qTjUKI3>4%DIj~iI;)0e+}}en3wo_8j8I)pzQx9yc3>=-YoruKaCPe-j zyO0~({R~J9a>(Yck59P%!;X(biJinp+EzqzQ}TWQc@j}WwVED@7Q$NOSCUja5ZJ6U zbTWnznAt$A02Iwt^&G(TLShv#N$#CT5}9?}wFzpkn9M{6zl8>lD2;j%FjfO^umZg_k0U zIuIpR9rQzWuvu&K3XOUJ52u`m($BnAhy29q#F#MaH5(P*8a3g^NLx)rA-*1ttZoHn z!Bq6}0S-qkHCeW5G7hRis3z^a#3BpNwDIHeyvm?QLnaushMKm%4H6raldi6wi()n1 ztk?axtx8d&EA*meCZ%^R;-Yj%T=X>J5iXP>>(|swaa#sPED4);n?6?6nY~BprHG6`2jEA@8t(|f4;D5aL4dyiFsbp$VUI2_LR;) zUC0gY$mMgnUJ0sK;^vZ?j{^lg9W||A=`87IleA~{RqI@EUpcBj$Lz4$Bs;v4@$1f` ziSdciWhe7}xn45Gnh>+?2JTt2Xm>Vheh@yR%kzGm7`u0-H0JGDo+sOrq26d%jw(S| zE$BV7f$e?Xgyf<%6>opLpd0O~v0|*R(boCC-PfEckdVj6ioGv+Zh;(QDl(P8GHyVo8iojg3kQl9-4`+nMB*pQmwNjURF(=BghrGPxF$jXqIU*wUJg=*FcnJ z7e&=I=ITtBvD!VJW`NXRi=vAY$8{ z;HN>v()OUz|QCFl$BsW1yUle`ZB*WCF_r1Gw6I6n=FY9 z<$rac9wbJ0&SdUVl?i1Ta4vi$zvkwGvJUxqw3-xYcRd<@J2uAata^&3OnD+KwWb;C ziqY}4vF^$|)nw7tG8IKPBqqhqQqrua@ownS|6FKyoo2rT>_}bSa9gfW@j*UcSE=b( ITBi_@% diff --git a/locales/fr_FR/LC_MESSAGES/iwla.pot b/locales/fr_FR/LC_MESSAGES/iwla.pot index ce1f3a9..2286f1d 100644 --- a/locales/fr_FR/LC_MESSAGES/iwla.pot +++ b/locales/fr_FR/LC_MESSAGES/iwla.pot @@ -5,240 +5,279 @@ msgid "" msgstr "" "Project-Id-Version: iwla\n" -"POT-Creation-Date: 2014-12-16 21:36+CET\n" -"PO-Revision-Date: 2014-12-17 19:06+0100\n" +"POT-Creation-Date: 2014-12-19 17:43+CET\n" +"PO-Revision-Date: 2014-12-19 17:43+0100\n" "Last-Translator: Soutadé \n" "Language-Team: iwla\n" +"Language: fr_FR\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Generated-By: pygettext.py 1.5\n" -"Language: fr_FR\n" "X-Generator: Poedit 1.6.10\n" "X-Poedit-SourceCharset: UTF-8\n" -#: iwla.py:343 -msgid "Statistics" -msgstr "Statistiques" +#: display.py:32 +msgid "April" +msgstr "Avril" -#: iwla.py:349 -msgid "By day" -msgstr "Par jour" +#: display.py:32 +msgid "February" +msgstr "Février" -#: iwla.py:349 -msgid "Day" -msgstr "Jour" +#: display.py:32 +msgid "January" +msgstr "Janvier" -#: iwla.py:349 iwla.py:402 -msgid "Not viewed Bandwidth" -msgstr "Traffic non vu" - -#: iwla.py:349 iwla.py:402 -msgid "Visits" -msgstr "Visites" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/referers.py:78 plugins/display/referers.py:118 -#: plugins/display/top_downloads.py:74 plugins/display/top_visitors.py:53 -msgid "Hits" -msgstr "Hits" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/referers.py:78 plugins/display/referers.py:118 -#: plugins/display/top_visitors.py:53 -msgid "Pages" -msgstr "Pages" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/top_visitors.py:53 -msgid "Bandwidth" -msgstr "Bande passante" - -#: iwla.py:386 -msgid "Average" -msgstr "Moyenne" - -#: iwla.py:391 iwla.py:429 -msgid "Total" -msgstr "Total" - -#: iwla.py:400 -msgid "Apr" -msgstr "Avr" - -#: iwla.py:400 -msgid "Aug" -msgstr "Août" - -#: iwla.py:400 -msgid "Dec" -msgstr "Déc" - -#: iwla.py:400 -msgid "Feb" -msgstr "Fév" - -#: iwla.py:400 -msgid "Jan" -msgstr "Jan" - -#: iwla.py:400 +#: display.py:32 msgid "July" -msgstr "Jui" +msgstr "Juillet" -#: iwla.py:400 +#: display.py:32 +msgid "March" +msgstr "Mars" + +#: display.py:32 iwla.py:428 msgid "June" msgstr "Juin" -#: iwla.py:400 -msgid "Mar" -msgstr "Mars" - -#: iwla.py:400 +#: display.py:32 iwla.py:428 msgid "May" msgstr "Mai" -#: iwla.py:400 +#: display.py:33 +msgid "August" +msgstr "Août" + +#: display.py:33 +msgid "December" +msgstr "Décembre" + +#: display.py:33 +msgid "November" +msgstr "Novembre" + +#: display.py:33 +msgid "October" +msgstr "Octobre" + +#: display.py:33 +msgid "September" +msgstr "Septembre" + +#: iwla.py:371 +msgid "Statistics" +msgstr "Statistiques" + +#: iwla.py:377 +msgid "By day" +msgstr "Par jour" + +#: iwla.py:377 +msgid "Day" +msgstr "Jour" + +#: iwla.py:377 iwla.py:430 +msgid "Not viewed Bandwidth" +msgstr "Traffic non vu" + +#: iwla.py:377 iwla.py:430 +msgid "Visits" +msgstr "Visites" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 +#: plugins/display/top_downloads.py:97 plugins/display/top_visitors.py:72 +msgid "Hits" +msgstr "Hits" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 +#: plugins/display/top_visitors.py:72 +msgid "Pages" +msgstr "Pages" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/top_visitors.py:72 +msgid "Bandwidth" +msgstr "Bande passante" + +#: iwla.py:414 +msgid "Average" +msgstr "Moyenne" + +#: iwla.py:419 iwla.py:457 +msgid "Total" +msgstr "Total" + +#: iwla.py:428 +msgid "Apr" +msgstr "Avr" + +#: iwla.py:428 +msgid "Aug" +msgstr "Août" + +#: iwla.py:428 +msgid "Dec" +msgstr "Déc" + +#: iwla.py:428 +msgid "Feb" +msgstr "Fév" + +#: iwla.py:428 +msgid "Jan" +msgstr "Jan" + +#: iwla.py:428 +msgid "Jul" +msgstr "Jui" + +#: iwla.py:428 +msgid "Mar" +msgstr "Mars" + +#: iwla.py:428 msgid "Nov" msgstr "Nov" -#: iwla.py:400 +#: iwla.py:428 msgid "Oct" msgstr "Oct" -#: iwla.py:400 +#: iwla.py:428 msgid "Sep" msgstr "Sep" -#: iwla.py:401 +#: iwla.py:429 msgid "Summary" msgstr "Résumé" -#: iwla.py:402 +#: iwla.py:430 msgid "Month" msgstr "Mois" -#: iwla.py:402 +#: iwla.py:430 msgid "Visitors" msgstr "Visiteurs" -#: iwla.py:402 iwla.py:412 +#: iwla.py:430 iwla.py:440 msgid "Details" msgstr "Détails" -#: iwla.py:436 +#: iwla.py:465 msgid "Statistics for" msgstr "Statistiques pour" -#: iwla.py:443 +#: iwla.py:472 msgid "Last update" msgstr "Dernière mise à jour" -#: plugins/display/all_visits.py:45 plugins/display/all_visits.py:73 -msgid "All visits" -msgstr "Toutes les visites" - -#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:70 plugins/display/top_visitors.py:72 msgid "Host" msgstr "Hôte" -#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:70 plugins/display/top_visitors.py:72 msgid "Last seen" msgstr "Dernière visite" -#: plugins/display/all_visits.py:74 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:92 +msgid "All visits" +msgstr "Toutes les visites" + +#: plugins/display/all_visits.py:93 plugins/display/top_visitors.py:72 msgid "Top visitors" msgstr "Top visiteurs" -#: plugins/display/referers.py:72 plugins/display/referers.py:78 +#: plugins/display/referers.py:95 msgid "Connexion from" msgstr "Connexion depuis" -#: plugins/display/referers.py:78 plugins/display/referers.py:118 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 msgid "Origin" msgstr "Origine" -#: plugins/display/referers.py:82 plugins/display/referers.py:121 +#: plugins/display/referers.py:99 plugins/display/referers.py:156 msgid "Search Engine" msgstr "Moteur de recherche" -#: plugins/display/referers.py:91 plugins/display/referers.py:132 -msgid "External URL" -msgstr "URL externe" - -#: plugins/display/referers.py:100 plugins/display/referers.py:143 -msgid "External URL (robot)" -msgstr "URL externe (robot)" - -#: plugins/display/referers.py:112 -msgid "Top Referers" -msgstr "Top Origines" - -#: plugins/display/referers.py:114 -msgid "All Referers" -msgstr "Toutes les origines" - -#: plugins/display/referers.py:128 plugins/display/referers.py:139 -#: plugins/display/referers.py:150 plugins/display/referers.py:187 -#: plugins/display/top_downloads.py:80 plugins/display/top_hits.py:79 -#: plugins/display/top_pages.py:79 plugins/display/top_visitors.py:73 +#: plugins/display/referers.py:110 plugins/display/referers.py:125 +#: plugins/display/referers.py:140 plugins/display/referers.py:163 +#: plugins/display/referers.py:174 plugins/display/referers.py:185 +#: plugins/display/referers.py:222 plugins/display/top_downloads.py:83 +#: plugins/display/top_downloads.py:103 plugins/display/top_hits.py:82 +#: plugins/display/top_hits.py:103 plugins/display/top_pages.py:82 +#: plugins/display/top_pages.py:102 plugins/display/top_visitors.py:92 msgid "Others" msgstr "Autres" -#: plugins/display/referers.py:158 -msgid "Key Phrases" -msgstr "Phrases clé" +#: plugins/display/referers.py:114 plugins/display/referers.py:167 +msgid "External URL" +msgstr "URL externe" -#: plugins/display/referers.py:165 plugins/display/referers.py:175 +#: plugins/display/referers.py:129 plugins/display/referers.py:178 +msgid "External URL (robot)" +msgstr "URL externe (robot)" + +#: plugins/display/referers.py:147 +msgid "Top Referers" +msgstr "Top Origines" + +#: plugins/display/referers.py:149 +msgid "All Referers" +msgstr "Toutes les origines" + +#: plugins/display/referers.py:200 plugins/display/referers.py:210 msgid "Top key phrases" msgstr "Top phrases clé" -#: plugins/display/referers.py:165 plugins/display/referers.py:181 +#: plugins/display/referers.py:200 plugins/display/referers.py:216 msgid "Key phrase" msgstr "Phrase clé" -#: plugins/display/referers.py:165 plugins/display/referers.py:181 +#: plugins/display/referers.py:200 plugins/display/referers.py:216 msgid "Search" msgstr "Recherche" -#: plugins/display/referers.py:177 +#: plugins/display/referers.py:212 msgid "All key phrases" msgstr "Toutes les phrases clé" -#: plugins/display/top_downloads.py:51 plugins/display/top_downloads.py:54 -#: plugins/display/top_downloads.py:68 -msgid "All Downloads" -msgstr "Tous les téléchargements" - -#: plugins/display/top_downloads.py:54 +#: plugins/display/top_downloads.py:71 msgid "Hit" msgstr "Hit" -#: plugins/display/top_downloads.py:54 plugins/display/top_downloads.py:74 -#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73 -#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73 +#: plugins/display/top_downloads.py:71 plugins/display/top_downloads.py:91 +msgid "All Downloads" +msgstr "Tous les téléchargements" + +#: plugins/display/top_downloads.py:71 plugins/display/top_downloads.py:97 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:97 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:96 msgid "URI" msgstr "URI" -#: plugins/display/top_downloads.py:66 +#: plugins/display/top_downloads.py:89 msgid "Top Downloads" msgstr "Top Téléchargements" -#: plugins/display/top_hits.py:49 plugins/display/top_hits.py:54 -#: plugins/display/top_hits.py:67 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:91 msgid "All Hits" msgstr "Tous les hits" -#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73 -#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:97 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:96 msgid "Entrance" msgstr "Entrées" -#: plugins/display/top_pages.py:49 plugins/display/top_pages.py:54 -#: plugins/display/top_pages.py:67 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:90 msgid "All Pages" msgstr "Toutes les pages" -#: plugins/display/top_pages.py:65 +#: plugins/display/top_pages.py:88 msgid "Top Pages" msgstr "Top Pages" + +#~ msgid "Key Phrases" +#~ msgstr "Phrases clé" From ea32f7f0ae59876fd601b373dda5450b9cb11f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9gory=20Soutad=C3=A9?= Date: Fri, 19 Dec 2014 17:50:45 +0100 Subject: [PATCH 7/7] Append domain_name in pages title --- display.py | 14 +- iwla.pot | 312 +++++++++++++++++++--------------- iwla.py | 4 +- plugins/display/all_visits.py | 2 +- 4 files changed, 189 insertions(+), 143 deletions(-) diff --git a/display.py b/display.py index 7fe7a8b..cb98693 100644 --- a/display.py +++ b/display.py @@ -27,6 +27,12 @@ import logging # Create output HTML files # +# Just for detection +def _(name): pass +_('January'), _('February'), _('March'), _('April'), _('May'), _('June'), _('July') +_('August'), _('September'), _('October'), _('November'), _('December') +del _ + class DisplayHTMLRaw(object): def __init__(self, iwla, html=u''): @@ -370,6 +376,12 @@ def generateHTMLLink(url, name=None, max_length=100, prefix=u'http'): return u'%s' % (url, name[:max_length]) def createCurTitle(iwla, title): - title = iwla._(title) + time.strftime(u' - %B %Y', iwla.getCurTime()) + title = iwla._(title) + month_name = time.strftime(u'%B', iwla.getCurTime()) + year = time.strftime(u'%Y', iwla.getCurTime()) + title += u' - %s %s' % (iwla._(month_name), year) + domain_name = iwla.getConfValue('domain_name', '') + if domain_name: + title += u' - %s' % (domain_name) return title diff --git a/iwla.pot b/iwla.pot index 3f9a174..d8b1510 100644 --- a/iwla.pot +++ b/iwla.pot @@ -4,242 +4,276 @@ # msgid "" msgstr "" -"Project-Id-Version: iwla\n" -"POT-Creation-Date: 2014-12-16 21:36+CET\n" -"PO-Revision-Date: 2014-12-17 19:07+0100\n" -"Last-Translator: Soutadé \n" -"Language-Team: iwla \n" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2014-12-19 17:46+CET\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" "MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: 8bit\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: ENCODING\n" "Generated-By: pygettext.py 1.5\n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" -"Language: en\n" -"X-Generator: Poedit 1.6.10\n" -"X-Poedit-SourceCharset: UTF-8\n" -#: iwla.py:343 -msgid "Statistics" + +#: display.py:32 +msgid "April" msgstr "" -#: iwla.py:349 -msgid "By day" +#: display.py:32 +msgid "February" msgstr "" -#: iwla.py:349 -msgid "Day" +#: display.py:32 +msgid "January" msgstr "" -#: iwla.py:349 iwla.py:402 -msgid "Not viewed Bandwidth" -msgstr "" - -#: iwla.py:349 iwla.py:402 -msgid "Visits" -msgstr "" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/referers.py:78 plugins/display/referers.py:118 -#: plugins/display/top_downloads.py:74 plugins/display/top_visitors.py:53 -msgid "Hits" -msgstr "" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/referers.py:78 plugins/display/referers.py:118 -#: plugins/display/top_visitors.py:53 -msgid "Pages" -msgstr "" - -#: iwla.py:349 iwla.py:402 plugins/display/all_visits.py:51 -#: plugins/display/top_visitors.py:53 -msgid "Bandwidth" -msgstr "" - -#: iwla.py:386 -msgid "Average" -msgstr "" - -#: iwla.py:391 iwla.py:429 -msgid "Total" -msgstr "" - -#: iwla.py:400 -msgid "Apr" -msgstr "" - -#: iwla.py:400 -msgid "Aug" -msgstr "" - -#: iwla.py:400 -msgid "Dec" -msgstr "" - -#: iwla.py:400 -msgid "Feb" -msgstr "" - -#: iwla.py:400 -msgid "Jan" -msgstr "" - -#: iwla.py:400 +#: display.py:32 msgid "July" msgstr "" -#: iwla.py:400 +#: display.py:32 +msgid "March" +msgstr "" + +#: display.py:32 iwla.py:428 msgid "June" msgstr "" -#: iwla.py:400 -msgid "Mar" -msgstr "" - -#: iwla.py:400 +#: display.py:32 iwla.py:428 msgid "May" msgstr "" -#: iwla.py:400 +#: display.py:33 +msgid "August" +msgstr "" + +#: display.py:33 +msgid "December" +msgstr "" + +#: display.py:33 +msgid "November" +msgstr "" + +#: display.py:33 +msgid "October" +msgstr "" + +#: display.py:33 +msgid "September" +msgstr "" + +#: iwla.py:371 +msgid "Statistics" +msgstr "" + +#: iwla.py:377 +msgid "By day" +msgstr "" + +#: iwla.py:377 +msgid "Day" +msgstr "" + +#: iwla.py:377 iwla.py:430 +msgid "Not viewed Bandwidth" +msgstr "" + +#: iwla.py:377 iwla.py:430 +msgid "Visits" +msgstr "" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 +#: plugins/display/top_downloads.py:97 plugins/display/top_visitors.py:72 +msgid "Hits" +msgstr "" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 +#: plugins/display/top_visitors.py:72 +msgid "Pages" +msgstr "" + +#: iwla.py:377 iwla.py:430 plugins/display/all_visits.py:70 +#: plugins/display/top_visitors.py:72 +msgid "Bandwidth" +msgstr "" + +#: iwla.py:414 +msgid "Average" +msgstr "" + +#: iwla.py:419 iwla.py:457 +msgid "Total" +msgstr "" + +#: iwla.py:428 +msgid "Apr" +msgstr "" + +#: iwla.py:428 +msgid "Aug" +msgstr "" + +#: iwla.py:428 +msgid "Dec" +msgstr "" + +#: iwla.py:428 +msgid "Feb" +msgstr "" + +#: iwla.py:428 +msgid "Jan" +msgstr "" + +#: iwla.py:428 +msgid "Jul" +msgstr "" + +#: iwla.py:428 +msgid "Mar" +msgstr "" + +#: iwla.py:428 msgid "Nov" msgstr "" -#: iwla.py:400 +#: iwla.py:428 msgid "Oct" msgstr "" -#: iwla.py:400 +#: iwla.py:428 msgid "Sep" msgstr "" -#: iwla.py:401 +#: iwla.py:429 msgid "Summary" msgstr "" -#: iwla.py:402 +#: iwla.py:430 msgid "Month" msgstr "" -#: iwla.py:402 +#: iwla.py:430 msgid "Visitors" msgstr "" -#: iwla.py:402 iwla.py:412 +#: iwla.py:430 iwla.py:440 msgid "Details" msgstr "" -#: iwla.py:436 +#: iwla.py:465 msgid "Statistics for" msgstr "" -#: iwla.py:443 +#: iwla.py:472 msgid "Last update" msgstr "" -#: plugins/display/all_visits.py:45 plugins/display/all_visits.py:73 -msgid "All visits" -msgstr "" - -#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:70 plugins/display/top_visitors.py:72 msgid "Host" msgstr "" -#: plugins/display/all_visits.py:51 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:70 plugins/display/top_visitors.py:72 msgid "Last seen" msgstr "" -#: plugins/display/all_visits.py:74 plugins/display/top_visitors.py:53 +#: plugins/display/all_visits.py:92 +msgid "All visits" +msgstr "" + +#: plugins/display/all_visits.py:93 plugins/display/top_visitors.py:72 msgid "Top visitors" msgstr "" -#: plugins/display/referers.py:72 plugins/display/referers.py:78 +#: plugins/display/referers.py:95 msgid "Connexion from" msgstr "" -#: plugins/display/referers.py:78 plugins/display/referers.py:118 +#: plugins/display/referers.py:95 plugins/display/referers.py:153 msgid "Origin" msgstr "" -#: plugins/display/referers.py:82 plugins/display/referers.py:121 +#: plugins/display/referers.py:99 plugins/display/referers.py:156 msgid "Search Engine" msgstr "" -#: plugins/display/referers.py:91 plugins/display/referers.py:132 -msgid "External URL" -msgstr "" - -#: plugins/display/referers.py:100 plugins/display/referers.py:143 -msgid "External URL (robot)" -msgstr "" - -#: plugins/display/referers.py:112 -msgid "Top Referers" -msgstr "" - -#: plugins/display/referers.py:114 -msgid "All Referers" -msgstr "" - -#: plugins/display/referers.py:128 plugins/display/referers.py:139 -#: plugins/display/referers.py:150 plugins/display/referers.py:187 -#: plugins/display/top_downloads.py:80 plugins/display/top_hits.py:79 -#: plugins/display/top_pages.py:79 plugins/display/top_visitors.py:73 +#: plugins/display/referers.py:110 plugins/display/referers.py:125 +#: plugins/display/referers.py:140 plugins/display/referers.py:163 +#: plugins/display/referers.py:174 plugins/display/referers.py:185 +#: plugins/display/referers.py:222 plugins/display/top_downloads.py:83 +#: plugins/display/top_downloads.py:103 plugins/display/top_hits.py:82 +#: plugins/display/top_hits.py:103 plugins/display/top_pages.py:82 +#: plugins/display/top_pages.py:102 plugins/display/top_visitors.py:92 msgid "Others" msgstr "" -#: plugins/display/referers.py:158 -msgid "Key Phrases" +#: plugins/display/referers.py:114 plugins/display/referers.py:167 +msgid "External URL" msgstr "" -#: plugins/display/referers.py:165 plugins/display/referers.py:175 +#: plugins/display/referers.py:129 plugins/display/referers.py:178 +msgid "External URL (robot)" +msgstr "" + +#: plugins/display/referers.py:147 +msgid "Top Referers" +msgstr "" + +#: plugins/display/referers.py:149 +msgid "All Referers" +msgstr "" + +#: plugins/display/referers.py:200 plugins/display/referers.py:210 msgid "Top key phrases" msgstr "" -#: plugins/display/referers.py:165 plugins/display/referers.py:181 +#: plugins/display/referers.py:200 plugins/display/referers.py:216 msgid "Key phrase" msgstr "" -#: plugins/display/referers.py:165 plugins/display/referers.py:181 +#: plugins/display/referers.py:200 plugins/display/referers.py:216 msgid "Search" msgstr "" -#: plugins/display/referers.py:177 +#: plugins/display/referers.py:212 msgid "All key phrases" msgstr "" -#: plugins/display/top_downloads.py:51 plugins/display/top_downloads.py:54 -#: plugins/display/top_downloads.py:68 -msgid "All Downloads" -msgstr "" - -#: plugins/display/top_downloads.py:54 +#: plugins/display/top_downloads.py:71 msgid "Hit" msgstr "" -#: plugins/display/top_downloads.py:54 plugins/display/top_downloads.py:74 -#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73 -#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73 +#: plugins/display/top_downloads.py:71 plugins/display/top_downloads.py:91 +msgid "All Downloads" +msgstr "" + +#: plugins/display/top_downloads.py:71 plugins/display/top_downloads.py:97 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:97 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:96 msgid "URI" msgstr "" -#: plugins/display/top_downloads.py:66 +#: plugins/display/top_downloads.py:89 msgid "Top Downloads" msgstr "" -#: plugins/display/top_hits.py:49 plugins/display/top_hits.py:54 -#: plugins/display/top_hits.py:67 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:91 msgid "All Hits" msgstr "" -#: plugins/display/top_hits.py:54 plugins/display/top_hits.py:73 -#: plugins/display/top_pages.py:54 plugins/display/top_pages.py:73 +#: plugins/display/top_hits.py:71 plugins/display/top_hits.py:97 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:96 msgid "Entrance" msgstr "" -#: plugins/display/top_pages.py:49 plugins/display/top_pages.py:54 -#: plugins/display/top_pages.py:67 +#: plugins/display/top_pages.py:71 plugins/display/top_pages.py:90 msgid "All Pages" msgstr "" -#: plugins/display/top_pages.py:65 +#: plugins/display/top_pages.py:88 msgid "Top Pages" msgstr "" + diff --git a/iwla.py b/iwla.py index 0bcc3fc..9eade1c 100755 --- a/iwla.py +++ b/iwla.py @@ -368,7 +368,7 @@ class IWLA(object): def _generateDisplayDaysStats(self): cur_time = self.meta_infos['last_time'] - title = '%s %d/%02d' % (self._('Statistics'), cur_time.tm_year, cur_time.tm_mon) + title = createCurTitle(self, self._('Statistics')) filename = self.getCurDisplayPath('index.html') self.logger.info('==> Generate display (%s)' % (filename)) page = self.display.createPage(title, filename, conf.css_path) @@ -425,7 +425,7 @@ class IWLA(object): def _generateDisplayMonthStats(self, page, year, month_stats): cur_time = time.localtime() - months_name = ['', self._('Jan'), self._('Feb'), self._('Mar'), self._('Apr'), self._('May'), self._('June'), self._('July'), self._('Aug'), self._('Sep'), self._('Oct'), self._('Nov'), self._('Dec')] + months_name = ['', self._('Jan'), self._('Feb'), self._('Mar'), self._('Apr'), self._('May'), self._('June'), self._('Jul'), self._('Aug'), self._('Sep'), self._('Oct'), self._('Nov'), self._('Dec')] title = '%s %d' % (self._('Summary'), year) cols = [self._('Month'), self._('Visitors'), self._('Visits'), self._('Pages'), self._('Hits'), self._('Bandwidth'), self._('Not viewed Bandwidth'), self._('Details')] graph_cols=range(1,7) diff --git a/plugins/display/all_visits.py b/plugins/display/all_visits.py index 4f4dd52..4dbc92c 100644 --- a/plugins/display/all_visits.py +++ b/plugins/display/all_visits.py @@ -61,7 +61,7 @@ class IWLADisplayAllVisits(IPlugin): last_access = sorted(hits.values(), key=lambda t: t['last_access'], reverse=True) - title = time.strftime(self.iwla._(u'All visits') + u' - %B %Y', self.iwla.getCurTime()) + title = createCurTitle(self.iwla, u'All visits') filename = 'all_visits.html' path = self.iwla.getCurDisplayPath(filename)