diff --git a/default_conf.py b/default_conf.py index c28a4c6..e1b94b7 100644 --- a/default_conf.py +++ b/default_conf.py @@ -2,9 +2,9 @@ import os # Default configuration -# Default directory where to store database information +# Default database directory DB_ROOT = './output_db' -# Default directory where to create html files +# Default HTML output directory DISPLAY_ROOT = './output' # Hooks directories (don't edit) @@ -49,7 +49,7 @@ icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon') # CSS path (you can add yours) css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')] -# Extensions to compress in gzip during display build +# Extensions to compress in gzip during display build compress_output_files = [] # Path to locales files diff --git a/iwla.py b/iwla.py index 609108d..0bcc3fc 100755 --- a/iwla.py +++ b/iwla.py @@ -41,88 +41,89 @@ del _ from iplugin import * from display import * -# -# Main class IWLA -# Parse Log, compute them, call plugins and produce output -# For now, only HTTP log are valid -# -# Plugin requirements : -# None -# -# Conf values needed : -# analyzed_filename -# domain_name -# locales_path -# compress_output_files* -# -# Output files : -# DB_ROOT/meta.db -# DB_ROOT/year/month/iwla.db -# OUTPUT_ROOT/index.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# -# meta : -# last_time -# start_analysis_time -# stats => -# year => -# month => -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# nb_visitors -# -# month_stats : -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# -# days_stats : -# day => -# viewed_bandwidth -# not_viewed_bandwidth -# viewed_pages -# viewed_hits -# nb_visits -# nb_visitors -# -# visits : -# remote_addr => -# remote_addr -# remote_ip -# viewed_pages -# viewed_hits -# not_viewed_pages -# not_viewed_hits -# bandwidth -# last_access -# requests => -# [fields_from_format_log] -# extract_request => -# extract_uri -# extract_parameters* -# extract_referer* => -# extract_uri -# extract_parameters* -# robot -# hit_only -# is_page -# -# valid_visitors: -# month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors) -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Main class IWLA +Parse Log, compute them, call plugins and produce output +For now, only HTTP log are valid + +Plugin requirements : + None + +Conf values needed : + analyzed_filename + domain_name + locales_path + compress_output_files* + +Output files : + DB_ROOT/meta.db + DB_ROOT/year/month/iwla.db + OUTPUT_ROOT/index.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + +meta : + last_time + start_analysis_time + stats => + year => + month => + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + nb_visitors + +month_stats : + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + +days_stats : + day => + viewed_bandwidth + not_viewed_bandwidth + viewed_pages + viewed_hits + nb_visits + nb_visitors + +visits : + remote_addr => + remote_addr + remote_ip + viewed_pages + viewed_hits + not_viewed_pages + not_viewed_hits + bandwidth + last_access + requests => + [fields_from_format_log] + extract_request => + extract_uri + extract_parameters* + extract_referer* => + extract_uri + extract_parameters* + robot + hit_only + is_page + +valid_visitors: + month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors) + +Statistics update : + None + +Statistics deletion : + None +""" + class IWLA(object): diff --git a/plugins/display/all_visits.py b/plugins/display/all_visits.py index 442f4e2..4f4dd52 100644 --- a/plugins/display/all_visits.py +++ b/plugins/display/all_visits.py @@ -24,30 +24,30 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create All visits page -# -# Plugin requirements : -# None -# -# Conf values needed : -# display_visitor_ip* -# -# Output files : -# OUTPUT_ROOT/year/month/all_visits.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create All visits page + +Plugin requirements : + None + +Conf values needed : + display_visitor_ip* + +Output files : + OUTPUT_ROOT/year/month/all_visits.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayAllVisits(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/referers.py b/plugins/display/referers.py index cbdbeea..bdd04a5 100644 --- a/plugins/display/referers.py +++ b/plugins/display/referers.py @@ -22,34 +22,34 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create Referers page -# -# Plugin requirements : -# post_analysis/referers -# -# Conf values needed : -# max_referers_displayed* -# create_all_referers_page* -# max_key_phrases_displayed* -# create_all_key_phrases_page* -# -# Output files : -# OUTPUT_ROOT/year/month/referers.html -# OUTPUT_ROOT/year/month/key_phrases.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create Referers page + +Plugin requirements : + post_analysis/referers + +Conf values needed : + max_referers_displayed* + create_all_referers_page* + max_key_phrases_displayed* + create_all_key_phrases_page* + +Output files : + OUTPUT_ROOT/year/month/referers.html + OUTPUT_ROOT/year/month/key_phrases.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayReferers(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_downloads.py b/plugins/display/top_downloads.py index b97c6a7..f5d4a60 100644 --- a/plugins/display/top_downloads.py +++ b/plugins/display/top_downloads.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP downloads page -# -# Plugin requirements : -# post_analysis/top_downloads -# -# Conf values needed : -# max_downloads_displayed* -# create_all_downloads_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_downloads.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP downloads page + +Plugin requirements : + post_analysis/top_downloads + +Conf values needed : + max_downloads_displayed* + create_all_downloads_page* + +Output files : + OUTPUT_ROOT/year/month/top_downloads.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopDownloads(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_hits.py b/plugins/display/top_hits.py index f080004..412b3fe 100644 --- a/plugins/display/top_hits.py +++ b/plugins/display/top_hits.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP hits page -# -# Plugin requirements : -# post_analysis/top_hits -# -# Conf values needed : -# max_hits_displayed* -# create_all_hits_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_hits.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP hits page + +Plugin requirements : + post_analysis/top_hits + +Conf values needed : + max_hits_displayed* + create_all_hits_page* + +Output files : + OUTPUT_ROOT/year/month/top_hits.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopHits(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_pages.py b/plugins/display/top_pages.py index 53378ce..b44bf81 100644 --- a/plugins/display/top_pages.py +++ b/plugins/display/top_pages.py @@ -22,31 +22,31 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP pages page -# -# Plugin requirements : -# post_analysis/top_pages -# -# Conf values needed : -# max_pages_displayed* -# create_all_pages_page* -# -# Output files : -# OUTPUT_ROOT/year/month/top_pages.html -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP pages page + +Plugin requirements : + post_analysis/top_pages + +Conf values needed : + max_pages_displayed* + create_all_pages_page* + +Output files : + OUTPUT_ROOT/year/month/top_pages.html + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopPages(IPlugin): def __init__(self, iwla): diff --git a/plugins/display/top_visitors.py b/plugins/display/top_visitors.py index a48255b..b7d3b66 100644 --- a/plugins/display/top_visitors.py +++ b/plugins/display/top_visitors.py @@ -24,29 +24,29 @@ from iwla import IWLA from iplugin import IPlugin from display import * -# -# Display hook -# -# Create TOP visitors block -# -# Plugin requirements : -# None -# -# Conf values needed : -# display_visitor_ip* -# -# Output files : -# OUTPUT_ROOT/year/month/index.html -# -# Statistics creation : -# None -# -# Statistics update : -# None -# -# Statistics deletion : -# None -# +""" +Display hook + +Create TOP visitors block + +Plugin requirements : + None + +Conf values needed : + display_visitor_ip* + +Output files : + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" class IWLADisplayTopVisitors(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/referers.py b/plugins/post_analysis/referers.py index 4a44b24..64ec66f 100644 --- a/plugins/post_analysis/referers.py +++ b/plugins/post_analysis/referers.py @@ -26,40 +26,40 @@ from iplugin import IPlugin import awstats_data -# -# Post analysis hook -# -# Extract referers and key phrases from requests -# -# Plugin requirements : -# None -# -# Conf values needed : -# domain_name -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats : -# referers => -# pages -# hits -# robots_referers => -# pages -# hits -# search_engine_referers => -# pages -# hits -# key_phrases => -# phrase -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Extract referers and key phrases from requests + +Plugin requirements : + None + +Conf values needed : + domain_name + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats : + referers => + pages + hits + robots_referers => + pages + hits + search_engine_referers => + pages + hits + key_phrases => + phrase + +Statistics deletion : + None +""" class IWLAPostAnalysisReferers(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/reverse_dns.py b/plugins/post_analysis/reverse_dns.py index 6def2ff..c63965f 100644 --- a/plugins/post_analysis/reverse_dns.py +++ b/plugins/post_analysis/reverse_dns.py @@ -23,32 +23,32 @@ import socket from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Replace IP by reverse DNS names -# -# Plugin requirements : -# None -# -# Conf values needed : -# reverse_dns_timeout* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# valid_visitors: -# remote_addr -# dns_name_replaced -# dns_analyzed -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Replace IP by reverse DNS names + +Plugin requirements : + None + +Conf values needed : + reverse_dns_timeout* + +Output files : + None + +Statistics creation : + None + +Statistics update : +valid_visitors: + remote_addr + dns_name_replaced + dns_analyzed + +Statistics deletion : + None +""" class IWLAPostAnalysisReverseDNS(IPlugin): DEFAULT_DNS_TIMEOUT = 0.5 diff --git a/plugins/post_analysis/top_downloads.py b/plugins/post_analysis/top_downloads.py index 31bb112..7b28c33 100644 --- a/plugins/post_analysis/top_downloads.py +++ b/plugins/post_analysis/top_downloads.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP downloads -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_downloads => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP downloads + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_downloads => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopDownloads(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/top_hits.py b/plugins/post_analysis/top_hits.py index 2f056b2..64446c7 100644 --- a/plugins/post_analysis/top_hits.py +++ b/plugins/post_analysis/top_hits.py @@ -21,31 +21,31 @@ from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP hits -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_hits => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP hits + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_hits => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopHits(IPlugin): def __init__(self, iwla): diff --git a/plugins/post_analysis/top_pages.py b/plugins/post_analysis/top_pages.py index e47413e..a5d086c 100644 --- a/plugins/post_analysis/top_pages.py +++ b/plugins/post_analysis/top_pages.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Post analysis hook -# -# Count TOP pages -# -# Plugin requirements : -# None -# -# Conf values needed : -# None -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# month_stats: -# top_pages => -# uri -# -# Statistics deletion : -# None -# +""" +Post analysis hook + +Count TOP pages + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : + None + +Statistics update : +month_stats: + top_pages => + uri + +Statistics deletion : + None +""" class IWLAPostAnalysisTopPages(IPlugin): def __init__(self, iwla): diff --git a/plugins/pre_analysis/page_to_hit.py b/plugins/pre_analysis/page_to_hit.py index 9093c66..a3919c4 100644 --- a/plugins/pre_analysis/page_to_hit.py +++ b/plugins/pre_analysis/page_to_hit.py @@ -23,31 +23,31 @@ import re from iwla import IWLA from iplugin import IPlugin -# -# Pre analysis hook -# Change page into hit and hit into page into statistics -# -# Plugin requirements : -# None -# -# Conf values needed : -# page_to_hit_conf* -# hit_to_page_conf* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# visits : -# remote_addr => -# is_page -# -# Statistics deletion : -# None -# +""" +Pre analysis hook +Change page into hit and hit into page into statistics + +Plugin requirements : + None + +Conf values needed : + page_to_hit_conf* + hit_to_page_conf* + +Output files : + None + +Statistics creation : + None + +Statistics update : +visits : + remote_addr => + is_page + +Statistics deletion : + None +""" class IWLAPreAnalysisPageToHit(IPlugin): diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index a311588..662ce57 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -25,32 +25,32 @@ from iplugin import IPlugin import awstats_data -# -# Pre analysis hook -# -# Filter robots -# -# Plugin requirements : -# None -# -# Conf values needed : -# page_to_hit_conf* -# hit_to_page_conf* -# -# Output files : -# None -# -# Statistics creation : -# None -# -# Statistics update : -# visits : -# remote_addr => -# robot -# -# Statistics deletion : -# None -# +""" +Pre analysis hook + +Filter robots + +Plugin requirements : + None + +Conf values needed : + page_to_hit_conf* + hit_to_page_conf* + +Output files : + None + +Statistics creation : + None + +Statistics update : +visits : + remote_addr => + robot + +Statistics deletion : + None +""" class IWLAPreAnalysisRobots(IPlugin): def __init__(self, iwla): diff --git a/iwla_convert.pl b/tools/iwla_convert.pl similarity index 100% rename from iwla_convert.pl rename to tools/iwla_convert.pl