soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Replace # for module description by """ (help auto extraction)

Browse Source
This commit is contained in:
Grégory Soutadé 2014-12-19 11:34:25 +01:00
parent 2f05a70ee5
commit a35d462cb7
16 changed files with 422 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
default_conf.py
View File

@ -2,9 +2,9 @@ import os
# Default configuration
# Default directory where to store database information
# Default database directory
DB_ROOT = './output_db'
# Default directory where to create html files
# Default HTML output directory
DISPLAY_ROOT = './output'
# Hooks directories (don't edit)
@ -49,7 +49,7 @@ icon_path = '%s/%s' % (os.path.basename(resources_path[0]), 'icon')
# CSS path (you can add yours)
css_path = ['%s/%s/%s' % (os.path.basename(resources_path[0]), 'css', 'iwla.css')]
# Extensions to compress in gzip during display build
# Extensions to compress in gzip during display build
compress_output_files = []
# Path to locales files

165
iwla.py
View File

@ -41,88 +41,89 @@ del _
from iplugin import *
from display import *
#
# Main class IWLA
# Parse Log, compute them, call plugins and produce output
# For now, only HTTP log are valid
#
# Plugin requirements :
# None
#
# Conf values needed :
# analyzed_filename
# domain_name
# locales_path
# compress_output_files*
#
# Output files :
# DB_ROOT/meta.db
# DB_ROOT/year/month/iwla.db
# OUTPUT_ROOT/index.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
#
# meta :
# last_time
# start_analysis_time
# stats =>
# year =>
# month =>
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
# nb_visitors
#
# month_stats :
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
#
# days_stats :
# day =>
# viewed_bandwidth
# not_viewed_bandwidth
# viewed_pages
# viewed_hits
# nb_visits
# nb_visitors
#
# visits :
# remote_addr =>
# remote_addr
# remote_ip
# viewed_pages
# viewed_hits
# not_viewed_pages
# not_viewed_hits
# bandwidth
# last_access
# requests =>
# [fields_from_format_log]
# extract_request =>
# extract_uri
# extract_parameters*
# extract_referer* =>
# extract_uri
# extract_parameters*
# robot
# hit_only
# is_page
#
# valid_visitors:
# month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Main class IWLA
Parse Log, compute them, call plugins and produce output
For now, only HTTP log are valid
Plugin requirements :
None
Conf values needed :
analyzed_filename
domain_name
locales_path
compress_output_files*
Output files :
DB_ROOT/meta.db
DB_ROOT/year/month/iwla.db
OUTPUT_ROOT/index.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
meta :
last_time
start_analysis_time
stats =>
year =>
month =>
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
nb_visitors
month_stats :
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
days_stats :
day =>
viewed_bandwidth
not_viewed_bandwidth
viewed_pages
viewed_hits
nb_visits
nb_visitors
visits :
remote_addr =>
remote_addr
remote_ip
viewed_pages
viewed_hits
not_viewed_pages
not_viewed_hits
bandwidth
last_access
requests =>
[fields_from_format_log]
extract_request =>
extract_uri
extract_parameters*
extract_referer* =>
extract_uri
extract_parameters*
robot
hit_only
is_page
valid_visitors:
month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
Statistics update :
None
Statistics deletion :
None
"""
class IWLA(object):

48
plugins/display/all_visits.py
View File

@ -24,30 +24,30 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create All visits page
#
# Plugin requirements :
# None
#
# Conf values needed :
# display_visitor_ip*
#
# Output files :
# OUTPUT_ROOT/year/month/all_visits.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create All visits page
Plugin requirements :
None
Conf values needed :
display_visitor_ip*
Output files :
OUTPUT_ROOT/year/month/all_visits.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayAllVisits(IPlugin):
def __init__(self, iwla):

56
plugins/display/referers.py
View File

@ -22,34 +22,34 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create Referers page
#
# Plugin requirements :
# post_analysis/referers
#
# Conf values needed :
# max_referers_displayed*
# create_all_referers_page*
# max_key_phrases_displayed*
# create_all_key_phrases_page*
#
# Output files :
# OUTPUT_ROOT/year/month/referers.html
# OUTPUT_ROOT/year/month/key_phrases.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create Referers page
Plugin requirements :
post_analysis/referers
Conf values needed :
max_referers_displayed*
create_all_referers_page*
max_key_phrases_displayed*
create_all_key_phrases_page*
Output files :
OUTPUT_ROOT/year/month/referers.html
OUTPUT_ROOT/year/month/key_phrases.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayReferers(IPlugin):
def __init__(self, iwla):

50
plugins/display/top_downloads.py
View File

@ -22,31 +22,31 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP downloads page
#
# Plugin requirements :
# post_analysis/top_downloads
#
# Conf values needed :
# max_downloads_displayed*
# create_all_downloads_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_downloads.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP downloads page
Plugin requirements :
post_analysis/top_downloads
Conf values needed :
max_downloads_displayed*
create_all_downloads_page*
Output files :
OUTPUT_ROOT/year/month/top_downloads.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopDownloads(IPlugin):
def __init__(self, iwla):

50
plugins/display/top_hits.py
View File

@ -22,31 +22,31 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP hits page
#
# Plugin requirements :
# post_analysis/top_hits
#
# Conf values needed :
# max_hits_displayed*
# create_all_hits_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_hits.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP hits page
Plugin requirements :
post_analysis/top_hits
Conf values needed :
max_hits_displayed*
create_all_hits_page*
Output files :
OUTPUT_ROOT/year/month/top_hits.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopHits(IPlugin):
def __init__(self, iwla):

50
plugins/display/top_pages.py
View File

@ -22,31 +22,31 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP pages page
#
# Plugin requirements :
# post_analysis/top_pages
#
# Conf values needed :
# max_pages_displayed*
# create_all_pages_page*
#
# Output files :
# OUTPUT_ROOT/year/month/top_pages.html
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP pages page
Plugin requirements :
post_analysis/top_pages
Conf values needed :
max_pages_displayed*
create_all_pages_page*
Output files :
OUTPUT_ROOT/year/month/top_pages.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopPages(IPlugin):
def __init__(self, iwla):

46
plugins/display/top_visitors.py
View File

@ -24,29 +24,29 @@ from iwla import IWLA
from iplugin import IPlugin
from display import *
#
# Display hook
#
# Create TOP visitors block
#
# Plugin requirements :
# None
#
# Conf values needed :
# display_visitor_ip*
#
# Output files :
# OUTPUT_ROOT/year/month/index.html
#
# Statistics creation :
# None
#
# Statistics update :
# None
#
# Statistics deletion :
# None
#
"""
Display hook
Create TOP visitors block
Plugin requirements :
None
Conf values needed :
display_visitor_ip*
Output files :
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayTopVisitors(IPlugin):
def __init__(self, iwla):

68
plugins/post_analysis/referers.py
View File

@ -26,40 +26,40 @@ from iplugin import IPlugin
import awstats_data
#
# Post analysis hook
#
# Extract referers and key phrases from requests
#
# Plugin requirements :
# None
#
# Conf values needed :
# domain_name
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats :
# referers =>
# pages
# hits
# robots_referers =>
# pages
# hits
# search_engine_referers =>
# pages
# hits
# key_phrases =>
# phrase
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Extract referers and key phrases from requests
Plugin requirements :
None
Conf values needed :
domain_name
Output files :
None
Statistics creation :
None
Statistics update :
month_stats :
referers =>
pages
hits
robots_referers =>
pages
hits
search_engine_referers =>
pages
hits
key_phrases =>
phrase
Statistics deletion :
None
"""
class IWLAPostAnalysisReferers(IPlugin):
def __init__(self, iwla):

52
plugins/post_analysis/reverse_dns.py
View File

@ -23,32 +23,32 @@ import socket
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Replace IP by reverse DNS names
#
# Plugin requirements :
# None
#
# Conf values needed :
# reverse_dns_timeout*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# valid_visitors:
# remote_addr
# dns_name_replaced
# dns_analyzed
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Replace IP by reverse DNS names
Plugin requirements :
None
Conf values needed :
reverse_dns_timeout*
Output files :
None
Statistics creation :
None
Statistics update :
valid_visitors:
remote_addr
dns_name_replaced
dns_analyzed
Statistics deletion :
None
"""
class IWLAPostAnalysisReverseDNS(IPlugin):
DEFAULT_DNS_TIMEOUT = 0.5

50
plugins/post_analysis/top_downloads.py
View File

@ -23,31 +23,31 @@ import re
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP downloads
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_downloads =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP downloads
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_downloads =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopDownloads(IPlugin):
def __init__(self, iwla):

50
plugins/post_analysis/top_hits.py
View File

@ -21,31 +21,31 @@
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP hits
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_hits =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP hits
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_hits =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopHits(IPlugin):
def __init__(self, iwla):

50
plugins/post_analysis/top_pages.py
View File

@ -23,31 +23,31 @@ import re
from iwla import IWLA
from iplugin import IPlugin
#
# Post analysis hook
#
# Count TOP pages
#
# Plugin requirements :
# None
#
# Conf values needed :
# None
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# month_stats:
# top_pages =>
# uri
#
# Statistics deletion :
# None
#
"""
Post analysis hook
Count TOP pages
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
month_stats:
top_pages =>
uri
Statistics deletion :
None
"""
class IWLAPostAnalysisTopPages(IPlugin):
def __init__(self, iwla):

50
plugins/pre_analysis/page_to_hit.py
View File

@ -23,31 +23,31 @@ import re
from iwla import IWLA
from iplugin import IPlugin
#
# Pre analysis hook
# Change page into hit and hit into page into statistics
#
# Plugin requirements :
# None
#
# Conf values needed :
# page_to_hit_conf*
# hit_to_page_conf*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# visits :
# remote_addr =>
# is_page
#
# Statistics deletion :
# None
#
"""
Pre analysis hook
Change page into hit and hit into page into statistics
Plugin requirements :
None
Conf values needed :
page_to_hit_conf*
hit_to_page_conf*
Output files :
None
Statistics creation :
None
Statistics update :
visits :
remote_addr =>
is_page
Statistics deletion :
None
"""
class IWLAPreAnalysisPageToHit(IPlugin):

52
plugins/pre_analysis/robots.py
View File

@ -25,32 +25,32 @@ from iplugin import IPlugin
import awstats_data
#
# Pre analysis hook
#
# Filter robots
#
# Plugin requirements :
# None
#
# Conf values needed :
# page_to_hit_conf*
# hit_to_page_conf*
#
# Output files :
# None
#
# Statistics creation :
# None
#
# Statistics update :
# visits :
# remote_addr =>
# robot
#
# Statistics deletion :
# None
#
"""
Pre analysis hook
Filter robots
Plugin requirements :
None
Conf values needed :
page_to_hit_conf*
hit_to_page_conf*
Output files :
None
Statistics creation :
None
Statistics update :
visits :
remote_addr =>
robot
Statistics deletion :
None
"""
class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):

0
iwla_convert.pl → tools/iwla_convert.pl
View File