840 lines
15 KiB
Markdown
840 lines
15 KiB
Markdown
* iwla.py
|
|
* plugins/display/all_visits.py
|
|
* plugins/display/browsers.py
|
|
* plugins/display/feeds.py
|
|
* plugins/display/hours_stats.py
|
|
* plugins/display/istats_diff.py
|
|
* plugins/display/operating_systems.py
|
|
* plugins/display/referers.py
|
|
* plugins/display/referers_diff.py
|
|
* plugins/display/top_downloads.py
|
|
* plugins/display/top_downloads_diff.py
|
|
* plugins/display/top_hits.py
|
|
* plugins/display/top_pages.py
|
|
* plugins/display/top_visitors.py
|
|
* plugins/display/track_users.py
|
|
* plugins/post_analysis/browsers.py
|
|
* plugins/post_analysis/feeds.py
|
|
* plugins/post_analysis/hours_stats.py
|
|
* plugins/post_analysis/operating_systems.py
|
|
* plugins/post_analysis/referers.py
|
|
* plugins/post_analysis/reverse_dns.py
|
|
* plugins/post_analysis/top_downloads.py
|
|
* plugins/post_analysis/top_hits.py
|
|
* plugins/post_analysis/top_pages.py
|
|
* plugins/pre_analysis/page_to_hit.py
|
|
* plugins/pre_analysis/robots.py
|
|
|
|
|
|
iwla
|
|
----
|
|
|
|
Main class IWLA
|
|
Parse Log, compute them, call plugins and produce output
|
|
For now, only HTTP log are valid
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
analyzed_filename
|
|
domain_name
|
|
locales_path
|
|
compress_output_files*
|
|
|
|
Output files :
|
|
DB_ROOT/meta.db
|
|
DB_ROOT/year/month/iwla.db
|
|
OUTPUT_ROOT/index.html
|
|
OUTPUT_ROOT/year/_stats.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
|
|
meta :
|
|
last_time
|
|
start_analysis_time
|
|
stats =>
|
|
year =>
|
|
month =>
|
|
viewed_bandwidth
|
|
not_viewed_bandwidth
|
|
viewed_pages
|
|
viewed_hits
|
|
nb_visits
|
|
nb_visitors
|
|
|
|
month_stats :
|
|
viewed_bandwidth
|
|
not_viewed_bandwidth
|
|
viewed_pages
|
|
viewed_hits
|
|
nb_visits
|
|
|
|
days_stats :
|
|
day =>
|
|
viewed_bandwidth
|
|
not_viewed_bandwidth
|
|
viewed_pages
|
|
viewed_hits
|
|
nb_visits
|
|
nb_visitors
|
|
|
|
visits :
|
|
remote_addr =>
|
|
remote_addr
|
|
remote_ip
|
|
viewed_pages
|
|
viewed_hits
|
|
not_viewed_pages
|
|
not_viewed_hits
|
|
bandwidth
|
|
last_access
|
|
requests =>
|
|
[fields_from_format_log]
|
|
extract_request =>
|
|
http_method
|
|
http_uri
|
|
http_version
|
|
extract_uri
|
|
extract_parameters*
|
|
extract_referer* =>
|
|
extract_uri
|
|
extract_parameters*
|
|
robot
|
|
hit_only
|
|
is_page
|
|
|
|
valid_visitors:
|
|
month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.all_visits
|
|
--------------------------
|
|
|
|
Display hook
|
|
|
|
Create All visits page
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
display_visitor_ip*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/all_visits.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.browsers
|
|
------------------------
|
|
|
|
Display hook
|
|
|
|
Create browsers page
|
|
|
|
Plugin requirements :
|
|
post_analysis/browsers
|
|
|
|
Conf values needed :
|
|
max_browsers_displayed*
|
|
create_browsers_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/browsers.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.feeds
|
|
---------------------
|
|
|
|
Display hook
|
|
|
|
Display feeds parsers
|
|
|
|
Plugin requirements :
|
|
post_analysis/feeds
|
|
|
|
Conf values needed :
|
|
create_all_feeds_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/index.html
|
|
OUTPUT_ROOT/year/month/all_feeds.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.hours_stats
|
|
---------------------------
|
|
|
|
Display hook
|
|
|
|
Display statistics by hour/week day
|
|
|
|
Plugin requirements :
|
|
post_analysis/hours_stats
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.istats_diff
|
|
---------------------------
|
|
|
|
Display hook interface
|
|
|
|
Enlight new and updated statistics
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.operating_systems
|
|
---------------------------------
|
|
|
|
Display hook
|
|
|
|
Add operating systems statistics
|
|
|
|
Plugin requirements :
|
|
post_analysis/operating_systems
|
|
|
|
Conf values needed :
|
|
create_families_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.referers
|
|
------------------------
|
|
|
|
Display hook
|
|
|
|
Create Referers page
|
|
|
|
Plugin requirements :
|
|
post_analysis/referers
|
|
|
|
Conf values needed :
|
|
max_referers_displayed*
|
|
create_all_referers_page*
|
|
max_key_phrases_displayed*
|
|
create_all_key_phrases_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/referers.html
|
|
OUTPUT_ROOT/year/month/key_phrases.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.referers_diff
|
|
-----------------------------
|
|
|
|
Display hook
|
|
|
|
Enlight new and updated key phrases in in all_key_phrases.html
|
|
|
|
Plugin requirements :
|
|
display/referers
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.top_downloads
|
|
-----------------------------
|
|
|
|
Display hook
|
|
|
|
Create TOP downloads page
|
|
|
|
Plugin requirements :
|
|
post_analysis/top_downloads
|
|
|
|
Conf values needed :
|
|
max_downloads_displayed*
|
|
create_all_downloads_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/top_downloads.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.top_downloads_diff
|
|
----------------------------------
|
|
|
|
Display hook
|
|
|
|
Enlight new and updated downloads in in top_downloads.html
|
|
|
|
Plugin requirements :
|
|
display/top_downloads
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.top_hits
|
|
------------------------
|
|
|
|
Display hook
|
|
|
|
Create TOP hits page
|
|
|
|
Plugin requirements :
|
|
post_analysis/top_hits
|
|
|
|
Conf values needed :
|
|
max_hits_displayed*
|
|
create_all_hits_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/top_hits.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.top_pages
|
|
-------------------------
|
|
|
|
Display hook
|
|
|
|
Create TOP pages page
|
|
|
|
Plugin requirements :
|
|
post_analysis/top_pages
|
|
|
|
Conf values needed :
|
|
max_pages_displayed*
|
|
create_all_pages_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/top_pages.html
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.top_visitors
|
|
----------------------------
|
|
|
|
Display hook
|
|
|
|
Create TOP visitors block
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
display_visitor_ip*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/index.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.display.track_users
|
|
---------------------------
|
|
|
|
Display hook
|
|
|
|
Track users
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
tracked_ip
|
|
create_tracked_page*
|
|
|
|
Output files :
|
|
OUTPUT_ROOT/year/month/index.html
|
|
OUTPUT_ROOT/year/month/tracked_users.html
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.browsers
|
|
------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Detect browser information from requests
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
visits :
|
|
remote_addr =>
|
|
browser
|
|
|
|
month_stats :
|
|
browsers =>
|
|
browser => count
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.feeds
|
|
---------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Find feeds parsers (first hit in feeds conf value and no viewed pages if it's a robot)
|
|
If there is ony one hit per day to a feed, merge feeds parsers with the same user agent
|
|
as it must be the same person with a different IP address.
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
feeds
|
|
merge_one_hit_only_feeds_parsers*
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
remote_addr =>
|
|
feed_parser
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.hours_stats
|
|
---------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Count pages, hits and bandwidth by hour/week day
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
month_stats:
|
|
hours_stats =>
|
|
00 .. 23 =>
|
|
pages
|
|
hits
|
|
bandwidth
|
|
|
|
days_stats =>
|
|
0 .. 6 =>
|
|
pages
|
|
hits
|
|
bandwidth
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.operating_systems
|
|
---------------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Detect operating systems from requests
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
visits :
|
|
remote_addr =>
|
|
operating_system
|
|
|
|
month_stats :
|
|
operating_systems =>
|
|
operating_system => count
|
|
|
|
os_families =>
|
|
family => count
|
|
|
|
Statistics update :
|
|
None
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.referers
|
|
------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Extract referers and key phrases from requests
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
domain_name
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
month_stats :
|
|
referers =>
|
|
pages => count
|
|
hits => count
|
|
robots_referers =>
|
|
pages => count
|
|
hits => count
|
|
search_engine_referers =>
|
|
pages => count
|
|
hits => count
|
|
key_phrases =>
|
|
phrase => count
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.reverse_dns
|
|
---------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Replace IP by reverse DNS names
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
reverse_dns_timeout*
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
valid_visitors:
|
|
remote_addr
|
|
dns_name_replaced
|
|
dns_analyzed
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.top_downloads
|
|
-----------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Count TOP downloads
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
month_stats:
|
|
top_downloads =>
|
|
uri => count
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.top_hits
|
|
------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Count TOP hits
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
month_stats:
|
|
top_hits =>
|
|
uri => count
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.post_analysis.top_pages
|
|
-------------------------------
|
|
|
|
Post analysis hook
|
|
|
|
Count TOP pages
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
None
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
month_stats:
|
|
top_pages =>
|
|
uri => count
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.pre_analysis.page_to_hit
|
|
--------------------------------
|
|
|
|
Pre analysis hook
|
|
Change page into hit and hit into page into statistics
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
page_to_hit_conf*
|
|
hit_to_page_conf*
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
visits :
|
|
remote_addr =>
|
|
is_page
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|
|
plugins.pre_analysis.robots
|
|
---------------------------
|
|
|
|
Pre analysis hook
|
|
|
|
Filter robots
|
|
|
|
Plugin requirements :
|
|
None
|
|
|
|
Conf values needed :
|
|
page_to_hit_conf*
|
|
hit_to_page_conf*
|
|
|
|
Output files :
|
|
None
|
|
|
|
Statistics creation :
|
|
None
|
|
|
|
Statistics update :
|
|
visits :
|
|
remote_addr =>
|
|
robot
|
|
|
|
Statistics deletion :
|
|
None
|
|
|
|
|