From 2b0fd9fb46e833ae62b5845bd2c1a07722841fcc Mon Sep 17 00:00:00 2001 From: Gregory Soutade Date: Wed, 31 Dec 2014 14:52:14 +0100 Subject: [PATCH] Generate index table in documentation --- docs/index.md | 424 +++++++++++++++++++++++------------------- docs/modules.md | 424 +++++++++++++++++++++++------------------- tools/extract_doc.py | 9 + tools/extract_docs.sh | 7 +- 4 files changed, 483 insertions(+), 381 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9fa98a3..6b8c2b1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,6 +90,23 @@ Plugins Optional configuration values ends with *. + * iwla.py + * plugins/display/all_visits.py + * plugins/display/referers.py + * plugins/display/top_downloads.py + * plugins/display/top_hits.py + * plugins/display/top_pages.py + * plugins/display/top_visitors.py + * plugins/display/referers_diff.py + * plugins/post_analysis/referers.py + * plugins/post_analysis/reverse_dns.py + * plugins/post_analysis/top_downloads.py + * plugins/post_analysis/top_hits.py + * plugins/post_analysis/top_pages.py + * plugins/pre_analysis/page_to_hit.py + * plugins/pre_analysis/robots.py + + iwla ---- @@ -110,6 +127,7 @@ iwla DB_ROOT/meta.db DB_ROOT/year/month/iwla.db OUTPUT_ROOT/index.html + OUTPUT_ROOT/year/_stats.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -233,48 +251,22 @@ plugins.display.referers None -plugins.display.top_visitors ----------------------------- +plugins.display.top_downloads +----------------------------- Display hook - Create TOP visitors block + Create TOP downloads page Plugin requirements : - None + post_analysis/top_downloads Conf values needed : - display_visitor_ip* + max_downloads_displayed* + create_all_downloads_page* Output files : - OUTPUT_ROOT/year/month/index.html - - Statistics creation : - None - - Statistics update : - None - - Statistics deletion : - None - - -plugins.display.top_pages -------------------------- - - Display hook - - Create TOP pages page - - Plugin requirements : - post_analysis/top_pages - - Conf values needed : - max_pages_displayed* - create_all_pages_page* - - Output files : - OUTPUT_ROOT/year/month/top_pages.html + OUTPUT_ROOT/year/month/top_downloads.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -315,22 +307,22 @@ plugins.display.top_hits None -plugins.display.top_downloads ------------------------------ +plugins.display.top_pages +------------------------- Display hook - Create TOP downloads page + Create TOP pages page Plugin requirements : - post_analysis/top_downloads + post_analysis/top_pages Conf values needed : - max_downloads_displayed* - create_all_downloads_page* + max_pages_displayed* + create_all_pages_page* Output files : - OUTPUT_ROOT/year/month/top_downloads.html + OUTPUT_ROOT/year/month/top_pages.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -343,6 +335,208 @@ plugins.display.top_downloads None +plugins.display.top_visitors +---------------------------- + + Display hook + + Create TOP visitors block + + Plugin requirements : + None + + Conf values needed : + display_visitor_ip* + + Output files : + OUTPUT_ROOT/year/month/index.html + + Statistics creation : + None + + Statistics update : + None + + Statistics deletion : + None + + +plugins.display.referers_diff +----------------------------- + + Display hook + + Enlight new and updated key phrases in in all_key_phrases.html + + Plugin requirements : + display/referers + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + None + + Statistics deletion : + None + + +plugins.post_analysis.referers +------------------------------ + + Post analysis hook + + Extract referers and key phrases from requests + + Plugin requirements : + None + + Conf values needed : + domain_name + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats : + referers => + pages + hits + robots_referers => + pages + hits + search_engine_referers => + pages + hits + key_phrases => + phrase + + Statistics deletion : + None + + +plugins.post_analysis.reverse_dns +--------------------------------- + + Post analysis hook + + Replace IP by reverse DNS names + + Plugin requirements : + None + + Conf values needed : + reverse_dns_timeout* + + Output files : + None + + Statistics creation : + None + + Statistics update : + valid_visitors: + remote_addr + dns_name_replaced + dns_analyzed + + Statistics deletion : + None + + +plugins.post_analysis.top_downloads +----------------------------------- + + Post analysis hook + + Count TOP downloads + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_downloads => + uri + + Statistics deletion : + None + + +plugins.post_analysis.top_hits +------------------------------ + + Post analysis hook + + Count TOP hits + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_hits => + uri + + Statistics deletion : + None + + +plugins.post_analysis.top_pages +------------------------------- + + Post analysis hook + + Count TOP pages + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_pages => + uri + + Statistics deletion : + None + + plugins.pre_analysis.page_to_hit -------------------------------- @@ -400,153 +594,3 @@ plugins.pre_analysis.robots None -plugins.post_analysis.referers ------------------------------- - - Post analysis hook - - Extract referers and key phrases from requests - - Plugin requirements : - None - - Conf values needed : - domain_name - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats : - referers => - pages - hits - robots_referers => - pages - hits - search_engine_referers => - pages - hits - key_phrases => - phrase - - Statistics deletion : - None - - -plugins.post_analysis.top_pages -------------------------------- - - Post analysis hook - - Count TOP pages - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_pages => - uri - - Statistics deletion : - None - - -plugins.post_analysis.reverse_dns ---------------------------------- - - Post analysis hook - - Replace IP by reverse DNS names - - Plugin requirements : - None - - Conf values needed : - reverse_dns_timeout* - - Output files : - None - - Statistics creation : - None - - Statistics update : - valid_visitors: - remote_addr - dns_name_replaced - dns_analyzed - - Statistics deletion : - None - - -plugins.post_analysis.top_hits ------------------------------- - - Post analysis hook - - Count TOP hits - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_hits => - uri - - Statistics deletion : - None - - -plugins.post_analysis.top_downloads ------------------------------------ - - Post analysis hook - - Count TOP downloads - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_downloads => - uri - - Statistics deletion : - None - - diff --git a/docs/modules.md b/docs/modules.md index 5067afb..7cf4999 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -1,3 +1,20 @@ + * iwla.py + * plugins/display/all_visits.py + * plugins/display/referers.py + * plugins/display/top_downloads.py + * plugins/display/top_hits.py + * plugins/display/top_pages.py + * plugins/display/top_visitors.py + * plugins/display/referers_diff.py + * plugins/post_analysis/referers.py + * plugins/post_analysis/reverse_dns.py + * plugins/post_analysis/top_downloads.py + * plugins/post_analysis/top_hits.py + * plugins/post_analysis/top_pages.py + * plugins/pre_analysis/page_to_hit.py + * plugins/pre_analysis/robots.py + + iwla ---- @@ -18,6 +35,7 @@ iwla DB_ROOT/meta.db DB_ROOT/year/month/iwla.db OUTPUT_ROOT/index.html + OUTPUT_ROOT/year/_stats.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -141,48 +159,22 @@ plugins.display.referers None -plugins.display.top_visitors ----------------------------- +plugins.display.top_downloads +----------------------------- Display hook - Create TOP visitors block + Create TOP downloads page Plugin requirements : - None + post_analysis/top_downloads Conf values needed : - display_visitor_ip* + max_downloads_displayed* + create_all_downloads_page* Output files : - OUTPUT_ROOT/year/month/index.html - - Statistics creation : - None - - Statistics update : - None - - Statistics deletion : - None - - -plugins.display.top_pages -------------------------- - - Display hook - - Create TOP pages page - - Plugin requirements : - post_analysis/top_pages - - Conf values needed : - max_pages_displayed* - create_all_pages_page* - - Output files : - OUTPUT_ROOT/year/month/top_pages.html + OUTPUT_ROOT/year/month/top_downloads.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -223,22 +215,22 @@ plugins.display.top_hits None -plugins.display.top_downloads ------------------------------ +plugins.display.top_pages +------------------------- Display hook - Create TOP downloads page + Create TOP pages page Plugin requirements : - post_analysis/top_downloads + post_analysis/top_pages Conf values needed : - max_downloads_displayed* - create_all_downloads_page* + max_pages_displayed* + create_all_pages_page* Output files : - OUTPUT_ROOT/year/month/top_downloads.html + OUTPUT_ROOT/year/month/top_pages.html OUTPUT_ROOT/year/month/index.html Statistics creation : @@ -251,6 +243,208 @@ plugins.display.top_downloads None +plugins.display.top_visitors +---------------------------- + + Display hook + + Create TOP visitors block + + Plugin requirements : + None + + Conf values needed : + display_visitor_ip* + + Output files : + OUTPUT_ROOT/year/month/index.html + + Statistics creation : + None + + Statistics update : + None + + Statistics deletion : + None + + +plugins.display.referers_diff +----------------------------- + + Display hook + + Enlight new and updated key phrases in in all_key_phrases.html + + Plugin requirements : + display/referers + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + None + + Statistics deletion : + None + + +plugins.post_analysis.referers +------------------------------ + + Post analysis hook + + Extract referers and key phrases from requests + + Plugin requirements : + None + + Conf values needed : + domain_name + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats : + referers => + pages + hits + robots_referers => + pages + hits + search_engine_referers => + pages + hits + key_phrases => + phrase + + Statistics deletion : + None + + +plugins.post_analysis.reverse_dns +--------------------------------- + + Post analysis hook + + Replace IP by reverse DNS names + + Plugin requirements : + None + + Conf values needed : + reverse_dns_timeout* + + Output files : + None + + Statistics creation : + None + + Statistics update : + valid_visitors: + remote_addr + dns_name_replaced + dns_analyzed + + Statistics deletion : + None + + +plugins.post_analysis.top_downloads +----------------------------------- + + Post analysis hook + + Count TOP downloads + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_downloads => + uri + + Statistics deletion : + None + + +plugins.post_analysis.top_hits +------------------------------ + + Post analysis hook + + Count TOP hits + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_hits => + uri + + Statistics deletion : + None + + +plugins.post_analysis.top_pages +------------------------------- + + Post analysis hook + + Count TOP pages + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + month_stats: + top_pages => + uri + + Statistics deletion : + None + + plugins.pre_analysis.page_to_hit -------------------------------- @@ -308,153 +502,3 @@ plugins.pre_analysis.robots None -plugins.post_analysis.referers ------------------------------- - - Post analysis hook - - Extract referers and key phrases from requests - - Plugin requirements : - None - - Conf values needed : - domain_name - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats : - referers => - pages - hits - robots_referers => - pages - hits - search_engine_referers => - pages - hits - key_phrases => - phrase - - Statistics deletion : - None - - -plugins.post_analysis.top_pages -------------------------------- - - Post analysis hook - - Count TOP pages - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_pages => - uri - - Statistics deletion : - None - - -plugins.post_analysis.reverse_dns ---------------------------------- - - Post analysis hook - - Replace IP by reverse DNS names - - Plugin requirements : - None - - Conf values needed : - reverse_dns_timeout* - - Output files : - None - - Statistics creation : - None - - Statistics update : - valid_visitors: - remote_addr - dns_name_replaced - dns_analyzed - - Statistics deletion : - None - - -plugins.post_analysis.top_hits ------------------------------- - - Post analysis hook - - Count TOP hits - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_hits => - uri - - Statistics deletion : - None - - -plugins.post_analysis.top_downloads ------------------------------------ - - Post analysis hook - - Count TOP downloads - - Plugin requirements : - None - - Conf values needed : - None - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats: - top_downloads => - uri - - Statistics deletion : - None - - diff --git a/tools/extract_doc.py b/tools/extract_doc.py index 46220eb..601b4cc 100755 --- a/tools/extract_doc.py +++ b/tools/extract_doc.py @@ -5,6 +5,11 @@ import sys excludes = ['istats_diff.py'] filename = sys.argv[1] +printName = False + +if filename == '-p': + filename = sys.argv[2] + printName = True if filename.endswith('__init__.py'): sys.exit(0) @@ -14,6 +19,10 @@ for e in excludes: sys.stderr.write('\tSkip %s\n' % (filename)) sys.exit(0) +if printName: + sys.stdout.write(' * %s\n' % (filename)) + sys.exit(0) + package_name = filename.replace('/', '.').replace('.py', '') sys.stdout.write('%s' % (package_name)) sys.stdout.write('\n') diff --git a/tools/extract_docs.sh b/tools/extract_docs.sh index e556330..007f21b 100755 --- a/tools/extract_docs.sh +++ b/tools/extract_docs.sh @@ -6,8 +6,13 @@ TARGET_MD="docs/index.md" rm -f "${MODULES_TARGET}" +echo "Generate plugins index" +python tools/extract_doc.py -p iwla.py > "${MODULES_TARGET}" +find plugins -name '*.py' -exec python tools/extract_doc.py -p \{\} \; >> "${MODULES_TARGET}" +echo "\n" >> "${MODULES_TARGET}" + echo "Generate doc from iwla.py" -python tools/extract_doc.py iwla.py > "${MODULES_TARGET}" +python tools/extract_doc.py iwla.py >> "${MODULES_TARGET}" echo "Generate plugins documentation" find plugins -name '*.py' -exec python tools/extract_doc.py \{\} \; >> "${MODULES_TARGET}"