diff --git a/docs/index.md b/docs/index.md index 1ce58a5..79ca06c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option). +Demo +---- + +A demonstration instance is available [here](https://iwla-demo.soutade.fr) + Usage ----- @@ -19,7 +24,7 @@ Usage -f : Analyse this log file, multiple files can be specified (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename -d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] -r : Reset analysis to a specific date (month/year) - -z : Don't compress databases (bigger but faster, not compatible with compressed databases) + -z : Don't compress databases (bigger but faster, not compatible with compressed databases) -p : Only generate display -d : Dry run (don't write/update files to disk) @@ -36,6 +41,8 @@ Main values to edit are : * **post_analysis_hooks** : List of post analysis hooks * **display_hooks** : List of display hooks * **locale** : Displayed locale (_en_ or _fr_) + * **feeds** : Address of your feeds files + * **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...) You can also append an element to an existing default configuration list by using "_append" suffix. Example : multimedia_files_append = ['xml'] @@ -82,7 +89,7 @@ Statistics are stored in dictionaries : * **month_stats** : Statistics of current analysed month * **valid_visitor** : A subset of month_stats without robots * **days_stats** : Statistics of current analysed day - * **visits** : All visitors with all of its requests + * **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered) * **meta** : Final result of month statistics (by year) Create a Plugins @@ -119,17 +126,15 @@ Optional configuration values ends with *. * plugins/display/top_pages_diff.py * plugins/display/top_pages.py * plugins/display/top_visitors.py + * plugins/post_analysis/anonymize_ip.py * plugins/post_analysis/browsers.py * plugins/post_analysis/feeds.py * plugins/post_analysis/filter_users.py - * plugins/post_analysis/google_console_api.py * plugins/post_analysis/hours_stats.py * plugins/post_analysis/ip_to_geo.py - * plugins/post_analysis/iptogeo.py * plugins/post_analysis/operating_systems.py * plugins/post_analysis/referers.py * plugins/post_analysis/reverse_dns.py - * plugins/post_analysis/search_analytics_api_sample.py * plugins/post_analysis/top_downloads.py * plugins/post_analysis/top_hits.py * plugins/post_analysis/top_pages.py @@ -688,6 +693,33 @@ plugins.display.top_visitors None +plugins.post_analysis.anonymize_ip +---------------------------------- + + Post analysis hook + + Replace remote_addr by a SHA1 + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + valid_visitors: + remote_addr + + Statistics deletion : + None + + plugins.post_analysis.browsers ------------------------------ @@ -807,34 +839,6 @@ plugins.post_analysis.filter_users None -plugins.post_analysis.google_console_api ----------------------------------------- - - Post analysis hook - - Extract key phrases from Google console API - - Plugin requirements : - None - - Conf values needed : - domain_name - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats : - key_phrases => - phrase => count - - Statistics deletion : - None - - plugins.post_analysis.hours_stats --------------------------------- @@ -902,11 +906,6 @@ plugins.post_analysis.ip_to_geo None -plugins.post_analysis.iptogeo ------------------------------ - - - plugins.post_analysis.operating_systems --------------------------------------- @@ -1008,141 +1007,6 @@ plugins.post_analysis.reverse_dns None -plugins.post_analysis.search_analytics_api_sample -------------------------------------------------- - - from __future__ import print_function - - import argparse - import sys - from googleapiclient import sample_tools - - # Declare command-line flags. - argparser = argparse.ArgumentParser(add_help=False) - argparser.add_argument('property_uri', type=str, - help=('Site or app URI to query data for (including ' - 'trailing slash).')) - argparser.add_argument('start_date', type=str, - help=('Start date of the requested date range in ' - 'YYYY-MM-DD format.')) - argparser.add_argument('end_date', type=str, - help=('End date of the requested date range in ' - 'YYYY-MM-DD format.')) - - - def main(argv): - service, flags = sample_tools.init( - argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser], - scope='https://www.googleapis.com/auth/webmasters.readonly') - - # First run a query to learn which dates we have data for. You should always - # check which days in a date range have data before running your main query. - # This query shows data for the entire range, grouped and sorted by day, - # descending; any days without data will be missing from the results. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'] - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Available dates') - - return - # Get totals for the date range. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Totals') - - # Get top 10 queries for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top Queries') - - # Get top 11-20 mobile queries for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'dimensionFilterGroups': [{ - 'filters': [{ - 'dimension': 'device', - 'expression': 'mobile' - }] - }], - 'rowLimit': 10, - 'startRow': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top 11-20 Mobile Queries') - - # Get top 10 pages for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['page'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top Pages') - - # Get the top 10 queries in India, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'dimensionFilterGroups': [{ - 'filters': [{ - 'dimension': 'country', - 'expression': 'ind' - }] - }], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top queries in India') - - # Group by both country and device. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['country', 'device'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Group by country and device') - - # Group by total number of Search Appearance count. - # Note: It is not possible to use searchAppearance with other - # dimensions. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['searchAppearance'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Search Appearance Features') - - def execute_request(service, property_uri, request): - """Executes a searchAnalytics.query request. - - Args: - service: The webmasters service to use when executing the query. - property_uri: The site or app URI to request data for. - request: The request to be executed. - - Returns: - An array of response rows. - - plugins.post_analysis.top_downloads ----------------------------------- diff --git a/docs/main.md b/docs/main.md index 535ab4c..9f8b1e1 100644 --- a/docs/main.md +++ b/docs/main.md @@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option). +Demo +---- + +A demonstration instance is available [here](https://iwla-demo.soutade.fr) + Usage ----- @@ -19,7 +24,7 @@ Usage -f : Analyse this log file, multiple files can be specified (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename -d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] -r : Reset analysis to a specific date (month/year) - -z : Don't compress databases (bigger but faster, not compatible with compressed databases) + -z : Don't compress databases (bigger but faster, not compatible with compressed databases) -p : Only generate display -d : Dry run (don't write/update files to disk) @@ -36,6 +41,8 @@ Main values to edit are : * **post_analysis_hooks** : List of post analysis hooks * **display_hooks** : List of display hooks * **locale** : Displayed locale (_en_ or _fr_) + * **feeds** : Address of your feeds files + * **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...) You can also append an element to an existing default configuration list by using "_append" suffix. Example : multimedia_files_append = ['xml'] @@ -82,7 +89,7 @@ Statistics are stored in dictionaries : * **month_stats** : Statistics of current analysed month * **valid_visitor** : A subset of month_stats without robots * **days_stats** : Statistics of current analysed day - * **visits** : All visitors with all of its requests + * **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered) * **meta** : Final result of month statistics (by year) Create a Plugins diff --git a/docs/modules.md b/docs/modules.md index 500a570..ce5c42c 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -16,17 +16,15 @@ * plugins/display/top_pages_diff.py * plugins/display/top_pages.py * plugins/display/top_visitors.py + * plugins/post_analysis/anonymize_ip.py * plugins/post_analysis/browsers.py * plugins/post_analysis/feeds.py * plugins/post_analysis/filter_users.py - * plugins/post_analysis/google_console_api.py * plugins/post_analysis/hours_stats.py * plugins/post_analysis/ip_to_geo.py - * plugins/post_analysis/iptogeo.py * plugins/post_analysis/operating_systems.py * plugins/post_analysis/referers.py * plugins/post_analysis/reverse_dns.py - * plugins/post_analysis/search_analytics_api_sample.py * plugins/post_analysis/top_downloads.py * plugins/post_analysis/top_hits.py * plugins/post_analysis/top_pages.py @@ -585,6 +583,33 @@ plugins.display.top_visitors None +plugins.post_analysis.anonymize_ip +---------------------------------- + + Post analysis hook + + Replace remote_addr by a SHA1 + + Plugin requirements : + None + + Conf values needed : + None + + Output files : + None + + Statistics creation : + None + + Statistics update : + valid_visitors: + remote_addr + + Statistics deletion : + None + + plugins.post_analysis.browsers ------------------------------ @@ -704,34 +729,6 @@ plugins.post_analysis.filter_users None -plugins.post_analysis.google_console_api ----------------------------------------- - - Post analysis hook - - Extract key phrases from Google console API - - Plugin requirements : - None - - Conf values needed : - domain_name - - Output files : - None - - Statistics creation : - None - - Statistics update : - month_stats : - key_phrases => - phrase => count - - Statistics deletion : - None - - plugins.post_analysis.hours_stats --------------------------------- @@ -799,11 +796,6 @@ plugins.post_analysis.ip_to_geo None -plugins.post_analysis.iptogeo ------------------------------ - - - plugins.post_analysis.operating_systems --------------------------------------- @@ -905,141 +897,6 @@ plugins.post_analysis.reverse_dns None -plugins.post_analysis.search_analytics_api_sample -------------------------------------------------- - - from __future__ import print_function - - import argparse - import sys - from googleapiclient import sample_tools - - # Declare command-line flags. - argparser = argparse.ArgumentParser(add_help=False) - argparser.add_argument('property_uri', type=str, - help=('Site or app URI to query data for (including ' - 'trailing slash).')) - argparser.add_argument('start_date', type=str, - help=('Start date of the requested date range in ' - 'YYYY-MM-DD format.')) - argparser.add_argument('end_date', type=str, - help=('End date of the requested date range in ' - 'YYYY-MM-DD format.')) - - - def main(argv): - service, flags = sample_tools.init( - argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser], - scope='https://www.googleapis.com/auth/webmasters.readonly') - - # First run a query to learn which dates we have data for. You should always - # check which days in a date range have data before running your main query. - # This query shows data for the entire range, grouped and sorted by day, - # descending; any days without data will be missing from the results. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'] - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Available dates') - - return - # Get totals for the date range. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Totals') - - # Get top 10 queries for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top Queries') - - # Get top 11-20 mobile queries for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'dimensionFilterGroups': [{ - 'filters': [{ - 'dimension': 'device', - 'expression': 'mobile' - }] - }], - 'rowLimit': 10, - 'startRow': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top 11-20 Mobile Queries') - - # Get top 10 pages for the date range, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['page'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top Pages') - - # Get the top 10 queries in India, sorted by click count, descending. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['query'], - 'dimensionFilterGroups': [{ - 'filters': [{ - 'dimension': 'country', - 'expression': 'ind' - }] - }], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Top queries in India') - - # Group by both country and device. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['country', 'device'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Group by country and device') - - # Group by total number of Search Appearance count. - # Note: It is not possible to use searchAppearance with other - # dimensions. - request = { - 'startDate': flags.start_date, - 'endDate': flags.end_date, - 'dimensions': ['searchAppearance'], - 'rowLimit': 10 - } - response = execute_request(service, flags.property_uri, request) - print_table(response, 'Search Appearance Features') - - def execute_request(service, property_uri, request): - """Executes a searchAnalytics.query request. - - Args: - service: The webmasters service to use when executing the query. - property_uri: The site or app URI to request data for. - request: The request to be executed. - - Returns: - An array of response rows. - - plugins.post_analysis.top_downloads ----------------------------------- diff --git a/locales/fr_FR/LC_MESSAGES/iwla.mo b/locales/fr_FR/LC_MESSAGES/iwla.mo index 5bac226..1f9b61e 100644 Binary files a/locales/fr_FR/LC_MESSAGES/iwla.mo and b/locales/fr_FR/LC_MESSAGES/iwla.mo differ diff --git a/locales/fr_FR/LC_MESSAGES/iwla.po b/locales/fr_FR/LC_MESSAGES/iwla.po index cec7514..bf56079 100644 --- a/locales/fr_FR/LC_MESSAGES/iwla.po +++ b/locales/fr_FR/LC_MESSAGES/iwla.po @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: iwla\n" -"POT-Creation-Date: 2022-11-07 21:14+0100\n" -"PO-Revision-Date: 2022-11-07 21:14+0100\n" +"POT-Creation-Date: 2022-11-10 20:07+0100\n" +"PO-Revision-Date: 2022-11-10 20:08+0100\n" "Last-Translator: Soutadé \n" "Language-Team: iwla\n" "Language: fr\n" @@ -84,14 +84,14 @@ msgstr "Visites" #: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70 #: plugins/display/feeds.py:76 plugins/display/filter_users.py:77 -#: plugins/display/filter_users.py:119 plugins/display/hours_stats.py:73 +#: plugins/display/filter_users.py:118 plugins/display/hours_stats.py:73 #: plugins/display/hours_stats.py:83 plugins/display/referers.py:95 #: plugins/display/referers.py:153 plugins/display/top_visitors.py:72 msgid "Pages" msgstr "Pages" #: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70 -#: plugins/display/feeds.py:76 plugins/display/filter_users.py:119 +#: plugins/display/feeds.py:76 plugins/display/filter_users.py:118 #: plugins/display/hours_stats.py:73 plugins/display/hours_stats.py:83 #: plugins/display/referers.py:95 plugins/display/referers.py:153 #: plugins/display/top_downloads.py:97 plugins/display/top_visitors.py:72 @@ -170,12 +170,11 @@ msgid "Month" msgstr "Mois" #: iwla.py:505 iwla.py:517 plugins/display/feeds.py:101 -#: plugins/display/filter_users.py:114 plugins/display/ip_to_geo.py:109 -#: plugins/display/operating_systems.py:90 +#: plugins/display/filter_users.py:113 plugins/display/operating_systems.py:90 msgid "Details" msgstr "Détails" -#: iwla.py:505 plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:114 +#: iwla.py:505 plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:112 msgid "Visitors" msgstr "Visiteurs" @@ -203,8 +202,12 @@ msgstr "minutes" msgid "seconds" msgstr "secondes" +#: plugins/display/all_visits.py:70 plugins/display/all_visits.py:92 +msgid "All visits" +msgstr "Toutes les visites" + #: plugins/display/all_visits.py:70 plugins/display/feeds.py:76 -#: plugins/display/filter_users.py:119 plugins/display/ip_to_geo.py:64 +#: plugins/display/filter_users.py:118 plugins/display/ip_to_geo.py:62 #: plugins/display/robot_bandwidth.py:81 plugins/display/robot_bandwidth.py:106 #: plugins/display/top_visitors.py:72 msgid "Host" @@ -215,10 +218,6 @@ msgstr "Hôte" msgid "Last seen" msgstr "Dernière visite" -#: plugins/display/all_visits.py:92 -msgid "All visits" -msgstr "Toutes les visites" - #: plugins/display/all_visits.py:93 plugins/display/top_visitors.py:72 msgid "Top visitors" msgstr "Top visiteurs" @@ -240,7 +239,7 @@ msgid "Entrance" msgstr "Entrées" #: plugins/display/browsers.py:99 plugins/display/browsers.py:130 -#: plugins/display/filter_users.py:129 plugins/display/referers.py:110 +#: plugins/display/filter_users.py:128 plugins/display/referers.py:110 #: plugins/display/referers.py:125 plugins/display/referers.py:140 #: plugins/display/referers.py:163 plugins/display/referers.py:174 #: plugins/display/referers.py:185 plugins/display/referers.py:222 @@ -291,11 +290,11 @@ msgstr "Origine" msgid "User Agent" msgstr "Navigateur" -#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:112 +#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:111 msgid "Filtered users" msgstr "Utilisateurs filtrés" -#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:119 +#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:118 msgid "Last Access" msgstr "Dernière visite" @@ -335,15 +334,19 @@ msgstr "Par heures" msgid "Hours" msgstr "Heures" -#: plugins/display/ip_to_geo.py:96 +#: plugins/display/ip_to_geo.py:94 msgid "Country" msgstr "Pays" -#: plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:107 -#: plugins/display/ip_to_geo.py:114 +#: plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:105 +#: plugins/display/ip_to_geo.py:112 msgid "Countries" msgstr "Pays" +#: plugins/display/ip_to_geo.py:107 +msgid "All countries" +msgstr "Tous les pays" + #: plugins/display/operating_systems.py:78 #: plugins/display/operating_systems.py:88 msgid "Operating Systems"