Update documentation

docs/index.md

@@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www
 
 Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option).
 
+Demo
+----
+
+A demonstration instance is available [here](https://iwla-demo.soutade.fr)
+
 Usage
 -----
 
@@ -19,7 +24,7 @@ Usage
     -f : Analyse this log file, multiple files can be specified    (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename
     -d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
     -r : Reset analysis to a specific date (month/year)
-    -z : Don't compress databases (bigger but faster, not  compatible with compressed databases)
+    -z : Don't compress databases (bigger but faster, not compatible with compressed databases)
     -p : Only generate display
     -d : Dry run (don't write/update files to disk)
   
@@ -36,6 +41,8 @@ Main values to edit are :
   * **post_analysis_hooks** : List of post analysis hooks
   * **display_hooks** : List of display hooks
   * **locale** : Displayed locale (_en_ or _fr_)
+  * **feeds** : Address of your feeds files
+  * **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...)
 
 You can also append an element to an existing default configuration list by using "_append" suffix. Example :
   multimedia_files_append = ['xml']
@@ -82,7 +89,7 @@ Statistics are stored in dictionaries :
   * **month_stats** : Statistics of current analysed month
   * **valid_visitor** : A subset of month_stats without robots
   * **days_stats** : Statistics of current analysed day
-  * **visits** : All visitors with all of its requests
+  * **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered)
   * **meta** : Final result of month statistics (by year)
 
 Create a Plugins
@@ -119,17 +126,15 @@ Optional configuration values ends with *.
   * plugins/display/top_pages_diff.py
   * plugins/display/top_pages.py
   * plugins/display/top_visitors.py
+  * plugins/post_analysis/anonymize_ip.py
   * plugins/post_analysis/browsers.py
   * plugins/post_analysis/feeds.py
   * plugins/post_analysis/filter_users.py
-  * plugins/post_analysis/google_console_api.py
   * plugins/post_analysis/hours_stats.py
   * plugins/post_analysis/ip_to_geo.py
-  * plugins/post_analysis/iptogeo.py
   * plugins/post_analysis/operating_systems.py
   * plugins/post_analysis/referers.py
   * plugins/post_analysis/reverse_dns.py
-  * plugins/post_analysis/search_analytics_api_sample.py
   * plugins/post_analysis/top_downloads.py
   * plugins/post_analysis/top_hits.py
   * plugins/post_analysis/top_pages.py
@@ -688,6 +693,33 @@ plugins.display.top_visitors
         None
 
 
+plugins.post_analysis.anonymize_ip
+----------------------------------
+
+    Post analysis hook
+    
+    Replace remote_addr by a SHA1
+    
+    Plugin requirements :
+        None
+    
+    Conf values needed :
+        None
+    
+    Output files :
+        None
+    
+    Statistics creation :
+        None
+    
+    Statistics update :
+    valid_visitors:
+       remote_addr
+    
+    Statistics deletion :
+        None
+
+
 plugins.post_analysis.browsers
 ------------------------------
 
@@ -807,34 +839,6 @@ plugins.post_analysis.filter_users
         None
 
 
-plugins.post_analysis.google_console_api
-----------------------------------------
-
-    Post analysis hook
-    
-    Extract key phrases from Google console API
-    
-    Plugin requirements :
-        None
-    
-    Conf values needed :
-        domain_name
-    
-    Output files :
-        None
-    
-    Statistics creation :
-        None
-    
-    Statistics update :
-    month_stats :
-       key_phrases =>
-           phrase => count
-    
-    Statistics deletion :
-        None
-
-
 plugins.post_analysis.hours_stats
 ---------------------------------
 
@@ -902,11 +906,6 @@ plugins.post_analysis.ip_to_geo
         None
 
 
-plugins.post_analysis.iptogeo
------------------------------
-
-
-
 plugins.post_analysis.operating_systems
 ---------------------------------------
 
@@ -1008,141 +1007,6 @@ plugins.post_analysis.reverse_dns
         None
 
 
-plugins.post_analysis.search_analytics_api_sample
--------------------------------------------------
-
-    from __future__ import print_function
-    
-    import argparse
-    import sys
-    from googleapiclient import sample_tools
-    
-    # Declare command-line flags.
-    argparser = argparse.ArgumentParser(add_help=False)
-    argparser.add_argument('property_uri', type=str,
-                           help=('Site or app URI to query data for (including '
-                                 'trailing slash).'))
-    argparser.add_argument('start_date', type=str,
-                           help=('Start date of the requested date range in '
-                                 'YYYY-MM-DD format.'))
-    argparser.add_argument('end_date', type=str,
-                           help=('End date of the requested date range in '
-                                 'YYYY-MM-DD format.'))
-    
-    
-    def main(argv):
-      service, flags = sample_tools.init(
-          argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser],
-          scope='https://www.googleapis.com/auth/webmasters.readonly')
-    
-      # First run a query to learn which dates we have data for. You should always
-      # check which days in a date range have data before running your main query.
-      # This query shows data for the entire range, grouped and sorted by day,
-      # descending; any days without data will be missing from the results.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query']
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Available dates')
-    
-      return
-      # Get totals for the date range.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Totals')
-    
-      # Get top 10 queries for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top Queries')
-    
-      # Get top 11-20 mobile queries for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'dimensionFilterGroups': [{
-              'filters': [{
-                  'dimension': 'device',
-                  'expression': 'mobile'
-              }]
-          }],
-          'rowLimit': 10,
-          'startRow': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top 11-20 Mobile Queries')
-    
-      # Get top 10 pages for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['page'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top Pages')
-    
-      # Get the top 10 queries in India, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'dimensionFilterGroups': [{
-              'filters': [{
-                  'dimension': 'country',
-                  'expression': 'ind'
-              }]
-          }],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top queries in India')
-    
-      # Group by both country and device.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['country', 'device'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Group by country and device')
-    
-      # Group by total number of Search Appearance count.
-      # Note: It is not possible to use searchAppearance with other
-      # dimensions.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['searchAppearance'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Search Appearance Features')
-    
-    def execute_request(service, property_uri, request):
-      """Executes a searchAnalytics.query request.
-    
-      Args:
-        service: The webmasters service to use when executing the query.
-        property_uri: The site or app URI to request data for.
-        request: The request to be executed.
-    
-      Returns:
-        An array of response rows.
-
-
 plugins.post_analysis.top_downloads
 -----------------------------------
 

docs/main.md

@@ -8,6 +8,11 @@ iwla (Intelligent Web Log Analyzer) is basically a clone of [awstats](http://www
 
 Nevertheless, iwla is only focused on HTTP logs. It uses data (robots definitions, search engines definitions) and design from awstats. Moreover, it's not dynamic, but only generates static HTML page (with gzip compression option).
 
+Demo
+----
+
+A demonstration instance is available [here](https://iwla-demo.soutade.fr)
+
 Usage
 -----
 
@@ -19,7 +24,7 @@ Usage
     -f : Analyse this log file, multiple files can be specified    (comma separated). gz files are acceptedRead data from FILE instead of conf.analyzed_filename
     -d : Loglevel in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
     -r : Reset analysis to a specific date (month/year)
-    -z : Don't compress databases (bigger but faster, not  compatible with compressed databases)
+    -z : Don't compress databases (bigger but faster, not compatible with compressed databases)
     -p : Only generate display
     -d : Dry run (don't write/update files to disk)
   
@@ -36,6 +41,8 @@ Main values to edit are :
   * **post_analysis_hooks** : List of post analysis hooks
   * **display_hooks** : List of display hooks
   * **locale** : Displayed locale (_en_ or _fr_)
+  * **feeds** : Address of your feeds files
+  * **count_hit_only_visitors** true/false (don't) count visitors that only do one hit (for a picture, ...)
 
 You can also append an element to an existing default configuration list by using "_append" suffix. Example :
   multimedia_files_append = ['xml']
@@ -82,7 +89,7 @@ Statistics are stored in dictionaries :
   * **month_stats** : Statistics of current analysed month
   * **valid_visitor** : A subset of month_stats without robots
   * **days_stats** : Statistics of current analysed day
-  * **visits** : All visitors with all of its requests
+  * **visits** : All visitors with all of its requests (only if 'keep_requests' is true or filtered)
   * **meta** : Final result of month statistics (by year)
 
 Create a Plugins

docs/modules.md

@@ -16,17 +16,15 @@
   * plugins/display/top_pages_diff.py
   * plugins/display/top_pages.py
   * plugins/display/top_visitors.py
+  * plugins/post_analysis/anonymize_ip.py
   * plugins/post_analysis/browsers.py
   * plugins/post_analysis/feeds.py
   * plugins/post_analysis/filter_users.py
-  * plugins/post_analysis/google_console_api.py
   * plugins/post_analysis/hours_stats.py
   * plugins/post_analysis/ip_to_geo.py
-  * plugins/post_analysis/iptogeo.py
   * plugins/post_analysis/operating_systems.py
   * plugins/post_analysis/referers.py
   * plugins/post_analysis/reverse_dns.py
-  * plugins/post_analysis/search_analytics_api_sample.py
   * plugins/post_analysis/top_downloads.py
   * plugins/post_analysis/top_hits.py
   * plugins/post_analysis/top_pages.py
@@ -585,6 +583,33 @@ plugins.display.top_visitors
         None
 
 
+plugins.post_analysis.anonymize_ip
+----------------------------------
+
+    Post analysis hook
+    
+    Replace remote_addr by a SHA1
+    
+    Plugin requirements :
+        None
+    
+    Conf values needed :
+        None
+    
+    Output files :
+        None
+    
+    Statistics creation :
+        None
+    
+    Statistics update :
+    valid_visitors:
+       remote_addr
+    
+    Statistics deletion :
+        None
+
+
 plugins.post_analysis.browsers
 ------------------------------
 
@@ -704,34 +729,6 @@ plugins.post_analysis.filter_users
         None
 
 
-plugins.post_analysis.google_console_api
-----------------------------------------
-
-    Post analysis hook
-    
-    Extract key phrases from Google console API
-    
-    Plugin requirements :
-        None
-    
-    Conf values needed :
-        domain_name
-    
-    Output files :
-        None
-    
-    Statistics creation :
-        None
-    
-    Statistics update :
-    month_stats :
-       key_phrases =>
-           phrase => count
-    
-    Statistics deletion :
-        None
-
-
 plugins.post_analysis.hours_stats
 ---------------------------------
 
@@ -799,11 +796,6 @@ plugins.post_analysis.ip_to_geo
         None
 
 
-plugins.post_analysis.iptogeo
------------------------------
-
-
-
 plugins.post_analysis.operating_systems
 ---------------------------------------
 
@@ -905,141 +897,6 @@ plugins.post_analysis.reverse_dns
         None
 
 
-plugins.post_analysis.search_analytics_api_sample
--------------------------------------------------
-
-    from __future__ import print_function
-    
-    import argparse
-    import sys
-    from googleapiclient import sample_tools
-    
-    # Declare command-line flags.
-    argparser = argparse.ArgumentParser(add_help=False)
-    argparser.add_argument('property_uri', type=str,
-                           help=('Site or app URI to query data for (including '
-                                 'trailing slash).'))
-    argparser.add_argument('start_date', type=str,
-                           help=('Start date of the requested date range in '
-                                 'YYYY-MM-DD format.'))
-    argparser.add_argument('end_date', type=str,
-                           help=('End date of the requested date range in '
-                                 'YYYY-MM-DD format.'))
-    
-    
-    def main(argv):
-      service, flags = sample_tools.init(
-          argv, 'webmasters', 'v3', __doc__, __file__, parents=[argparser],
-          scope='https://www.googleapis.com/auth/webmasters.readonly')
-    
-      # First run a query to learn which dates we have data for. You should always
-      # check which days in a date range have data before running your main query.
-      # This query shows data for the entire range, grouped and sorted by day,
-      # descending; any days without data will be missing from the results.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query']
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Available dates')
-    
-      return
-      # Get totals for the date range.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Totals')
-    
-      # Get top 10 queries for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top Queries')
-    
-      # Get top 11-20 mobile queries for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'dimensionFilterGroups': [{
-              'filters': [{
-                  'dimension': 'device',
-                  'expression': 'mobile'
-              }]
-          }],
-          'rowLimit': 10,
-          'startRow': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top 11-20 Mobile Queries')
-    
-      # Get top 10 pages for the date range, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['page'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top Pages')
-    
-      # Get the top 10 queries in India, sorted by click count, descending.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['query'],
-          'dimensionFilterGroups': [{
-              'filters': [{
-                  'dimension': 'country',
-                  'expression': 'ind'
-              }]
-          }],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Top queries in India')
-    
-      # Group by both country and device.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['country', 'device'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Group by country and device')
-    
-      # Group by total number of Search Appearance count.
-      # Note: It is not possible to use searchAppearance with other
-      # dimensions.
-      request = {
-          'startDate': flags.start_date,
-          'endDate': flags.end_date,
-          'dimensions': ['searchAppearance'],
-          'rowLimit': 10
-      }
-      response = execute_request(service, flags.property_uri, request)
-      print_table(response, 'Search Appearance Features')
-    
-    def execute_request(service, property_uri, request):
-      """Executes a searchAnalytics.query request.
-    
-      Args:
-        service: The webmasters service to use when executing the query.
-        property_uri: The site or app URI to request data for.
-        request: The request to be executed.
-    
-      Returns:
-        An array of response rows.
-
-
 plugins.post_analysis.top_downloads
 -----------------------------------
 

locales/fr_FR/LC_MESSAGES/iwla.po

@@ -5,8 +5,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: iwla\n"
-"POT-Creation-Date: 2022-11-07 21:14+0100\n"
+"POT-Creation-Date: 2022-11-10 20:07+0100\n"
-"PO-Revision-Date: 2022-11-07 21:14+0100\n"
+"PO-Revision-Date: 2022-11-10 20:08+0100\n"
 "Last-Translator: Soutadé <soutade@gmail.com>\n"
 "Language-Team: iwla\n"
 "Language: fr\n"
@@ -84,14 +84,14 @@ msgstr "Visites"
 
 #: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70
 #: plugins/display/feeds.py:76 plugins/display/filter_users.py:77
-#: plugins/display/filter_users.py:119 plugins/display/hours_stats.py:73
+#: plugins/display/filter_users.py:118 plugins/display/hours_stats.py:73
 #: plugins/display/hours_stats.py:83 plugins/display/referers.py:95
 #: plugins/display/referers.py:153 plugins/display/top_visitors.py:72
 msgid "Pages"
 msgstr "Pages"
 
 #: iwla.py:454 iwla.py:505 plugins/display/all_visits.py:70
-#: plugins/display/feeds.py:76 plugins/display/filter_users.py:119
+#: plugins/display/feeds.py:76 plugins/display/filter_users.py:118
 #: plugins/display/hours_stats.py:73 plugins/display/hours_stats.py:83
 #: plugins/display/referers.py:95 plugins/display/referers.py:153
 #: plugins/display/top_downloads.py:97 plugins/display/top_visitors.py:72
@@ -170,12 +170,11 @@ msgid "Month"
 msgstr "Mois"
 
 #: iwla.py:505 iwla.py:517 plugins/display/feeds.py:101
-#: plugins/display/filter_users.py:114 plugins/display/ip_to_geo.py:109
+#: plugins/display/filter_users.py:113 plugins/display/operating_systems.py:90
-#: plugins/display/operating_systems.py:90
 msgid "Details"
 msgstr "Détails"
 
-#: iwla.py:505 plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:114
+#: iwla.py:505 plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:112
 msgid "Visitors"
 msgstr "Visiteurs"
 
@@ -203,8 +202,12 @@ msgstr "minutes"
 msgid "seconds"
 msgstr "secondes"
 
+#: plugins/display/all_visits.py:70 plugins/display/all_visits.py:92
+msgid "All visits"
+msgstr "Toutes les visites"
+
 #: plugins/display/all_visits.py:70 plugins/display/feeds.py:76
-#: plugins/display/filter_users.py:119 plugins/display/ip_to_geo.py:64
+#: plugins/display/filter_users.py:118 plugins/display/ip_to_geo.py:62
 #: plugins/display/robot_bandwidth.py:81 plugins/display/robot_bandwidth.py:106
 #: plugins/display/top_visitors.py:72
 msgid "Host"
@@ -215,10 +218,6 @@ msgstr "Hôte"
 msgid "Last seen"
 msgstr "Dernière visite"
 
-#: plugins/display/all_visits.py:92
-msgid "All visits"
-msgstr "Toutes les visites"
-
 #: plugins/display/all_visits.py:93 plugins/display/top_visitors.py:72
 msgid "Top visitors"
 msgstr "Top visiteurs"
@@ -240,7 +239,7 @@ msgid "Entrance"
 msgstr "Entrées"
 
 #: plugins/display/browsers.py:99 plugins/display/browsers.py:130
-#: plugins/display/filter_users.py:129 plugins/display/referers.py:110
+#: plugins/display/filter_users.py:128 plugins/display/referers.py:110
 #: plugins/display/referers.py:125 plugins/display/referers.py:140
 #: plugins/display/referers.py:163 plugins/display/referers.py:174
 #: plugins/display/referers.py:185 plugins/display/referers.py:222
@@ -291,11 +290,11 @@ msgstr "Origine"
 msgid "User Agent"
 msgstr "Navigateur"
 
-#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:112
+#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:111
 msgid "Filtered users"
 msgstr "Utilisateurs filtrés"
 
-#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:119
+#: plugins/display/filter_users.py:77 plugins/display/filter_users.py:118
 msgid "Last Access"
 msgstr "Dernière visite"
 
@@ -335,15 +334,19 @@ msgstr "Par heures"
 msgid "Hours"
 msgstr "Heures"
 
-#: plugins/display/ip_to_geo.py:96
+#: plugins/display/ip_to_geo.py:94
 msgid "Country"
 msgstr "Pays"
 
-#: plugins/display/ip_to_geo.py:96 plugins/display/ip_to_geo.py:107
+#: plugins/display/ip_to_geo.py:94 plugins/display/ip_to_geo.py:105
-#: plugins/display/ip_to_geo.py:114
+#: plugins/display/ip_to_geo.py:112
 msgid "Countries"
 msgstr "Pays"
 
+#: plugins/display/ip_to_geo.py:107
+msgid "All countries"
+msgstr "Tous les pays"
+
 #: plugins/display/operating_systems.py:78
 #: plugins/display/operating_systems.py:88
 msgid "Operating Systems"