From 4d3c2107f0167466888e2fe10074605ce1cdd8b0 Mon Sep 17 00:00:00 2001
From: Gregory Soutade <gregory@soutade.fr>
Date: Thu, 23 Jun 2022 21:16:30 +0200
Subject: [PATCH] Don't save all visitors requests into database (save space
 and computing). Can be changed in deufalt_conf.py with keep_requests value

---
 default_conf.py                | 5 ++++-
 iwla.py                        | 7 +++++++
 plugins/pre_analysis/robots.py | 5 +++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/default_conf.py b/default_conf.py
index 96ce509..e99b9ef 100644
--- a/default_conf.py
+++ b/default_conf.py
@@ -15,7 +15,7 @@ PRE_HOOK_DIRECTORY = HOOKS_ROOT + '.pre_analysis'
 POST_HOOK_DIRECTORY = HOOKS_ROOT + '.post_analysis'
 DISPLAY_HOOK_DIRECTORY = HOOKS_ROOT + '.display'
 # Meta Database filename
-META_PATH = os.path.join(DB_ROOT, 'meta.db')
+META_FILENAME = 'meta.db'
 # Database filename per month
 DB_FILENAME = 'iwla.db'
 
@@ -60,3 +60,6 @@ locales_path = './locales'
 
 # Default locale (english)
 locale = 'en_EN'
+
+# Don't keep requests of all visitors into database
+keep_requests = False
diff --git a/iwla.py b/iwla.py
index c021d01..8f37700 100755
--- a/iwla.py
+++ b/iwla.py
@@ -115,6 +115,7 @@ visits :
        robot
        hit_only
        is_page
+       keep_requests
 
 valid_visitors:
     month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
@@ -646,6 +647,12 @@ class IWLA(object):
             self._generateDisplay()
             return
 
+        for (k,v) in visits.items():
+            # Keep at least one request (for referers...)
+            if not v.get('keep_requests', conf.keep_requests):
+                if len(v['requests']) > 1:
+                    v['requests'] = [v['requests'][0]]
+                    
         path = self.getDBFilename(cur_time)
 
         self.logger.info("==> Serialize to %s" % (path))
diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py
index 086dd8d..94a3adf 100644
--- a/plugins/pre_analysis/robots.py
+++ b/plugins/pre_analysis/robots.py
@@ -36,8 +36,7 @@ Plugin requirements :
     None
 
 Conf values needed :
-    page_to_hit_conf*
-    hit_to_page_conf*
+    None
 
 Output files :
     None
@@ -49,6 +48,7 @@ Statistics update :
 visits :
    remote_addr =>
        robot
+       keep_requests
 
 Statistics deletion :
     None
@@ -73,6 +73,7 @@ class IWLAPreAnalysisRobots(IPlugin):
 
         self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
         super_hit['robot'] = 1
+        super_hit['keep_requests'] = False
 
 # Basic rule to detect robots
     def hook(self):