From a0a1f42df4c067d18fc2e1540156262a53441c96 Mon Sep 17 00:00:00 2001
From: Gregory Soutade <gregory@soutade.fr>
Date: Sat, 11 Mar 2023 20:48:17 +0100
Subject: [PATCH] Update robot detection plugin  :   * Do analyze only one time
 by month   * Reactivate rule : no page view if count_hit_only_visitors is
 False   * Add exception for "Less than 1 hit per page" rule if a phone is
 used   * Check for all error codes in 400..499, not only 403 and 404   *
 Referer '-' now counted as null

---
 plugins/pre_analysis/robots.py | 50 +++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 22 deletions(-)

diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py
index eee6ac1..efde8b9 100644
--- a/plugins/pre_analysis/robots.py
+++ b/plugins/pre_analysis/robots.py
@@ -55,9 +55,6 @@ Statistics deletion :
 """
 
 class IWLAPreAnalysisRobots(IPlugin):
-    def __init__(self, iwla):
-        super(IWLAPreAnalysisRobots, self).__init__(iwla)
-        self.API_VERSION = 1
 
     def load(self):
         self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
@@ -65,6 +62,7 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
         self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*')
         self.logger = logging.getLogger(self.__class__.__name__)
+        self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
 
         return True
 
@@ -74,7 +72,7 @@ class IWLAPreAnalysisRobots(IPlugin):
         info = inspect.getframeinfo(frame)
 
         self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
-        super_hit['robot'] = 1
+        super_hit['robot'] = True
         super_hit['keep_requests'] = False
         for hit in super_hit['requests']:
             robot_name = self.compatible_re.match(hit['http_user_agent'])
@@ -86,14 +84,17 @@ class IWLAPreAnalysisRobots(IPlugin):
     def hook(self):
         hits = self.iwla.getCurrentVisits()
         for (k, super_hit) in hits.items():
-            if super_hit['robot']:
-                self.logger.debug('%s is a robot' % (k))
+            # Already analyzed
+            if super_hit.get('robot', None) in (True, False):
+                if super_hit['robot'] == True:
+                    self.logger.debug('%s is a robot' % (k))
                 continue
 
             if super_hit.get('feed_parser', False):
                 self.logger.debug('%s is feed parser' % (k))
                 continue
             
+            super_hit['robot'] = False
             isRobot = False
             referers = 0
 
@@ -117,21 +118,27 @@ class IWLAPreAnalysisRobots(IPlugin):
                 continue
 
 # 1) no pages view --> robot
-            # if not super_hit['viewed_pages'][0]:
-            #     super_hit['robot'] = 1
-            #     continue
-
-# 2) Less than 1 hit per page
-            if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]):
+            if not self.one_hit_only and not super_hit['viewed_pages'][0]:
                 self._setRobot(k, super_hit)
                 continue
 
-            # 3) no pages and not hit --> robot
+# 2) Less than 1 hit per page (except for mobile phones that can do caching)
+            if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]) \
+               and not 'iPhone' in first_page['http_user_agent'] \
+               and not 'Android' in first_page['http_user_agent'] \
+            :
+                isRobot = True
+
+            if isRobot:
+                self._setRobot(k, super_hit)
+                continue
+
+# 3) no pages and not hit --> robot
             if not super_hit['viewed_hits'][0] and not super_hit['viewed_pages'][0]:
                 self._setRobot(k, super_hit)
                 continue
 
-            not_found_pages = 0
+            error_codes = 0
             not_modified_pages = 0
             for hit in super_hit['requests']:
 # 5) /robots.txt read
@@ -139,26 +146,25 @@ class IWLAPreAnalysisRobots(IPlugin):
                     self._setRobot(k, super_hit)
                     break
 
-                if int(hit['status']) in (404, 403):
-                    not_found_pages += 1
+                if int(hit['status']) >= 400 and int(hit['status']) <= 499:
+                    error_codes += 1
                 elif int(hit['status']) in (304,):
                     not_modified_pages += 1
 
-# 6) Any referer for hits
-                if not hit['is_page'] and hit['http_referer']:
+                if not hit['is_page'] and hit['http_referer'] not in ('', '-'):
                     referers += 1
 
             if isRobot:
                 self._setRobot(k, super_hit)
                 continue
 
-# 7) more than 10 404/403 or 304 pages
-            if not_found_pages > 10 or not_modified_pages > 10:
+# 6) Any referer for hits
+            if super_hit['viewed_hits'][0] and not referers:
                 self._setRobot(k, super_hit)
                 continue
 
-            if not super_hit['viewed_pages'][0] and \
-                    (super_hit['viewed_hits'][0] and not referers):
+# 7) more than 10 4XX or 304 pages
+            if error_codes > 10 or not_modified_pages > 10:
                 self._setRobot(k, super_hit)
                 continue