Update robot detection plugin :

* Do analyze only one time by month
  * Reactivate rule : no page view if count_hit_only_visitors is False
  * Add exception for "Less than 1 hit per page" rule if a phone is used
  * Check for all error codes in 400..499, not only 403 and 404
  * Referer '-' now counted as null
This commit is contained in:
Gregory Soutade 2023-03-11 20:48:17 +01:00
parent 31bc67ceba
commit a0a1f42df4

View File

@ -55,9 +55,6 @@ Statistics deletion :
""" """
class IWLAPreAnalysisRobots(IPlugin): class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):
super(IWLAPreAnalysisRobots, self).__init__(iwla)
self.API_VERSION = 1
def load(self): def load(self):
self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)) self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
@ -65,6 +62,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE) self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*') self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*')
self.logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
return True return True
@ -74,7 +72,7 @@ class IWLAPreAnalysisRobots(IPlugin):
info = inspect.getframeinfo(frame) info = inspect.getframeinfo(frame)
self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno)) self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
super_hit['robot'] = 1 super_hit['robot'] = True
super_hit['keep_requests'] = False super_hit['keep_requests'] = False
for hit in super_hit['requests']: for hit in super_hit['requests']:
robot_name = self.compatible_re.match(hit['http_user_agent']) robot_name = self.compatible_re.match(hit['http_user_agent'])
@ -86,14 +84,17 @@ class IWLAPreAnalysisRobots(IPlugin):
def hook(self): def hook(self):
hits = self.iwla.getCurrentVisits() hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items(): for (k, super_hit) in hits.items():
if super_hit['robot']: # Already analyzed
self.logger.debug('%s is a robot' % (k)) if super_hit.get('robot', None) in (True, False):
if super_hit['robot'] == True:
self.logger.debug('%s is a robot' % (k))
continue continue
if super_hit.get('feed_parser', False): if super_hit.get('feed_parser', False):
self.logger.debug('%s is feed parser' % (k)) self.logger.debug('%s is feed parser' % (k))
continue continue
super_hit['robot'] = False
isRobot = False isRobot = False
referers = 0 referers = 0
@ -117,21 +118,27 @@ class IWLAPreAnalysisRobots(IPlugin):
continue continue
# 1) no pages view --> robot # 1) no pages view --> robot
# if not super_hit['viewed_pages'][0]: if not self.one_hit_only and not super_hit['viewed_pages'][0]:
# super_hit['robot'] = 1
# continue
# 2) Less than 1 hit per page
if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]):
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue
# 3) no pages and not hit --> robot # 2) Less than 1 hit per page (except for mobile phones that can do caching)
if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]) \
and not 'iPhone' in first_page['http_user_agent'] \
and not 'Android' in first_page['http_user_agent'] \
:
isRobot = True
if isRobot:
self._setRobot(k, super_hit)
continue
# 3) no pages and not hit --> robot
if not super_hit['viewed_hits'][0] and not super_hit['viewed_pages'][0]: if not super_hit['viewed_hits'][0] and not super_hit['viewed_pages'][0]:
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue
not_found_pages = 0 error_codes = 0
not_modified_pages = 0 not_modified_pages = 0
for hit in super_hit['requests']: for hit in super_hit['requests']:
# 5) /robots.txt read # 5) /robots.txt read
@ -139,26 +146,25 @@ class IWLAPreAnalysisRobots(IPlugin):
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
break break
if int(hit['status']) in (404, 403): if int(hit['status']) >= 400 and int(hit['status']) <= 499:
not_found_pages += 1 error_codes += 1
elif int(hit['status']) in (304,): elif int(hit['status']) in (304,):
not_modified_pages += 1 not_modified_pages += 1
# 6) Any referer for hits if not hit['is_page'] and hit['http_referer'] not in ('', '-'):
if not hit['is_page'] and hit['http_referer']:
referers += 1 referers += 1
if isRobot: if isRobot:
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue
# 7) more than 10 404/403 or 304 pages # 6) Any referer for hits
if not_found_pages > 10 or not_modified_pages > 10: if super_hit['viewed_hits'][0] and not referers:
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue
if not super_hit['viewed_pages'][0] and \ # 7) more than 10 4XX or 304 pages
(super_hit['viewed_hits'][0] and not referers): if error_codes > 10 or not_modified_pages > 10:
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue