soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Update robot detection plugin :

Browse Source 
* Do analyze only one time by month
  * Reactivate rule : no page view if count_hit_only_visitors is False
  * Add exception for "Less than 1 hit per page" rule if a phone is used
  * Check for all error codes in 400..499, not only 403 and 404
  * Referer '-' now counted as null
This commit is contained in:
Gregory Soutade
2023-03-11 20:48:17 +01:00
parent 31bc67ceba
commit a0a1f42df4
1 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
plugins/pre_analysis/robots.py
View File

@@ -55,9 +55,6 @@ Statistics deletion :
"""
class IWLAPreAnalysisRobots(IPlugin):
def __init__(self, iwla):
super(IWLAPreAnalysisRobots, self).__init__(iwla)
self.API_VERSION = 1
def load(self):
self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
@@ -65,6 +62,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*')
self.logger = logging.getLogger(self.__class__.__name__)
self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
return True
@@ -74,7 +72,7 @@ class IWLAPreAnalysisRobots(IPlugin):
info = inspect.getframeinfo(frame)
self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
super_hit['robot'] = 1
super_hit['robot'] = True
super_hit['keep_requests'] = False
for hit in super_hit['requests']:
robot_name = self.compatible_re.match(hit['http_user_agent'])
@@ -86,7 +84,9 @@ class IWLAPreAnalysisRobots(IPlugin):
def hook(self):
hits = self.iwla.getCurrentVisits()
for (k, super_hit) in hits.items():
if super_hit['robot']:
# Already analyzed
if super_hit.get('robot', None) in (True, False):
if super_hit['robot'] == True:
self.logger.debug('%s is a robot' % (k))
continue
@@ -94,6 +94,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self.logger.debug('%s is feed parser' % (k))
continue
super_hit['robot'] = False
isRobot = False
referers = 0
@@ -117,12 +118,18 @@ class IWLAPreAnalysisRobots(IPlugin):
continue
# 1) no pages view --> robot
# if not super_hit['viewed_pages'][0]:
# super_hit['robot'] = 1
# continue
if not self.one_hit_only and not super_hit['viewed_pages'][0]:
self._setRobot(k, super_hit)
continue
# 2) Less than 1 hit per page
if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]):
# 2) Less than 1 hit per page (except for mobile phones that can do caching)
if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]) \
and not 'iPhone' in first_page['http_user_agent'] \
and not 'Android' in first_page['http_user_agent'] \
:
isRobot = True
if isRobot:
self._setRobot(k, super_hit)
continue
@@ -131,7 +138,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self._setRobot(k, super_hit)
continue
not_found_pages = 0
error_codes = 0
not_modified_pages = 0
for hit in super_hit['requests']:
# 5) /robots.txt read
@@ -139,26 +146,25 @@ class IWLAPreAnalysisRobots(IPlugin):
self._setRobot(k, super_hit)
break
if int(hit['status']) in (404, 403):
not_found_pages += 1
if int(hit['status']) >= 400 and int(hit['status']) <= 499:
error_codes += 1
elif int(hit['status']) in (304,):
not_modified_pages += 1
# 6) Any referer for hits
if not hit['is_page'] and hit['http_referer']:
if not hit['is_page'] and hit['http_referer'] not in ('', '-'):
referers += 1
if isRobot:
self._setRobot(k, super_hit)
continue
# 7) more than 10 404/403 or 304 pages
if not_found_pages > 10 or not_modified_pages > 10:
# 6) Any referer for hits
if super_hit['viewed_hits'][0] and not referers:
self._setRobot(k, super_hit)
continue
if not super_hit['viewed_pages'][0] and \
(super_hit['viewed_hits'][0] and not referers):
# 7) more than 10 4XX or 304 pages
if error_codes > 10 or not_modified_pages > 10:
self._setRobot(k, super_hit)
continue