soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Add rule for robot : forbid only "1 page and 1 hit"

Browse Source
This commit is contained in:
Gregory Soutade 2025-02-03 08:00:25 +01:00
parent 4d0b993aec
commit 7b0ca661a1
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
plugins/pre_analysis/robots.py
View File

@ -132,6 +132,9 @@ class IWLAPreAnalysisRobots(IPlugin):
# 2) Less than 1 hit per page
if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]):
isRobot = True
# 2.5) 1 page, 1 hit
elif super_hit['viewed_pages'][0] == 1 and super_hit['viewed_hits'][0] == 1:
isRobot = True
if isRobot:
self._setRobot(k, super_hit)
@ -152,8 +155,7 @@ class IWLAPreAnalysisRobots(IPlugin):
# Exception for favicon.png and all apple-*icon*
if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\
'icon' not in hit['extract_request']['http_uri'] and\
hit['server_name'] != 'forge.soutade.fr':
'icon' not in hit['extract_request']['http_uri']:
error_codes += 1
elif int(hit['status']) in (304,):
not_modified_pages += 1