Increase not modified page threshold for robot detection
This commit is contained in:
parent
d32b2440ee
commit
16cd817fec
|
@ -166,7 +166,7 @@ class IWLAPreAnalysisRobots(IPlugin):
|
|||
continue
|
||||
|
||||
# 7) more than 10 4XX or 304 pages
|
||||
if error_codes > 10 or not_modified_pages > 10:
|
||||
if error_codes > 10 or not_modified_pages > 50:
|
||||
self._setRobot(k, super_hit)
|
||||
continue
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user