diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 9367b6e..0ce9433 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -166,7 +166,7 @@ class IWLAPreAnalysisRobots(IPlugin): continue # 7) more than 10 4XX or 304 pages - if error_codes > 10 or not_modified_pages > 10: + if error_codes > 10 or not_modified_pages > 50: self._setRobot(k, super_hit) continue