diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 9dbeb62..af71d27 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -108,8 +108,8 @@ class IWLAPreAnalysisRobots(IPlugin): # super_hit['robot'] = 1 # continue -# 2) pages without hit --> robot - if not super_hit['viewed_hits'][0] and super_hit['viewed_pages'][0]: +# 2) Less than 1 hit per page + if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]): self._setRobot(k, super_hit) continue @@ -118,12 +118,6 @@ class IWLAPreAnalysisRobots(IPlugin): self._setRobot(k, super_hit) continue -# 4) pages without hit --> robot - if not super_hit['viewed_hits'][0] and super_hit['viewed_pages'][0]: - self.logger.debug(super_hit) - self._setRobot(k, super_hit) - continue - not_found_pages = 0 for hit in super_hit['requests']: # 5) /robots.txt read