diff --git a/plugins/pre_analysis/robots.py b/plugins/pre_analysis/robots.py index 47036d4..dbc1148 100644 --- a/plugins/pre_analysis/robots.py +++ b/plugins/pre_analysis/robots.py @@ -61,7 +61,11 @@ class IWLAPreAnalysisRobots(IPlugin): self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)) self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE) self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE) - self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*') + self.compatible_re = [] + self.compatible_re.append(re.compile(r'.*\(.*compatible; ([^;]+);.*\).*')) + self.compatible_re.append(re.compile(r'.*\(.*compatible; (.*)\).*')) + self.compatible_re.append(re.compile(r'.*\(([^;]+); \+.*\).*')) + self.compatible_re.append(re.compile(r'(.*); \(\+.*\)*')) self.logger = logging.getLogger(self.__class__.__name__) self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False) self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', []) @@ -76,12 +80,14 @@ class IWLAPreAnalysisRobots(IPlugin): self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno)) super_hit['robot'] = True super_hit['keep_requests'] = False - for hit in super_hit['requests']: - robot_name = self.compatible_re.match(hit['http_user_agent']) + + agent = super_hit['requests'][0]['http_user_agent'] + for compatible_re in self.compatible_re: + robot_name = compatible_re.match(agent) if robot_name: super_hit['robot_name'] = robot_name[1] break - + # Basic rule to detect robots def hook(self): hits = self.iwla.getCurrentVisits() @@ -101,10 +107,9 @@ class IWLAPreAnalysisRobots(IPlugin): referers = 0 first_page = super_hit['requests'][0] - + if self.robot_re.match(first_page['http_user_agent']) or\ - self.crawl_re.match(first_page['http_user_agent']) or\ - self.compatible_re.match(first_page['http_user_agent']): + self.crawl_re.match(first_page['http_user_agent']): self.logger.debug(first_page['http_user_agent']) self._setRobot(k, super_hit) continue @@ -147,7 +152,8 @@ class IWLAPreAnalysisRobots(IPlugin): # Exception for favicon.png and all apple-*icon* if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\ - 'icon' not in hit['extract_request']['http_uri']: + 'icon' not in hit['extract_request']['http_uri'] and\ + hit['server_name'] != 'forge.soutade.fr': error_codes += 1 elif int(hit['status']) in (304,): not_modified_pages += 1