soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Robots: Improve compatible keyword detection for robots

Browse Source
This commit is contained in:
Gregory Soutade 2024-07-28 09:25:40 +02:00
parent 46c9ae4f15
commit 6d46ac4461
1 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
plugins/pre_analysis/robots.py
View File

@ -61,7 +61,11 @@ class IWLAPreAnalysisRobots(IPlugin):
self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)) self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE) self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE) self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*') self.compatible_re = []
self.compatible_re.append(re.compile(r'.*\(.*compatible; ([^;]+);.*\).*'))
self.compatible_re.append(re.compile(r'.*\(.*compatible; (.*)\).*'))
self.compatible_re.append(re.compile(r'.*\(([^;]+); \+.*\).*'))
self.compatible_re.append(re.compile(r'(.*); \(\+.*\)*'))
self.logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False) self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', []) self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', [])
@ -76,8 +80,10 @@ class IWLAPreAnalysisRobots(IPlugin):
self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno)) self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
super_hit['robot'] = True super_hit['robot'] = True
super_hit['keep_requests'] = False super_hit['keep_requests'] = False
for hit in super_hit['requests']:
robot_name = self.compatible_re.match(hit['http_user_agent']) agent = super_hit['requests'][0]['http_user_agent']
for compatible_re in self.compatible_re:
robot_name = compatible_re.match(agent)
if robot_name: if robot_name:
super_hit['robot_name'] = robot_name[1] super_hit['robot_name'] = robot_name[1]
break break
@ -103,8 +109,7 @@ class IWLAPreAnalysisRobots(IPlugin):
first_page = super_hit['requests'][0] first_page = super_hit['requests'][0]
if self.robot_re.match(first_page['http_user_agent']) or\ if self.robot_re.match(first_page['http_user_agent']) or\
self.crawl_re.match(first_page['http_user_agent']) or\ self.crawl_re.match(first_page['http_user_agent']):
self.compatible_re.match(first_page['http_user_agent']):
self.logger.debug(first_page['http_user_agent']) self.logger.debug(first_page['http_user_agent'])
self._setRobot(k, super_hit) self._setRobot(k, super_hit)
continue continue
@ -147,7 +152,8 @@ class IWLAPreAnalysisRobots(IPlugin):
# Exception for favicon.png and all apple-*icon* # Exception for favicon.png and all apple-*icon*
if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\ if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\
'icon' not in hit['extract_request']['http_uri']: 'icon' not in hit['extract_request']['http_uri'] and\
hit['server_name'] != 'forge.soutade.fr':
error_codes += 1 error_codes += 1
elif int(hit['status']) in (304,): elif int(hit['status']) in (304,):
not_modified_pages += 1 not_modified_pages += 1