soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Remove final slashes for referers

Browse Source
This commit is contained in:
Gregory Soutade 2016-01-16 08:43:29 +01:00
parent ec0a3a52a8
commit 5c750e8b0c
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
iwla.py
View File

@ -148,7 +148,7 @@ class IWLA(object):
self.log_re = re.compile(self.log_format_extracted) self.log_re = re.compile(self.log_format_extracted)
self.uri_re = re.compile(r'(?P<extract_uri>[^\?#]+)(\?(?P<extract_parameters>[^#]+))?(#.*)?') self.uri_re = re.compile(r'(?P<extract_uri>[^\?#]+)(\?(?P<extract_parameters>[^#]+))?(#.*)?')
self.domain_name_re = re.compile(r'.*%s' % conf.domain_name) self.domain_name_re = re.compile(r'.*%s' % conf.domain_name)
self.normalize_uri_final_slashes = re.compile(r'/+$') self.final_slashes_re = re.compile(r'/+$')
self.plugins = [(conf.PRE_HOOK_DIRECTORY , conf.pre_analysis_hooks), self.plugins = [(conf.PRE_HOOK_DIRECTORY , conf.pre_analysis_hooks),
(conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks), (conf.POST_HOOK_DIRECTORY , conf.post_analysis_hooks),
(conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)] (conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
@ -337,9 +337,13 @@ class IWLA(object):
def _normalizeURI(self, uri): def _normalizeURI(self, uri):
if uri == '/': return uri if uri == '/': return uri
uri = self.normalize_uri_final_slashes.sub('/', uri) uri = self.final_slashes_re.sub('/', uri)
return uri return uri
def _removeFinalSlashes(self, uri):
if uri == '/': return uri
return self.final_slashes_re.sub('', uri)
def _normalizeParameters(self, parameters): def _normalizeParameters(self, parameters):
# No parameters # No parameters
if parameters == '?': return None if parameters == '?': return None
@ -368,7 +372,7 @@ class IWLA(object):
referer_groups = self.uri_re.match(hit['http_referer']) referer_groups = self.uri_re.match(hit['http_referer'])
if referer_groups: if referer_groups:
hit['extract_referer'] = referer_groups.groupdict() hit['extract_referer'] = referer_groups.groupdict()
hit['extract_referer']['extract_uri'] = self._normalizeURI(hit['extract_referer']['extract_uri']) hit['extract_referer']['extract_uri'] = self._removeFinalSlashes(hit['extract_referer']['extract_uri'])
hit['extract_referer']['extract_parameters'] = self._normalizeParameters(hit['extract_referer']['extract_parameters']) hit['extract_referer']['extract_parameters'] = self._normalizeParameters(hit['extract_referer']['extract_parameters'])
return True return True