soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Feeds: Add domain and number of subscribers for feed parser.

Browse Source 
Set correct date for merged feed parsers
Remove bad BAD_FEED_PARSER state
This commit is contained in:
Gregory Soutade
2024-07-28 09:25:06 +02:00
parent 122ee875fa
commit 46c9ae4f15
2 changed files with 87 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
plugins/post_analysis/feeds.py
View File

@@ -19,6 +19,7 @@
#
import re
import time
from iwla import IWLA
from iplugin import IPlugin
@@ -47,8 +48,11 @@ Output files :
Statistics creation :
remote_ip =>
feed_parser
feed_name_analysed
feed_name_analyzed
feed_parser_last_access (for merged parser)
feed_domain
feed_uri
feed_subscribers
Statistics update :
None
@@ -91,15 +95,21 @@ class IWLAPostAnalysisFeeds(IPlugin):
for f in feeds_agents:
self.user_agents_re.append(re.compile(f))
self.bad_user_agents_re = []
self.bad_user_agents_re.append(re.compile(r'.*feedback.*'))
self.subscribers_re = re.compile(r'.* ([0-9]+) subscriber.*')
self.merge_feeds_parsers_list = []
for f in _merge_feeds_parsers_list:
self.merge_feeds_parsers_list.append(re.compile(f))
self.merged_feeds = {}
return True
def _appendToMergeCache(self, isFeedParser, key, hit):
hit['feed_parser'] = isFeedParser
# First time, register into dict
if self.merged_feeds.get(key, None) is None:
# Merged
@@ -108,21 +118,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
# Next time
# Current must be ignored
hit['feed_parser'] = self.NOT_A_FEED_PARSER
merged_hit = hit
last_access = hit['last_access']
# Previous matched hit must be set as merged
isFeedParser = self.MERGED_FEED_PARSER
hit = self.merged_feeds[key]
if hit['last_access'] < last_access:
hit['feed_parser_last_access'] = last_access
hit['feed_parser'] = self.MERGED_FEED_PARSER
hit['viewed_pages'][0] += merged_hit['viewed_pages'][0]
hit['viewed_hits'][0] += merged_hit['viewed_hits'][0]
hit['not_viewed_pages'][0] += merged_hit['not_viewed_pages'][0]
hit['not_viewed_hits'][0] += merged_hit['not_viewed_hits'][0]
if hit['last_access'] < merged_hit['last_access']:
hit['feed_parser_last_access'] = merged_hit['last_access']
else:
hit['feed_parser_last_access'] = hit['last_access']
hit['feed_parser'] = isFeedParser
def mergeFeedsParsers(self, isFeedParser, hit):
if isFeedParser:
if isFeedParser in (self.FEED_PARSER, self.MERGED_FEED_PARSER):
for r in self.merge_feeds_parsers_list:
if r.match(hit['remote_addr']) or r.match(hit['remote_ip']):
self._appendToMergeCache(isFeedParser, r, hit)
# One group can view multiple different feeds
key = r.pattern + hit.get('feed_domain', '') + hit.get('feed_uri', '')
self._appendToMergeCache(isFeedParser, key, hit)
return
#print("No match for %s : %d" % (hit['remote_addr'], hit['viewed_hits'][0] + hit['not_viewed_hits'][0]))
# Other cases, look for user agent
@@ -134,22 +150,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
for hit in hits.values():
isFeedParser = hit.get('feed_parser', None)
# Register already tagged feed parser in merged_feeds
if self.merge_feeds_parsers and\
not isFeedParser in (None, self.BAD_FEED_PARSER):
self.mergeFeedsParsers(isFeedParser, hit)
if isFeedParser == self.NOT_A_FEED_PARSER:
continue
# Second time
if isFeedParser:
if hit['feed_parser'] == self.BAD_FEED_PARSER: continue
if not hit.get('feed_name_analysed', False) and\
# Update last access time
if hit['last_access'] > hit.get('feed_parser_last_access', time.gmtime(0)):
hit['feed_parser_last_access'] = hit['last_access']
if not hit.get('feed_name_analyzed', False) and\
hit.get('dns_name_replaced', False):
hit['feed_name_analysed'] = True
hit['feed_name_analyzed'] = True
addr = hit.get('remote_addr', None)
for r in self.bad_feeds_re:
if r.match(addr):
hit['feed_parser'] = self.BAD_FEED_PARSER
hit['feed_parser'] = self.NOT_A_FEED_PARSER
break
# Register already tagged feed parser in merged_feeds
if self.merge_feeds_parsers:
self.mergeFeedsParsers(isFeedParser, hit)
continue
request = hit['requests'][0]
@@ -164,14 +185,38 @@ class IWLAPostAnalysisFeeds(IPlugin):
isFeedParser = self.NOT_A_FEED_PARSER
break
user_agent = request['http_user_agent'].lower()
if isFeedParser == self.NOT_A_FEED_PARSER:
user_agent = request['http_user_agent'].lower()
for regexp in self.user_agents_re:
if regexp.match(user_agent):
isFeedParser = self.FEED_PARSER
break
if isFeedParser == self.FEED_PARSER:
for regexp in self.bad_user_agents_re:
if regexp.match(user_agent):
isFeedParser = self.NOT_A_FEED_PARSER
break
if not hit.get('feed_name_analyzed', False) and\
hit.get('dns_name_replaced', False):
hit['feed_name_analyzed'] = True
addr = hit.get('remote_addr', None)
for r in self.bad_feeds_re:
if r.match(addr):
isFeedParser = hit['feed_parser'] = self.NOT_A_FEED_PARSER
break
if isFeedParser == self.FEED_PARSER:
hit['feed_domain'] = request['server_name']
hit['feed_uri'] = uri
hit['feed_subscribers'] = 0
subscribers = self.subscribers_re.match(user_agent)
if subscribers:
hit['feed_subscribers'] = int(subscribers.groups()[0])
hit['feed_parser'] = isFeedParser
if self.merge_feeds_parsers:
self.mergeFeedsParsers(isFeedParser, hit)
else:
hit['feed_parser'] = isFeedParser