Feeds: Add domain and number of subscribers for feed parser.
Set correct date for merged feed parsers Remove bad BAD_FEED_PARSER state
This commit is contained in:
Gregory Soutade
parent
122ee875fa
commit
46c9ae4f15
|
|
@ -72,11 +72,13 @@ class IWLADisplayFeeds(IPlugin):
|
|||
path = self.iwla.getCurDisplayPath(filename)
|
||||
|
||||
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
|
||||
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Last Access')])
|
||||
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', ''])
|
||||
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits')
|
||||
, self.iwla._(u'Domain'), self.iwla._(u'Subscribers'), self.iwla._(u'Last Access')])
|
||||
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', '', '', ''])
|
||||
rows = []
|
||||
for super_hit in hits.values():
|
||||
if not super_hit.get('feed_parser', False): continue
|
||||
if super_hit['feed_parser'] == IWLAPostAnalysisFeeds.BAD_FEED_PARSER:
|
||||
if super_hit.get('feed_parser', None) not in (IWLAPostAnalysisFeeds.FEED_PARSER,\
|
||||
IWLAPostAnalysisFeeds.MERGED_FEED_PARSER):
|
||||
continue
|
||||
nb_feeds_parsers += 1
|
||||
address = super_hit['remote_addr']
|
||||
|
|
@ -84,11 +86,21 @@ class IWLADisplayFeeds(IPlugin):
|
|||
address += ' *'
|
||||
pages = super_hit['not_viewed_pages'][0] + super_hit['viewed_pages'][0]
|
||||
hits = super_hit['not_viewed_hits'][0] + super_hit['viewed_hits'][0]
|
||||
last_access = super_hit.get('feed_parser_last_access', None)
|
||||
if not last_access:
|
||||
last_access = super_hit['last_access']
|
||||
row = [address, pages, hits, time.asctime(last_access)]
|
||||
table.appendRow(row, super_hit['remote_ip'])
|
||||
last_access = super_hit.get('feed_parser_last_access', super_hit['last_access'])
|
||||
feed_domain = super_hit.get('feed_domain', '')
|
||||
if feed_domain:
|
||||
link = '<a href=\'https://%s/%s\'>%s</a>' % (feed_domain, super_hit.get('feed_uri', ''), feed_domain)
|
||||
else:
|
||||
link = ''
|
||||
subscribers = super_hit.get('feed_subscribers', '')
|
||||
# Don't overload interface
|
||||
if subscribers <= 1: subscribers = ''
|
||||
row = [address, pages, hits, link, subscribers, time.asctime(last_access),
|
||||
super_hit['remote_ip'], last_access]
|
||||
rows.append(row)
|
||||
rows = sorted(rows, key=lambda t: t[7], reverse=True)
|
||||
for row in rows:
|
||||
table.appendRow(row[:6], row[6])
|
||||
page.appendBlock(table)
|
||||
note = DisplayHTMLRaw(self.iwla, ('<small>*%s</small>' % (self.iwla._(u'Merged feeds parsers'))))
|
||||
page.appendBlock(note)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#
|
||||
|
||||
import re
|
||||
import time
|
||||
|
||||
from iwla import IWLA
|
||||
from iplugin import IPlugin
|
||||
|
|
@ -47,8 +48,11 @@ Output files :
|
|||
Statistics creation :
|
||||
remote_ip =>
|
||||
feed_parser
|
||||
feed_name_analysed
|
||||
feed_name_analyzed
|
||||
feed_parser_last_access (for merged parser)
|
||||
feed_domain
|
||||
feed_uri
|
||||
feed_subscribers
|
||||
|
||||
Statistics update :
|
||||
None
|
||||
|
|
@ -91,15 +95,21 @@ class IWLAPostAnalysisFeeds(IPlugin):
|
|||
for f in feeds_agents:
|
||||
self.user_agents_re.append(re.compile(f))
|
||||
|
||||
self.bad_user_agents_re = []
|
||||
self.bad_user_agents_re.append(re.compile(r'.*feedback.*'))
|
||||
|
||||
self.subscribers_re = re.compile(r'.* ([0-9]+) subscriber.*')
|
||||
|
||||
self.merge_feeds_parsers_list = []
|
||||
for f in _merge_feeds_parsers_list:
|
||||
self.merge_feeds_parsers_list.append(re.compile(f))
|
||||
|
||||
|
||||
self.merged_feeds = {}
|
||||
|
||||
return True
|
||||
|
||||
def _appendToMergeCache(self, isFeedParser, key, hit):
|
||||
hit['feed_parser'] = isFeedParser
|
||||
# First time, register into dict
|
||||
if self.merged_feeds.get(key, None) is None:
|
||||
# Merged
|
||||
|
|
@ -108,21 +118,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
|
|||
# Next time
|
||||
# Current must be ignored
|
||||
hit['feed_parser'] = self.NOT_A_FEED_PARSER
|
||||
merged_hit = hit
|
||||
last_access = hit['last_access']
|
||||
# Previous matched hit must be set as merged
|
||||
isFeedParser = self.MERGED_FEED_PARSER
|
||||
hit = self.merged_feeds[key]
|
||||
if hit['last_access'] < last_access:
|
||||
hit['feed_parser_last_access'] = last_access
|
||||
hit['feed_parser'] = self.MERGED_FEED_PARSER
|
||||
hit['viewed_pages'][0] += merged_hit['viewed_pages'][0]
|
||||
hit['viewed_hits'][0] += merged_hit['viewed_hits'][0]
|
||||
hit['not_viewed_pages'][0] += merged_hit['not_viewed_pages'][0]
|
||||
hit['not_viewed_hits'][0] += merged_hit['not_viewed_hits'][0]
|
||||
if hit['last_access'] < merged_hit['last_access']:
|
||||
hit['feed_parser_last_access'] = merged_hit['last_access']
|
||||
else:
|
||||
hit['feed_parser_last_access'] = hit['last_access']
|
||||
hit['feed_parser'] = isFeedParser
|
||||
|
||||
def mergeFeedsParsers(self, isFeedParser, hit):
|
||||
if isFeedParser:
|
||||
if isFeedParser in (self.FEED_PARSER, self.MERGED_FEED_PARSER):
|
||||
for r in self.merge_feeds_parsers_list:
|
||||
if r.match(hit['remote_addr']) or r.match(hit['remote_ip']):
|
||||
self._appendToMergeCache(isFeedParser, r, hit)
|
||||
# One group can view multiple different feeds
|
||||
key = r.pattern + hit.get('feed_domain', '') + hit.get('feed_uri', '')
|
||||
self._appendToMergeCache(isFeedParser, key, hit)
|
||||
return
|
||||
#print("No match for %s : %d" % (hit['remote_addr'], hit['viewed_hits'][0] + hit['not_viewed_hits'][0]))
|
||||
# Other cases, look for user agent
|
||||
|
|
@ -134,22 +150,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
|
|||
for hit in hits.values():
|
||||
isFeedParser = hit.get('feed_parser', None)
|
||||
|
||||
# Register already tagged feed parser in merged_feeds
|
||||
if self.merge_feeds_parsers and\
|
||||
not isFeedParser in (None, self.BAD_FEED_PARSER):
|
||||
self.mergeFeedsParsers(isFeedParser, hit)
|
||||
if isFeedParser == self.NOT_A_FEED_PARSER:
|
||||
continue
|
||||
|
||||
|
||||
# Second time
|
||||
if isFeedParser:
|
||||
if hit['feed_parser'] == self.BAD_FEED_PARSER: continue
|
||||
if not hit.get('feed_name_analysed', False) and\
|
||||
# Update last access time
|
||||
if hit['last_access'] > hit.get('feed_parser_last_access', time.gmtime(0)):
|
||||
hit['feed_parser_last_access'] = hit['last_access']
|
||||
|
||||
if not hit.get('feed_name_analyzed', False) and\
|
||||
hit.get('dns_name_replaced', False):
|
||||
hit['feed_name_analysed'] = True
|
||||
hit['feed_name_analyzed'] = True
|
||||
addr = hit.get('remote_addr', None)
|
||||
for r in self.bad_feeds_re:
|
||||
if r.match(addr):
|
||||
hit['feed_parser'] = self.BAD_FEED_PARSER
|
||||
hit['feed_parser'] = self.NOT_A_FEED_PARSER
|
||||
break
|
||||
|
||||
# Register already tagged feed parser in merged_feeds
|
||||
if self.merge_feeds_parsers:
|
||||
self.mergeFeedsParsers(isFeedParser, hit)
|
||||
continue
|
||||
|
||||
request = hit['requests'][0]
|
||||
|
|
@ -164,14 +185,38 @@ class IWLAPostAnalysisFeeds(IPlugin):
|
|||
isFeedParser = self.NOT_A_FEED_PARSER
|
||||
break
|
||||
|
||||
user_agent = request['http_user_agent'].lower()
|
||||
|
||||
if isFeedParser == self.NOT_A_FEED_PARSER:
|
||||
user_agent = request['http_user_agent'].lower()
|
||||
for regexp in self.user_agents_re:
|
||||
if regexp.match(user_agent):
|
||||
isFeedParser = self.FEED_PARSER
|
||||
break
|
||||
|
||||
|
||||
if isFeedParser == self.FEED_PARSER:
|
||||
for regexp in self.bad_user_agents_re:
|
||||
if regexp.match(user_agent):
|
||||
isFeedParser = self.NOT_A_FEED_PARSER
|
||||
break
|
||||
|
||||
if not hit.get('feed_name_analyzed', False) and\
|
||||
hit.get('dns_name_replaced', False):
|
||||
hit['feed_name_analyzed'] = True
|
||||
addr = hit.get('remote_addr', None)
|
||||
for r in self.bad_feeds_re:
|
||||
if r.match(addr):
|
||||
isFeedParser = hit['feed_parser'] = self.NOT_A_FEED_PARSER
|
||||
break
|
||||
|
||||
if isFeedParser == self.FEED_PARSER:
|
||||
hit['feed_domain'] = request['server_name']
|
||||
hit['feed_uri'] = uri
|
||||
hit['feed_subscribers'] = 0
|
||||
|
||||
subscribers = self.subscribers_re.match(user_agent)
|
||||
if subscribers:
|
||||
hit['feed_subscribers'] = int(subscribers.groups()[0])
|
||||
|
||||
hit['feed_parser'] = isFeedParser
|
||||
if self.merge_feeds_parsers:
|
||||
self.mergeFeedsParsers(isFeedParser, hit)
|
||||
else:
|
||||
hit['feed_parser'] = isFeedParser
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user