soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Add a star for merged feeds parsers

Browse Source
This commit is contained in:
Gregory Soutade 2015-02-18 20:56:03 +01:00
parent cfbd35d818
commit f5b0c35bad
2 changed files with 24 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
plugins/display/feeds.py
View File

@ -21,8 +21,7 @@
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from iplugin import IPlugin
from display import * from display import *
from plugins.post_analysis.feeds import IWLAPostAnalysisFeeds
import awstats_data
""" """
Display hook Display hook
@ -81,12 +80,15 @@ class IWLADisplayFeeds(IPlugin):
if display_visitor_ip and\ if display_visitor_ip and\
super_hit.get('dns_name_replaced', False): super_hit.get('dns_name_replaced', False):
address = '%s [%s]' % (address, super_hit['remote_ip']) address = '%s [%s]' % (address, super_hit['remote_ip'])
if super_hit['feed_parser'] == IWLAPostAnalysisFeeds.MERGED_FEED_PARSER:
address += '*'
if super_hit['robot']: if super_hit['robot']:
table.appendRow([address, super_hit['not_viewed_pages'], super_hit['not_viewed_hits']]) table.appendRow([address, super_hit['not_viewed_pages'], super_hit['not_viewed_hits']])
else: else:
table.appendRow([address, super_hit['viewed_pages'], super_hit['viewed_hits']]) table.appendRow([address, super_hit['viewed_pages'], super_hit['viewed_hits']])
page.appendBlock(table) page.appendBlock(table)
note = DisplayHTMLRaw(self.iwla, ('<small>*%s</small>' % (self.iwla._('Merged feeds parsers'))))
page.appendBlock(note)
display.addPage(page) display.addPage(page)
# Found in index # Found in index

23
plugins/post_analysis/feeds.py
View File

@ -35,6 +35,7 @@ Plugin requirements :
Conf values needed : Conf values needed :
feeds feeds
merge_one_hit_only_feeds_parsers*
Output files : Output files :
None None
@ -51,12 +52,18 @@ Statistics deletion :
""" """
class IWLAPostAnalysisFeeds(IPlugin): class IWLAPostAnalysisFeeds(IPlugin):
NOT_A_FEED_PARSER = 0
FEED_PARSER = 1
MERGED_FEED_PARSER = 2
def __init__(self, iwla): def __init__(self, iwla):
super(IWLAPostAnalysisFeeds, self).__init__(iwla) super(IWLAPostAnalysisFeeds, self).__init__(iwla)
self.API_VERSION = 1 self.API_VERSION = 1
self.conf_requires = ['feeds']
def load(self): def load(self):
feeds = self.iwla.getConfValue('feeds', None) feeds = self.iwla.getConfValue('feeds', None)
self.merge_one_hit_only_feeds_parsers = self.iwla.getConfValue('merge_one_hit_only_feeds_parsers', True)
if feeds is None: return False if feeds is None: return False
@ -70,9 +77,11 @@ class IWLAPostAnalysisFeeds(IPlugin):
if isFeedParser and (hit['viewed_hits'] + hit['not_viewed_hits']) == 1: if isFeedParser and (hit['viewed_hits'] + hit['not_viewed_hits']) == 1:
user_agent = hit['requests'][0]['http_user_agent'].lower() user_agent = hit['requests'][0]['http_user_agent'].lower()
if one_hit_only.get(user_agent, None) is None: if one_hit_only.get(user_agent, None) is None:
# Merged
isFeedParser = self.MERGED_FEED_PARSER
one_hit_only[user_agent] = (hit) one_hit_only[user_agent] = (hit)
else: else:
isFeedParser = False isFeedParser = self.NOT_A_FEED_PARSER
hit['feed_parser'] = isFeedParser hit['feed_parser'] = isFeedParser
def hook(self): def hook(self):
@ -81,19 +90,23 @@ class IWLAPostAnalysisFeeds(IPlugin):
for hit in hits.values(): for hit in hits.values():
isFeedParser = hit.get('feed_parser', None) isFeedParser = hit.get('feed_parser', None)
if isFeedParser == True: if isFeedParser == self.FEED_PARSER and\
self.merge_one_hit_only_feeds_parsers:
self.mergeOneHitOnlyFeedsParsers(one_hit_only, hit) self.mergeOneHitOnlyFeedsParsers(one_hit_only, hit)
if not isFeedParser is None: continue if not isFeedParser is None: continue
isFeedParser = False isFeedParser = self.NOT_A_FEED_PARSER
uri = hit['requests'][0]['extract_request']['extract_uri'].lower() uri = hit['requests'][0]['extract_request']['extract_uri'].lower()
for regexp in self.feeds_re: for regexp in self.feeds_re:
if regexp.match(uri): if regexp.match(uri):
isFeedParser = True isFeedParser = self.FEED_PARSER
# Robot that views pages -> bot # Robot that views pages -> bot
if hit['robot']: if hit['robot']:
if hit['viewed_pages']: if hit['viewed_pages']:
isFeedParser = False isFeedParser = self.NOT_A_FEED_PARSER
break break
if self.merge_one_hit_only_feeds_parsers:
self.mergeOneHitOnlyFeedsParsers(isFeedParser, one_hit_only, hit) self.mergeOneHitOnlyFeedsParsers(isFeedParser, one_hit_only, hit)
else:
hit['feed_parser'] = isFeedParser