soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

New format for (not_)viewed pages/hits and bandwidth that are now recorded by day (in a dictionnary were only element 0 is initialized). Element 0 is the total. WARNING : not backward compatible with previous databases.

Browse Source
This commit is contained in:
Gregory Soutade
2017-08-24 07:55:53 +02:00
parent fffab335fa
commit 007be71ad6
8 changed files with 61 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
plugins/display/all_visits.py
View File

@@ -78,9 +78,9 @@ class IWLADisplayAllVisits(IPlugin):
row = [
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),
super_hit['viewed_pages'][0],
super_hit['viewed_hits'][0],
bytesToStr(super_hit['bandwidth'][0]),
time.asctime(super_hit['last_access'])
]
table.appendRow(row)

4
plugins/display/feeds.py
View File

@@ -87,9 +87,9 @@ class IWLADisplayFeeds(IPlugin):
if super_hit['feed_parser'] == IWLAPostAnalysisFeeds.MERGED_FEED_PARSER:
address += '*'
if super_hit['robot']:
table.appendRow([address, super_hit['not_viewed_pages'], super_hit['not_viewed_hits']])
table.appendRow([address, super_hit['not_viewed_pages'][0], super_hit['not_viewed_hits'][0]])
else:
table.appendRow([address, super_hit['viewed_pages'], super_hit['viewed_hits']])
table.appendRow([address, super_hit['viewed_pages'][0], super_hit['viewed_hits'][0]])
page.appendBlock(table)
note = DisplayHTMLRaw(self.iwla, ('<small>*%s</small>' % (self.iwla._(u'Merged feeds parsers'))))
page.appendBlock(note)

2
plugins/display/robot_bandwidth.py
View File

@@ -68,7 +68,7 @@ class IWLADisplayRobotBandwidth(IPlugin):
for (k, super_hit) in hits.items():
if not self.iwla.isRobot(super_hit):
continue
bandwidths.append((super_hit, super_hit['bandwidth']))
bandwidths.append((super_hit, super_hit['bandwidth'][0]))
bandwidths.sort(key=lambda tup: tup[1], reverse=True)
# All in a page

20
plugins/display/top_visitors.py
View File

@@ -60,11 +60,11 @@ class IWLADisplayTopVisitors(IPlugin):
total = [0]*5
for super_hit in hits.values():
total[1] += super_hit['viewed_pages']
total[2] += super_hit['viewed_hits']
total[3] += super_hit['bandwidth']
total[1] += super_hit['viewed_pages'][0]
total[2] += super_hit['viewed_hits'][0]
total[3] += super_hit['bandwidth'][0]
top_bandwidth = [(k,v['bandwidth']) for (k,v) in hits.items()]
top_bandwidth = [(k,v['bandwidth'][0]) for (k,v) in hits.items()]
top_bandwidth = sorted(top_bandwidth, key=lambda t: t[1], reverse=True)
top_visitors = [hits[h[0]] for h in top_bandwidth[:10]]
@@ -79,14 +79,14 @@ class IWLADisplayTopVisitors(IPlugin):
row = [
address,
super_hit['viewed_pages'],
super_hit['viewed_hits'],
bytesToStr(super_hit['bandwidth']),
super_hit['viewed_pages'][0],
super_hit['viewed_hits'][0],
bytesToStr(super_hit['bandwidth'][0]),
time.asctime(super_hit['last_access'])
]
total[1] -= super_hit['viewed_pages']
total[2] -= super_hit['viewed_hits']
total[3] -= super_hit['bandwidth']
total[1] -= super_hit['viewed_pages'][0]
total[2] -= super_hit['viewed_hits'][0]
total[3] -= super_hit['bandwidth'][0]
table.appendRow(row)
if total[1] or total[2] or total[3]:
total[0] = self.iwla._(u'Others')

4
plugins/post_analysis/feeds.py
View File

@@ -78,7 +78,7 @@ class IWLAPostAnalysisFeeds(IPlugin):
return True
def mergeOneHitOnlyFeedsParsers(self, isFeedParser, one_hit_only, hit):
if isFeedParser and (hit['viewed_hits'] + hit['not_viewed_hits']) == 1:
if isFeedParser and (hit['viewed_hits'][0] + hit['not_viewed_hits'][0]) == 1:
user_agent = hit['requests'][0]['http_user_agent'].lower()
if one_hit_only.get(user_agent, None) is None:
# Merged
@@ -117,7 +117,7 @@ class IWLAPostAnalysisFeeds(IPlugin):
isFeedParser = self.FEED_PARSER
# Robot that views pages -> bot
if hit['robot']:
if hit['not_viewed_pages']:
if hit['not_viewed_pages'][0]:
isFeedParser = self.NOT_A_FEED_PARSER
break
if self.merge_one_hit_only_feeds_parsers:

13
plugins/pre_analysis/page_to_hit.py
View File

@@ -83,14 +83,17 @@ class IWLAPreAnalysisPageToHit(IPlugin):
uri = request['extract_request']['extract_uri']
day = request['time_decoded'].tm_mday
if request['is_page']:
# Page to hit
for regexp in self.ph_regexps:
if regexp.match(uri):
self.logger.debug('%s changed from page to hit' % (uri))
request['is_page'] = False
super_hit['viewed_pages'] -= 1
super_hit['viewed_hits'] += 1
super_hit['viewed_pages'][day] -= 1
super_hit['viewed_hits'][day] = super_hit['viewed_hits'].get(day, 0) + 1
super_hit['viewed_pages'][0] -= 1
super_hit['viewed_hits'][0] += 1
break
else:
# Hit to page
@@ -98,6 +101,8 @@ class IWLAPreAnalysisPageToHit(IPlugin):
if regexp.match(uri):
self.logger.debug('%s changed from hit to page' % (uri))
request['is_page'] = True
super_hit['viewed_pages'] += 1
super_hit['viewed_hits'] -= 1
super_hit['viewed_pages'][day] = super_hit['viewed_pages'].get(day, 0) + 1
super_hit['viewed_hits'][day] -= 1
super_hit['viewed_pages'][0] += 1
super_hit['viewed_hits'][0] -= 1
break

8
plugins/pre_analysis/robots.py
View File

@@ -104,12 +104,12 @@ class IWLAPreAnalysisRobots(IPlugin):
continue
# 1) no pages view --> robot
# if not super_hit['viewed_pages']:
# if not super_hit['viewed_pages'][0]:
# super_hit['robot'] = 1
# continue
# 2) pages without hit --> robot
if not super_hit['viewed_hits']:
if not super_hit['viewed_hits'][0] and super_hit['viewed_pages'][0]:
self.logger.debug(super_hit)
self._setRobot(k, super_hit)
continue
@@ -137,7 +137,7 @@ class IWLAPreAnalysisRobots(IPlugin):
self._setRobot(k, super_hit)
continue
if not super_hit['viewed_pages'] and \
(super_hit['viewed_hits'] and not referers):
if not super_hit['viewed_pages'][0] and \
(super_hit['viewed_hits'][0] and not referers):
self._setRobot(k, super_hit)
continue