iwla/plugins/display/referers.py
2014-12-30 16:21:49 +01:00

226 lines
9.8 KiB
Python

# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from display import *
"""
Display hook
Create Referers page
Plugin requirements :
post_analysis/referers
Conf values needed :
max_referers_displayed*
create_all_referers_page*
max_key_phrases_displayed*
create_all_key_phrases_page*
Output files :
OUTPUT_ROOT/year/month/referers.html
OUTPUT_ROOT/year/month/key_phrases.html
OUTPUT_ROOT/year/month/index.html
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayReferers(IPlugin):
def __init__(self, iwla):
super(IWLADisplayReferers, self).__init__(iwla)
self.API_VERSION = 1
self.requires = ['IWLAPostAnalysisReferers']
self.max_referers = self.iwla.getConfValue('max_referers_displayed', 0)
self.create_all_referers = self.iwla.getConfValue('create_all_referers_page', True)
self.max_key_phrases = self.iwla.getConfValue('max_key_phrases_displayed', 0)
self.create_all_key_phrases = self.iwla.getConfValue('create_all_key_phrases_page', True)
def hook(self):
display = self.iwla.getDisplay()
month_stats = self.iwla.getMonthStats()
referers = month_stats.get('referers', {})
robots_referers = month_stats.get('robots_referers', {})
search_engine_referers = month_stats.get('search_engine_referers', {})
key_phrases = month_stats.get('key_phrases', {})
top_referers = [(k, referers[k]['pages']) for k in referers.keys()]
top_referers = sorted(top_referers, key=lambda t: t[1], reverse=True)
top_robots_referers = [(k, robots_referers[k]['pages']) for k in robots_referers.keys()]
top_robots_referers = sorted(top_robots_referers, key=lambda t: t[1], reverse=True)
top_search_engine_referers = [(k, search_engine_referers[k]['pages']) for k in search_engine_referers.keys()]
top_search_engine_referers = sorted(top_search_engine_referers, key=lambda t: t[1], reverse=True)
top_key_phrases = key_phrases.items()
top_key_phrases = sorted(top_key_phrases, key=lambda t: t[1], reverse=True)
cur_time = self.iwla.getCurTime()
index = self.iwla.getDisplayIndex()
# All referers in a file
if self.create_all_referers:
title = createCurTitle(self.iwla, u'Connexion from')
filename = 'referers.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Connexion from'), [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
total_search = [0]*3
table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
new_list = self.max_referers and top_search_engine_referers[:self.max_referers] or top_search_engine_referers
for r,_ in new_list:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
total_search[1] += search_engine_referers[r]['pages']
total_search[2] += search_engine_referers[r]['hits']
table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
total_external = [0]*3
table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
new_list = self.max_referers and top_referers[:self.max_referers] or top_referers
for r,_ in new_list:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
total_external[1] += referers[r]['pages']
total_external[2] += referers[r]['hits']
table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
total_robot = [0]*3
table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
new_list = self.max_referers and top_robots_referers[:self.max_referers] or top_robots_referers
for r,_ in new_list:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
total_robot[1] += robots_referers[r]['pages']
total_robot[2] += robots_referers[r]['hits']
table.appendRow(row)
if self.max_referers:
others = 0
for (uri, entrance) in top_referers[self.max_referers:]:
others += entrance
table.appendRow([self.iwla._(u'Others'), others])
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
page.appendBlock(table)
display.addPage(page)
title = self.iwla._(u'Top Referers')
if self.create_all_referers:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All Referers'))
title = '%s - %s' % (title, link)
# Top referers in index
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Origin'), self.iwla._(u'Pages'), self.iwla._(u'Hits')])
table.setColsCSSClass(['', 'iwla_page', 'iwla_hit'])
table.appendRow(['<b>%s</b>' % (self.iwla._(u'Search Engine')), '', ''])
for r,_ in top_search_engine_referers[:10]:
row = [r, search_engine_referers[r]['pages'], search_engine_referers[r]['hits']]
total_search[1] -= search_engine_referers[r]['pages']
total_search[2] -= search_engine_referers[r]['hits']
table.appendRow(row)
if total_search[1] or total_search[2]:
total_search[0] = self.iwla._(u'Others')
table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL')), '', ''])
for r,_ in top_referers[:10]:
row = [generateHTMLLink(r), referers[r]['pages'], referers[r]['hits']]
total_external[1] -= referers[r]['pages']
total_external[2] -= referers[r]['hits']
table.appendRow(row)
if total_external[1] or total_external[2]:
total_external[0] = self.iwla._(u'Others')
table.appendRow(total_external)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
table.appendRow(['<b>%s</b>' % (self.iwla._(u'External URL (robot)')), '', ''])
for r,_ in top_robots_referers[:10]:
row = [generateHTMLLink(r), robots_referers[r]['pages'], robots_referers[r]['hits']]
total_robot[1] -= robots_referers[r]['pages']
total_robot[2] -= robots_referers[r]['hits']
table.appendRow(row)
if total_robot[1] or total_robot[2]:
total_robot[0] = self.iwla._(u'Others')
table.appendRow(total_robot)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)
# All key phrases in a file
if self.create_all_key_phrases:
title = createCurTitle(self.iwla, u'Key Phrases')
filename = 'key_phrases.html'
path = self.iwla.getCurDisplayPath(filename)
total_search = [0]*2
page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top key phrases'), [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
table.setColsCSSClass(['', 'iwla_search'])
new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases
for phrase in new_list:
table.appendRow([phrase[0], phrase[1]])
total_search[1] += phrase[1]
page.appendBlock(table)
display.addPage(page)
title = self.iwla._(u'Top key phrases')
if self.create_all_key_phrases:
link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'All key phrases'))
title = '%s - %s' % (title, link)
# Top key phrases in index
table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
table.setColsCSSClass(['', 'iwla_search'])
for phrase in top_key_phrases[:10]:
table.appendRow([phrase[0], phrase[1]])
total_search[1] -= phrase[1]
if total_search[1]:
total_search[0] = self.iwla._(u'Others')
table.appendRow(total_search)
table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
index.appendBlock(table)