soutade/iwla
1
0
Fork 0
Code Issues Pull Requests Releases 2 Activity

Merge with origin:dev

Browse Source
This commit is contained in:
Gregory Soutade 2014-12-31 14:22:46 +01:00
parent a17bbbccf8
commit f1fb8cb674
10 changed files with 157 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
*~
*.pyc
*.gz

10
ChangeLog Normal file
View File

@ -0,0 +1,10 @@
v0.2 (31/12/2014)
** User **
Add referers_diff display plugin
Add year statistics in month details
** Dev **
Add istats_diff interface
** Bugs **
Forgot <body> tag

2
conf.py
View File

@ -19,7 +19,7 @@ reverse_dns_timeout = 0.2
# Count this addresses as hit # Count this addresses as hit
page_to_hit_conf = [r'^.+/logo[/]?$'] page_to_hit_conf = [r'^.+/logo[/]?$']
# Count this addresses as page # Count this addresses as page
hit_to_page_conf = [r'^.+/category/.+$', r'^.+/tag/.+$', r'^.+/archive/.+$', r'^.+/ljdc[/]?$'] hit_to_page_conf = [r'^.+/category/.+$', r'^.+/tag/.+$', r'^.+/archive/.+$', r'^.+/ljdc[/]?$', r'^.+/source/tree/.*$']
# Because it's too long to build HTML when there is too much entries # Because it's too long to build HTML when there is too much entries
max_hits_displayed = 100 max_hits_displayed = 100

8
display.py
View File

@ -52,6 +52,9 @@ class DisplayHTMLRaw(object):
self._buildHTML() self._buildHTML()
self._build(f, self.html) self._build(f, self.html)
def getTitle(self):
return ''
class DisplayHTMLBlock(DisplayHTMLRaw): class DisplayHTMLBlock(DisplayHTMLRaw):
def __init__(self, iwla, title=''): def __init__(self, iwla, title=''):
@ -287,7 +290,7 @@ class DisplayHTMLPage(object):
def appendBlock(self, block): def appendBlock(self, block):
self.blocks.append(block) self.blocks.append(block)
def build(self, root): def build(self, root, displayVersion=True):
filename = os.path.join(root, self.filename) filename = os.path.join(root, self.filename)
base = os.path.dirname(filename) base = os.path.dirname(filename)
@ -305,9 +308,10 @@ class DisplayHTMLPage(object):
f.write(u'<link rel="stylesheet" href="/%s"/>' % (css)) f.write(u'<link rel="stylesheet" href="/%s"/>' % (css))
if self.title: if self.title:
f.write(u'<title>%s</title>' % (self.title)) f.write(u'<title>%s</title>' % (self.title))
f.write(u'</head>') f.write(u'</head><body>')
for block in self.blocks: for block in self.blocks:
block.build(f) block.build(f)
if displayVersion:
f.write(u'<center>Generated by <a href="%s">IWLA %s</a></center>' % f.write(u'<center>Generated by <a href="%s">IWLA %s</a></center>' %
("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion())) ("http://indefero.soutade.fr/p/iwla", self.iwla.getVersion()))
f.write(u'</body></html>') f.write(u'</body></html>')

18
iwla.py
View File

@ -59,6 +59,7 @@ Output files :
DB_ROOT/meta.db DB_ROOT/meta.db
DB_ROOT/year/month/iwla.db DB_ROOT/year/month/iwla.db
OUTPUT_ROOT/index.html OUTPUT_ROOT/index.html
OUTPUT_ROOT/year/_stats.html
OUTPUT_ROOT/year/month/index.html OUTPUT_ROOT/year/month/index.html
Statistics creation : Statistics creation :
@ -129,7 +130,7 @@ class IWLA(object):
ANALYSIS_CLASS = 'HTTP' ANALYSIS_CLASS = 'HTTP'
API_VERSION = 1 API_VERSION = 1
IWLA_VERSION = '0.1' IWLA_VERSION = '0.2-dev'
def __init__(self, logLevel): def __init__(self, logLevel):
self.meta_infos = {} self.meta_infos = {}
@ -371,6 +372,8 @@ class IWLA(object):
filename = self.getCurDisplayPath('index.html') filename = self.getCurDisplayPath('index.html')
self.logger.info('==> Generate display (%s)' % (filename)) self.logger.info('==> Generate display (%s)' % (filename))
page = self.display.createPage(title, filename, conf.css_path) page = self.display.createPage(title, filename, conf.css_path)
link = DisplayHTMLRaw(self, '<iframe src="../_stats.html"></iframe>')
page.appendBlock(link)
_, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon) _, nb_month_days = monthrange(cur_time.tm_year, cur_time.tm_mon)
days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, self._('By day'), [self._('Day'), self._('Visits'), self._('Pages'), self._('Hits'), self._('Bandwidth'), self._('Not viewed Bandwidth')], None, nb_month_days, range(1,6)) days = self.display.createBlock(DisplayHTMLBlockTableWithGraph, self._('By day'), [self._('Day'), self._('Visits'), self._('Pages'), self._('Hits'), self._('Bandwidth'), self._('Not viewed Bandwidth')], None, nb_month_days, range(1,6))
@ -430,6 +433,8 @@ class IWLA(object):
graph_cols=range(1,7) graph_cols=range(1,7)
months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols) months = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols, None, 12, graph_cols)
months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', '']) months.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth', ''])
months_ = self.display.createBlock(DisplayHTMLBlockTableWithGraph, title, cols[:-1], None, 12, graph_cols[:-1])
months_.setColsCSSClass(['', 'iwla_visitor', 'iwla_visit', 'iwla_page', 'iwla_hit', 'iwla_bandwidth', 'iwla_bandwidth'])
total = [0] * len(cols) total = [0] * len(cols)
for i in range(1, 13): for i in range(1, 13):
month = '%s<br/>%d' % (months_name[i], year) month = '%s<br/>%d' % (months_name[i], year)
@ -447,11 +452,16 @@ class IWLA(object):
months.setCellValue(i-1, 5, bytesToStr(row[5])) months.setCellValue(i-1, 5, bytesToStr(row[5]))
months.setCellValue(i-1, 6, bytesToStr(row[6])) months.setCellValue(i-1, 6, bytesToStr(row[6]))
months.appendShortTitle(month) months.appendShortTitle(month)
months_.appendRow(row[:-1])
months_.setCellValue(i-1, 5, bytesToStr(row[5]))
months_.setCellValue(i-1, 6, bytesToStr(row[6]))
months_.appendShortTitle(month)
if year == cur_time.tm_year and i == cur_time.tm_mon: if year == cur_time.tm_year and i == cur_time.tm_mon:
css = months.getCellCSSClass(i-1, 0) css = months.getCellCSSClass(i-1, 0)
if css: css = '%s %s' % (css, 'iwla_curday') if css: css = '%s %s' % (css, 'iwla_curday')
else: css = 'iwla_curday' else: css = 'iwla_curday'
months.setCellCSSClass(i-1, 0, css) months.setCellCSSClass(i-1, 0, css)
months_.setCellCSSClass(i-1, 0, css)
total[0] = self._('Total') total[0] = self._('Total')
total[5] = bytesToStr(total[5]) total[5] = bytesToStr(total[5])
@ -460,6 +470,12 @@ class IWLA(object):
months.appendRow(total) months.appendRow(total)
page.appendBlock(months) page.appendBlock(months)
months_.appendRow(total[:-1])
filename = '%d/_stats.html' % (year)
page_ = self.display.createPage(u'', filename, conf.css_path)
page_.appendBlock(months_)
page_.build(conf.DISPLAY_ROOT, False)
def _generateDisplayWholeMonthStats(self): def _generateDisplayWholeMonthStats(self):
title = '%s %s' % (self._('Statistics for'), conf.domain_name) title = '%s %s' % (self._('Statistics for'), conf.domain_name)
filename = 'index.html' filename = 'index.html'

98
plugins/display/istats_diff.py Normal file
View File

@ -0,0 +1,98 @@
# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015
# This file is part of iwla
# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla. If not, see <http://www.gnu.org/licenses/>.
#
from iwla import IWLA
from iplugin import IPlugin
from display import *
import logging
"""
Display hook itnerface
Enlight new and updated statistics
Plugin requirements :
None
Conf values needed :
None
Output files :
None
Statistics creation :
None
Statistics update :
None
Statistics deletion :
None
"""
class IWLADisplayStatsDiff(IPlugin):
def __init__(self, iwla):
super(IWLADisplayStatsDiff, self).__init__(iwla)
self.API_VERSION = 1
self.month_stats_key = None
# Set >= if month_stats[self.month_stats_key] is a list or a tuple
self.stats_index = -1
self.filename = None
self.block_name = None
self.logger = logging.getLogger(__name__)
def load(self):
if not self.month_stats_key or not self.filename or\
not self.block_name:
self.logger('Bad parametrization')
return False
month_stats = self.iwla.getMonthStats()
self.cur_stats = {k:v for (k,v) in month_stats.get(self.month_stats_key, {}).items()}
return True
def hook(self):
display = self.iwla.getDisplay()
month_stats = self.iwla.getMonthStats()
path = self.iwla.getCurDisplayPath(self.filename)
page = display.getPage(path)
if not page: return
title = self.iwla._(self.block_name)
block = page.getBlock(title)
if not block:
self.logger.error('Block %s not found' % (title))
return
stats_diff = {}
for (k, v) in month_stats[self.month_stats_key].items():
new_value = self.cur_stats.get(k, 0)
if new_value:
if self.stats_index != -1:
if new_value[self.stats_index] != v[self.stats_index]:
stats_diff[k] = 'iwla_update'
else:
if new_value != v:
stats_diff[k] = 'iwla_update'
else:
stats_diff[k] = 'iwla_new'
for (idx, row) in enumerate(block.rows):
if row[0] in stats_diff.keys():
block.setCellCSSClass(idx, 0, stats_diff[row[0]])

4
plugins/display/referers.py
View File

@ -190,14 +190,14 @@ class IWLADisplayReferers(IPlugin):
# All key phrases in a file # All key phrases in a file
if self.create_all_key_phrases: if self.create_all_key_phrases:
title = createCurTitle(self.iwla, u'Key Phrases') title = createCurTitle(self.iwla, u'All Key Phrases')
filename = 'key_phrases.html' filename = 'key_phrases.html'
path = self.iwla.getCurDisplayPath(filename) path = self.iwla.getCurDisplayPath(filename)
total_search = [0]*2 total_search = [0]*2
page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Top key phrases'), [self.iwla._(u'Key phrase'), self.iwla._(u'Search')]) table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Key phrases'), [self.iwla._(u'Key phrase'), self.iwla._(u'Search')])
table.setColsCSSClass(['', 'iwla_search']) table.setColsCSSClass(['', 'iwla_search'])
new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases new_list = self.max_key_phrases and top_key_phrases[:self.max_key_phrases] or top_key_phrases
for phrase in new_list: for phrase in new_list:

35
plugins/display/referers_diff.py
View File

@ -19,7 +19,7 @@
# #
from iwla import IWLA from iwla import IWLA
from iplugin import IPlugin from istats_diff import IWLADisplayStatsDiff
from display import * from display import *
""" """
@ -46,39 +46,16 @@ Statistics deletion :
None None
""" """
class IWLADisplayReferersDiff(IPlugin): class IWLADisplayReferersDiff(IWLADisplayStatsDiff):
def __init__(self, iwla): def __init__(self, iwla):
super(IWLADisplayReferersDiff, self).__init__(iwla) super(IWLADisplayReferersDiff, self).__init__(iwla)
self.API_VERSION = 1 self.API_VERSION = 1
self.requires = ['IWLADisplayReferers'] self.requires = ['IWLADisplayReferers']
self.month_stats_key = 'key_phrases'
self.filename = 'key_phrases.html'
self.block_name = u'Key phrases'
def load(self): def load(self):
if not self.iwla.getConfValue('create_all_key_phrases_page', True): if not self.iwla.getConfValue('create_all_key_phrases_page', True):
return False return False
month_stats = self.iwla.getMonthStats() return super(IWLADisplayReferersDiff, self).load()
self.cur_key_phrases = {k:v for (k,v) in month_stats.get('key_phrases', {})}
return True
def hook(self):
display = self.iwla.getDisplay()
month_stats = self.iwla.getMonthStats()
filename = 'key_phrases.html'
path = self.iwla.getCurDisplayPath(filename)
page = display.getPage(path)
if not page: return
title = self.iwla._(u'All key phrases')
referers_block = page.getBlock(title)
kp_diff = {}
for (k, v) in month_stats['key_phrases'].items():
new_value = self.cur_key_phrases.get(k, 0)
if new_value:
if new_value != v:
kp_diff[k] = 'iwla_update'
else:
kp_diff[k] = 'iwla_new'
for (idx, row) in enumerate(referers_block.rows):
if row[0] in kp_diff.keys():
referers_block.setCellCSSClass(idx, 0, kp_diff[row[0]])

5
resources/css/iwla.css
View File

@ -69,6 +69,9 @@ td:first-child
.iwla_weekend { background : #ECECEC; } .iwla_weekend { background : #ECECEC; }
.iwla_curday { font-weight: bold; } .iwla_curday { font-weight: bold; }
.iwla_others { color: #668; } .iwla_others { color: #668; }
.iwla_update { background : orange; }
.iwla_new { background : green }
.iwla_graph_table .iwla_graph_table
{ {
margin-left:auto; margin-left:auto;
@ -85,3 +88,5 @@ table.iwla_graph_table td
{ {
text-align:center; text-align:center;
} }
iframe {outline:none; border:0px; width:100%; height:500px; display:block;}

7
tools/extract_doc.py
View File

@ -2,11 +2,18 @@
import sys import sys
excludes = ['istats_diff.py']
filename = sys.argv[1] filename = sys.argv[1]
if filename.endswith('__init__.py'): if filename.endswith('__init__.py'):
sys.exit(0) sys.exit(0)
for e in excludes:
if filename.endswith(e):
sys.stderr.write('\tSkip %s\n' % (filename))
sys.exit(0)
package_name = filename.replace('/', '.').replace('.py', '') package_name = filename.replace('/', '.').replace('.py', '')
sys.stdout.write('%s' % (package_name)) sys.stdout.write('%s' % (package_name))
sys.stdout.write('\n') sys.stdout.write('\n')