iwla/plugins/display/filter_users.py

# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2020

# This file is part of iwla

# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
#

import re

from iwla import IWLA
from iplugin import IPlugin
from display import *

"""
Display hook

Filter users

Plugin requirements :
    None

Conf values needed :
    create_filtered_page*

Output files :
    OUTPUT_ROOT/year/month/index.html
    OUTPUT_ROOT/year/month/filtered_users.html

Statistics creation :
    None

Statistics update :
    None

Statistics deletion :
    None
"""

class IWLADisplayFilterUsers(IPlugin):
    def __init__(self, iwla):
        super(IWLADisplayFilterUsers, self).__init__(iwla)
        self.API_VERSION = 1
        self.requires = ['IWLAPostAnalysisFilterUsers']
        self.create_filtered_page = self.iwla.getConfValue('create_filtered_page', True)
        
    def hook(self):
        display = self.iwla.getDisplay()
        hits = self.iwla.getValidVisitors()
        stats = {}
        self.filtered_users = []

        _hits = sorted(hits.items(), key=lambda x: x[1]['last_access'], reverse=True)
        for (key,value) in _hits:
            if value.get('filtered', False):
                self.filtered_users.append(value)
            
        # All in a page
        if self.create_filtered_page:
            title = createCurTitle(self.iwla, u'Filtered users')
            filename = 'filtered_users.html'
            path = self.iwla.getCurDisplayPath(filename)

            page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Filtered users'), [self.iwla._(u'Pages'), self.iwla._(u'Last Access'), self.iwla._(u'User Agent')])
            table.setColsCSSClass(['iwla_page', '', ''])
            row = 0
            unknown = self.iwla._('Unknown')
            for filtered_user in self.filtered_users:
                ip = filtered_user['remote_ip']
                ip_title = ip
                if 'dns_name_replaced' in hits[ip].keys():
                    ip_title = '%s [%s]' % (hits[ip]['remote_addr'], ip)
                if filtered_user.get('operating_system', 'android') not in ('android', 'ios_iphone'):
                    location = filtered_user.get('geo_location', {})
                    if location:
                        location_str = '(%s/%s)' % (location.get('city', unknown), location.get('countryname', unknown))
                        ip_title = f'{ip_title}<br/>{location_str}'
                table.appendRow([f'<b>{ip_title}</b>', '', ''])
                table.setCellCSSClass(row, 0, '')
                for r in hits[ip]['requests'][::-1]:
                    uri = r['extract_request']['extract_uri'].lower()
                    if not self.iwla.isPage(uri): continue
                    if not self.iwla.hasBeenViewed(r): continue

                    uri = "%s%s" % (r.get('server_name', ''),
                                    r['extract_request']['extract_uri'])
                    table.appendRow([generateHTMLLink(uri), time.asctime(r['time_decoded']), r['http_user_agent']])
            page.appendBlock(table)

            display.addPage(page)

        # Last access in index
        title = self.iwla._(u'Filtered users')
        if self.create_filtered_page:
            link = '<a href=\'%s\'>%s</a>' % (filename, self.iwla._(u'Details'))
            title = '%s - %s' % (title, link)

        index = self.iwla.getDisplayIndex()
        
        table = display.createBlock(DisplayHTMLBlockTable, title, [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Last Access')])
        table.setColsCSSClass(['', '', 'iwla_page', 'iwla_hit'])
        for filtered_user in self.filtered_users[:10]:
            ip = filtered_user['remote_ip']
            if 'dns_name_replaced' in hits[ip].keys():
                ip_title = '%s [%s]' % (hits[ip]['remote_addr'], ip)
            else:
                ip_title = ip
            table.appendRow([ip_title, filtered_user['viewed_pages'][0], filtered_user['viewed_hits'][0], time.asctime(hits[ip]['last_access'])])
        if len(self.filtered_users) > 10:
            table.appendRow([self.iwla._(u'Others'), len(self.filtered_users)-10, '', ''])
            table.setCellCSSClass(table.getNbRows()-1, 0, 'iwla_others')
        index.appendBlock(table)