iwla/plugins/pre_analysis/robots.py

# -*- coding: utf-8 -*-
#
# Copyright Grégory Soutadé 2015

# This file is part of iwla

# iwla is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# iwla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with iwla.  If not, see <http://www.gnu.org/licenses/>.
#

import re
import logging
import inspect

from iwla import IWLA
from iplugin import IPlugin

import awstats_data

"""
Pre analysis hook

Filter robots

Plugin requirements :
    None

Conf values needed :
    count_hit_only_visitors
    no_referrer_domains

Output files :
    None

Statistics creation :
    None

Statistics update :
visits :
   remote_addr =>
       robot
       keep_requests

Statistics deletion :
    None
"""

class IWLAPreAnalysisRobots(IPlugin):

    def load(self):
        self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
        self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
        self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
        self.compatible_re = []
        self.compatible_re.append(re.compile(r'.*\(.*compatible; ([^;]+);.*\).*'))
        self.compatible_re.append(re.compile(r'.*\(.*compatible; (.*)\).*'))
        self.compatible_re.append(re.compile(r'.*\(([^;]+); \+.*\).*'))
        self.compatible_re.append(re.compile(r'(.*); \(\+.*\)*'))
        self.logger = logging.getLogger(self.__class__.__name__)
        self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
        self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', [])

        return True

    def _setRobot(self, k, super_hit):
        callerframerecord = inspect.stack()[1]
        frame = callerframerecord[0]
        info = inspect.getframeinfo(frame)

        self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
        super_hit['robot'] = True
        super_hit['keep_requests'] = False

        agent = super_hit['requests'][0]['http_user_agent']
        for compatible_re in self.compatible_re:
            robot_name = compatible_re.match(agent)
            if robot_name:
                super_hit['robot_name'] = robot_name[1]
                break
            
# Basic rule to detect robots
    def hook(self):
        hits = self.iwla.getCurrentVisits()
        for (k, super_hit) in hits.items():
            # Already analyzed
            if super_hit.get('robot', None) in (True, False):
                if super_hit['robot'] == True:
                    self.logger.debug('%s is a robot' % (k))
                continue

            if super_hit.get('feed_parser', False):
                self.logger.debug('%s is feed parser' % (k))
                continue
            
            super_hit['robot'] = False
            isRobot = False
            referers = 0

            first_page = super_hit['requests'][0]
                
            if self.robot_re.match(first_page['http_user_agent']) or\
                    self.crawl_re.match(first_page['http_user_agent']):
                self.logger.debug(first_page['http_user_agent'])
                self._setRobot(k, super_hit)
                continue

            for r in self.awstats_robots:
                if r.match(first_page['http_user_agent']):
                    isRobot = True
                    break

            if isRobot:
                self.logger.debug(first_page['http_user_agent'])
                self._setRobot(k, super_hit)
                continue

# 1) no pages view --> robot
            if not self.one_hit_only and not super_hit['viewed_pages'][0]:
                self._setRobot(k, super_hit)
                continue

# 2) Less than 1 hit per page
            if super_hit['viewed_pages'][0] and (super_hit['viewed_hits'][0] < super_hit['viewed_pages'][0]):
                isRobot = True

            if isRobot:
                self._setRobot(k, super_hit)
                continue

# 3) no pages and not hit --> robot
            if not super_hit['viewed_hits'][0] and not super_hit['viewed_pages'][0]:
                self._setRobot(k, super_hit)
                continue

            error_codes = 0
            not_modified_pages = 0
            for hit in super_hit['requests']:
# 5) /robots.txt read
                if hit['extract_request']['http_uri'].endswith('/robots.txt'):
                    self._setRobot(k, super_hit)
                    break

                # Exception for favicon.png and all apple-*icon*
                if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\
                   'icon' not in hit['extract_request']['http_uri'] and\
                   hit['server_name'] != 'forge.soutade.fr':
                    error_codes += 1
                elif int(hit['status']) in (304,):
                    not_modified_pages += 1

                if not hit['is_page'] and hit['http_referer'] not in ('', '-'):
                    referers += 1

            if isRobot:
                self._setRobot(k, super_hit)
                continue

# 6) Any referer for hits
            if super_hit['viewed_hits'][0] and not referers and\
               not super_hit['requests'][0]['server_name'] in self.no_referrer_domains:
                self._setRobot(k, super_hit)
                continue

# 7) more than 10 4XX or 304 pages
            if error_codes > 10 or not_modified_pages > 50:
                self._setRobot(k, super_hit)
                continue

# 8) Special case : 1 page and 1 hit, but not from the same source
            if (super_hit['viewed_pages'][0] == 1 and super_hit['viewed_hits'][0] == 1 and len(super_hit['requests']) == 2) and\
               (super_hit['requests'][0]['server_name'] != super_hit['requests'][1]['server_name']):
                self._setRobot(k, super_hit)
                continue