Add debug traces in robots plugin
This commit is contained in:
parent
157868dc3e
commit
62be78845a
|
@ -19,6 +19,7 @@
|
||||||
#
|
#
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
import logging
|
||||||
|
|
||||||
from iwla import IWLA
|
from iwla import IWLA
|
||||||
from iplugin import IPlugin
|
from iplugin import IPlugin
|
||||||
|
@ -61,13 +62,20 @@ class IWLAPreAnalysisRobots(IPlugin):
|
||||||
self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
|
self.awstats_robots = map(lambda (x) : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots)
|
||||||
self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
|
self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
|
||||||
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
|
self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
|
||||||
|
self.logger = logging.getLogger(self.__class__.__name__)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _setRobot(self, k, super_hit):
|
||||||
|
self.logger.debug('%s is a robot' % (k))
|
||||||
|
super_hit['robot'] = 1
|
||||||
|
|
||||||
# Basic rule to detect robots
|
# Basic rule to detect robots
|
||||||
def hook(self):
|
def hook(self):
|
||||||
hits = self.iwla.getCurrentVisists()
|
hits = self.iwla.getCurrentVisists()
|
||||||
for (k, super_hit) in hits.items():
|
for (k, super_hit) in hits.items():
|
||||||
if super_hit['robot']: continue
|
if super_hit['robot']:
|
||||||
|
self.logger.debug('%s is a robot' % (k))
|
||||||
|
continue
|
||||||
|
|
||||||
isRobot = False
|
isRobot = False
|
||||||
referers = 0
|
referers = 0
|
||||||
|
@ -76,7 +84,7 @@ class IWLAPreAnalysisRobots(IPlugin):
|
||||||
|
|
||||||
if self.robot_re.match(first_page['http_user_agent']) or\
|
if self.robot_re.match(first_page['http_user_agent']) or\
|
||||||
self.crawl_re.match(first_page['http_user_agent']):
|
self.crawl_re.match(first_page['http_user_agent']):
|
||||||
super_hit['robot'] = 1
|
self._setRobot(k, super_hit)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for r in self.awstats_robots:
|
for r in self.awstats_robots:
|
||||||
|
@ -85,7 +93,7 @@ class IWLAPreAnalysisRobots(IPlugin):
|
||||||
break
|
break
|
||||||
|
|
||||||
if isRobot:
|
if isRobot:
|
||||||
super_hit['robot'] = 1
|
self._setRobot(k, super_hit)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# 1) no pages view --> robot
|
# 1) no pages view --> robot
|
||||||
|
@ -95,13 +103,13 @@ class IWLAPreAnalysisRobots(IPlugin):
|
||||||
|
|
||||||
# 2) pages without hit --> robot
|
# 2) pages without hit --> robot
|
||||||
if not super_hit['viewed_hits']:
|
if not super_hit['viewed_hits']:
|
||||||
super_hit['robot'] = 1
|
self._setRobot(k, super_hit)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for hit in super_hit['requests']:
|
for hit in super_hit['requests']:
|
||||||
# 3) /robots.txt read
|
# 3) /robots.txt read
|
||||||
if hit['extract_request']['http_uri'].endswith('/robots.txt'):
|
if hit['extract_request']['http_uri'].endswith('/robots.txt'):
|
||||||
isRobot = True
|
self._setRobot(k, super_hit)
|
||||||
break
|
break
|
||||||
|
|
||||||
# 4) Any referer for hits
|
# 4) Any referer for hits
|
||||||
|
@ -109,10 +117,10 @@ class IWLAPreAnalysisRobots(IPlugin):
|
||||||
referers += 1
|
referers += 1
|
||||||
|
|
||||||
if isRobot:
|
if isRobot:
|
||||||
super_hit['robot'] = 1
|
self._setRobot(k, super_hit)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not super_hit['viewed_pages'] and \
|
if not super_hit['viewed_pages'] and \
|
||||||
(super_hit['viewed_hits'] and not referers):
|
(super_hit['viewed_hits'] and not referers):
|
||||||
super_hit['robot'] = 1
|
self._setRobot(k, super_hit)
|
||||||
continue
|
continue
|
||||||
|
|
Loading…
Reference in New Issue
Block a user