Robots: Improve compatible keyword detection for robots

Set correct date for merged feed parsers
Remove bad BAD_FEED_PARSER state

default_conf.py

@@ -44,6 +44,7 @@ count_hit_only_visitors = False
 # Multimedia extensions (not accounted as downloaded files)
 multimedia_files = ['png', 'jpg', 'jpeg', 'gif', 'ico', 'svg',
                     'css', 'js']
+multimedia_files_re = []
 
 # Default resources path (will be symlinked in DISPLAY_OUTPUT)
 resources_path = ['resources']

iwla.py

@@ -159,6 +159,9 @@ class IWLA(object):
         self.excluded_domain_name = []
         for domain_name in conf.excluded_domain_name:
             self.excluded_domain_name += [re.compile(domain_name)]
+        self.multimedia_files_re = []
+        for file_re in conf.multimedia_files_re:
+            self.multimedia_files_re += [re.compile(file_re)]
         self.plugins = [(conf.PRE_HOOK_DIRECTORY     , conf.pre_analysis_hooks),
                         (conf.POST_HOOK_DIRECTORY    , conf.post_analysis_hooks),
                         (conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
@@ -311,13 +314,18 @@ class IWLA(object):
         self.logger.debug("False")
         return False
 
-    def isMultimediaFile(self, request):
+    def isMultimediaFile(self, uri):
-        self.logger.debug("Is multimedia %s" % (request))
+        self.logger.debug("Is multimedia %s" % (uri))
         for e in conf.multimedia_files:
-            if request.lower().endswith(e):
+            if uri.lower().endswith(e):
                 self.logger.debug("True")
                 return True
         self.logger.debug("False")
+
+        for file_re in self.multimedia_files_re:
+            if file_re.match(uri):
+                self.logger.debug("Is multimedia re True")
+                return True
         return False
 
     def isValidVisitor(self, hit):
@@ -810,12 +818,15 @@ class IWLA(object):
         for l in _file:
             # print "line " + l
 
-            groups = self.log_re.match(l)
+            sanitized = l.replace('<', '')
+            sanitized = sanitized.replace('>', '')
+            
+            groups = self.log_re.match(sanitized)
 
             if groups:
                 self._newHit(groups.groupdict(""))
             else:
-                self.logger.warning("No match for %s" % (l))
+                self.logger.warning("No match for %s" % (sanitized))
                 #break
 
         if self.analyse_started:

plugins/display/feeds.py

@@ -72,11 +72,13 @@ class IWLADisplayFeeds(IPlugin):
             path = self.iwla.getCurDisplayPath(filename)
 
             page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
-            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Last Access')])
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits')
-            table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', ''])
+                                                                                                   , self.iwla._(u'Domain'), self.iwla._(u'Subscribers'), self.iwla._(u'Last Access')])
+            table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', '', '', ''])
+            rows = []
             for super_hit in hits.values():
-                if not super_hit.get('feed_parser', False): continue
+                if super_hit.get('feed_parser', None) not in (IWLAPostAnalysisFeeds.FEED_PARSER,\
-                if super_hit['feed_parser'] == IWLAPostAnalysisFeeds.BAD_FEED_PARSER:
+                                                              IWLAPostAnalysisFeeds.MERGED_FEED_PARSER):
                     continue
                 nb_feeds_parsers += 1
                 address = super_hit['remote_addr']
@@ -84,11 +86,21 @@ class IWLADisplayFeeds(IPlugin):
                     address += ' *'
                 pages = super_hit['not_viewed_pages'][0] + super_hit['viewed_pages'][0]
                 hits = super_hit['not_viewed_hits'][0] + super_hit['viewed_hits'][0]
-                last_access = super_hit.get('feed_parser_last_access', None)
+                last_access = super_hit.get('feed_parser_last_access', super_hit['last_access'])
-                if not last_access:
+                feed_domain = super_hit.get('feed_domain', '')
-                    last_access = super_hit['last_access']
+                if feed_domain:
-                row = [address, pages, hits, time.asctime(last_access)]
+                    link = '<a href=\'https://%s/%s\'>%s</a>' % (feed_domain, super_hit.get('feed_uri', ''), feed_domain)
-                table.appendRow(row, super_hit['remote_ip'])
+                else:
+                    link = ''
+                subscribers = super_hit.get('feed_subscribers', '')
+                # Don't overload interface
+                if subscribers <= 1: subscribers = ''
+                row = [address, pages, hits, link, subscribers, time.asctime(last_access),
+                       super_hit['remote_ip'], last_access]
+                rows.append(row)
+            rows = sorted(rows, key=lambda t: t[7], reverse=True)
+            for row in rows:
+                table.appendRow(row[:6], row[6])
             page.appendBlock(table)
             note = DisplayHTMLRaw(self.iwla, ('<small>*%s</small>' % (self.iwla._(u'Merged feeds parsers'))))
             page.appendBlock(note)

plugins/post_analysis/feeds.py

@@ -19,6 +19,7 @@
 #
 
 import re
+import time
 
 from iwla import IWLA
 from iplugin import IPlugin
@@ -47,8 +48,11 @@ Output files :
 Statistics creation :
    remote_ip =>
        feed_parser
-       feed_name_analysed
+       feed_name_analyzed
        feed_parser_last_access (for merged parser)
+       feed_domain
+       feed_uri
+       feed_subscribers
 
 Statistics update :
     None
@@ -91,15 +95,21 @@ class IWLAPostAnalysisFeeds(IPlugin):
         for f in feeds_agents:
             self.user_agents_re.append(re.compile(f))
 
+        self.bad_user_agents_re = []
+        self.bad_user_agents_re.append(re.compile(r'.*feedback.*'))
+
+        self.subscribers_re = re.compile(r'.* ([0-9]+) subscriber.*')
+        
         self.merge_feeds_parsers_list = []
         for f in _merge_feeds_parsers_list:
             self.merge_feeds_parsers_list.append(re.compile(f))
-            
+
         self.merged_feeds = {}
 
         return True
 
     def _appendToMergeCache(self, isFeedParser, key, hit):
+        hit['feed_parser'] = isFeedParser
         # First time, register into dict
         if self.merged_feeds.get(key, None) is None:
             # Merged
@@ -108,21 +118,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
             # Next time
             # Current must be ignored
             hit['feed_parser'] = self.NOT_A_FEED_PARSER
+            merged_hit = hit
             last_access = hit['last_access']
             # Previous matched hit must be set as merged
-            isFeedParser = self.MERGED_FEED_PARSER
             hit = self.merged_feeds[key]
-            if hit['last_access'] < last_access:
+            hit['feed_parser'] = self.MERGED_FEED_PARSER
-                hit['feed_parser_last_access'] = last_access
+            hit['viewed_pages'][0]     += merged_hit['viewed_pages'][0]
+            hit['viewed_hits'][0]      += merged_hit['viewed_hits'][0]
+            hit['not_viewed_pages'][0] += merged_hit['not_viewed_pages'][0]
+            hit['not_viewed_hits'][0]  += merged_hit['not_viewed_hits'][0]
+            if hit['last_access'] < merged_hit['last_access']:
+                hit['feed_parser_last_access'] = merged_hit['last_access']
             else:
                 hit['feed_parser_last_access'] = hit['last_access']
-        hit['feed_parser'] = isFeedParser
     
     def mergeFeedsParsers(self, isFeedParser, hit):
-        if isFeedParser:
+        if isFeedParser in (self.FEED_PARSER, self.MERGED_FEED_PARSER):
             for r in self.merge_feeds_parsers_list:
                 if r.match(hit['remote_addr']) or r.match(hit['remote_ip']):
-                    self._appendToMergeCache(isFeedParser, r, hit)
+                    # One group can view multiple different feeds
+                    key = r.pattern + hit.get('feed_domain', '') + hit.get('feed_uri', '')
+                    self._appendToMergeCache(isFeedParser, key, hit)
                     return
             #print("No match for %s : %d" % (hit['remote_addr'], hit['viewed_hits'][0] + hit['not_viewed_hits'][0]))
             # Other cases, look for user agent
@@ -134,22 +150,27 @@ class IWLAPostAnalysisFeeds(IPlugin):
         for hit in hits.values():
             isFeedParser = hit.get('feed_parser', None)
 
-            # Register already tagged feed parser in merged_feeds
+            if isFeedParser == self.NOT_A_FEED_PARSER:
-            if self.merge_feeds_parsers and\
-               not isFeedParser in (None, self.BAD_FEED_PARSER):
-                self.mergeFeedsParsers(isFeedParser, hit)
                 continue
-
+            
+            # Second time
             if isFeedParser:
-                if hit['feed_parser'] == self.BAD_FEED_PARSER: continue
+                # Update last access time
-                if not hit.get('feed_name_analysed', False) and\
+                if hit['last_access'] > hit.get('feed_parser_last_access', time.gmtime(0)):
+                    hit['feed_parser_last_access'] = hit['last_access']
+                    
+                if not hit.get('feed_name_analyzed', False) and\
                    hit.get('dns_name_replaced', False):
-                    hit['feed_name_analysed'] = True
+                    hit['feed_name_analyzed'] = True
                     addr = hit.get('remote_addr', None)
                     for r in self.bad_feeds_re:
                         if r.match(addr):
-                            hit['feed_parser'] = self.BAD_FEED_PARSER
+                            hit['feed_parser'] = self.NOT_A_FEED_PARSER
                             break
+                        
+                # Register already tagged feed parser in merged_feeds
+                if self.merge_feeds_parsers:
+                    self.mergeFeedsParsers(isFeedParser, hit)
                 continue
 
             request = hit['requests'][0]
@@ -164,14 +185,38 @@ class IWLAPostAnalysisFeeds(IPlugin):
                             isFeedParser = self.NOT_A_FEED_PARSER
                     break
 
+            user_agent = request['http_user_agent'].lower()
+                    
             if isFeedParser == self.NOT_A_FEED_PARSER:
-                user_agent = request['http_user_agent'].lower()
                 for regexp in self.user_agents_re:
                     if regexp.match(user_agent):
                         isFeedParser = self.FEED_PARSER
                         break
-                
+                  
+            if isFeedParser == self.FEED_PARSER:
+                for regexp in self.bad_user_agents_re:
+                    if regexp.match(user_agent):
+                        isFeedParser = self.NOT_A_FEED_PARSER
+                        break
+
+            if not hit.get('feed_name_analyzed', False) and\
+               hit.get('dns_name_replaced', False):
+                hit['feed_name_analyzed'] = True
+                addr = hit.get('remote_addr', None)
+                for r in self.bad_feeds_re:
+                    if r.match(addr):
+                        isFeedParser = hit['feed_parser'] = self.NOT_A_FEED_PARSER
+                        break
+
+            if isFeedParser == self.FEED_PARSER:
+                hit['feed_domain'] = request['server_name']
+                hit['feed_uri'] = uri
+                hit['feed_subscribers'] = 0
+
+                subscribers = self.subscribers_re.match(user_agent)
+                if subscribers:
+                    hit['feed_subscribers'] = int(subscribers.groups()[0])
+            
+            hit['feed_parser'] = isFeedParser
             if self.merge_feeds_parsers:
                 self.mergeFeedsParsers(isFeedParser, hit)
-            else:
-                hit['feed_parser'] = isFeedParser

plugins/pre_analysis/robots.py

@@ -61,7 +61,11 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
         self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
         self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
-        self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*')
+        self.compatible_re = []
+        self.compatible_re.append(re.compile(r'.*\(.*compatible; ([^;]+);.*\).*'))
+        self.compatible_re.append(re.compile(r'.*\(.*compatible; (.*)\).*'))
+        self.compatible_re.append(re.compile(r'.*\(([^;]+); \+.*\).*'))
+        self.compatible_re.append(re.compile(r'(.*); \(\+.*\)*'))
         self.logger = logging.getLogger(self.__class__.__name__)
         self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
         self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', [])
@@ -76,12 +80,14 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
         super_hit['robot'] = True
         super_hit['keep_requests'] = False
-        for hit in super_hit['requests']:
+
-            robot_name = self.compatible_re.match(hit['http_user_agent'])
+        agent = super_hit['requests'][0]['http_user_agent']
+        for compatible_re in self.compatible_re:
+            robot_name = compatible_re.match(agent)
             if robot_name:
                 super_hit['robot_name'] = robot_name[1]
                 break
-
+            
 # Basic rule to detect robots
     def hook(self):
         hits = self.iwla.getCurrentVisits()
@@ -101,10 +107,9 @@ class IWLAPreAnalysisRobots(IPlugin):
             referers = 0
 
             first_page = super_hit['requests'][0]
-
+                
             if self.robot_re.match(first_page['http_user_agent']) or\
-                    self.crawl_re.match(first_page['http_user_agent']) or\
+                    self.crawl_re.match(first_page['http_user_agent']):
-                    self.compatible_re.match(first_page['http_user_agent']):
                 self.logger.debug(first_page['http_user_agent'])
                 self._setRobot(k, super_hit)
                 continue
@@ -147,7 +152,8 @@ class IWLAPreAnalysisRobots(IPlugin):
 
                 # Exception for favicon.png and all apple-*icon*
                 if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\
-                   'icon' not in hit['extract_request']['http_uri']:
+                   'icon' not in hit['extract_request']['http_uri'] and\
+                   hit['server_name'] != 'forge.soutade.fr':
                     error_codes += 1
                 elif int(hit['status']) in (304,):
                     not_modified_pages += 1