default_conf.py

@@ -44,7 +44,6 @@ count_hit_only_visitors = False
 # Multimedia extensions (not accounted as downloaded files)
 multimedia_files = ['png', 'jpg', 'jpeg', 'gif', 'ico', 'svg',
                     'css', 'js']
-multimedia_files_re = []
 
 # Default resources path (will be symlinked in DISPLAY_OUTPUT)
 resources_path = ['resources']

iwla.py

@@ -159,9 +159,6 @@ class IWLA(object):
         self.excluded_domain_name = []
         for domain_name in conf.excluded_domain_name:
             self.excluded_domain_name += [re.compile(domain_name)]
-        self.multimedia_files_re = []
-        for file_re in conf.multimedia_files_re:
-            self.multimedia_files_re += [re.compile(file_re)]
         self.plugins = [(conf.PRE_HOOK_DIRECTORY     , conf.pre_analysis_hooks),
                         (conf.POST_HOOK_DIRECTORY    , conf.post_analysis_hooks),
                         (conf.DISPLAY_HOOK_DIRECTORY , conf.display_hooks)]
@@ -314,18 +311,13 @@ class IWLA(object):
         self.logger.debug("False")
         return False
 
-    def isMultimediaFile(self, uri):
+    def isMultimediaFile(self, request):
-        self.logger.debug("Is multimedia %s" % (uri))
+        self.logger.debug("Is multimedia %s" % (request))
         for e in conf.multimedia_files:
-            if uri.lower().endswith(e):
+            if request.lower().endswith(e):
                 self.logger.debug("True")
                 return True
         self.logger.debug("False")
-
-        for file_re in self.multimedia_files_re:
-            if file_re.match(uri):
-                self.logger.debug("Is multimedia re True")
-                return True
         return False
 
     def isValidVisitor(self, hit):
@@ -818,15 +810,12 @@ class IWLA(object):
         for l in _file:
             # print "line " + l
 
-            sanitized = l.replace('<', '')
+            groups = self.log_re.match(l)
-            sanitized = sanitized.replace('>', '')
-            
-            groups = self.log_re.match(sanitized)
 
             if groups:
                 self._newHit(groups.groupdict(""))
             else:
-                self.logger.warning("No match for %s" % (sanitized))
+                self.logger.warning("No match for %s" % (l))
                 #break
 
         if self.analyse_started:

plugins/display/feeds.py

@@ -72,13 +72,11 @@ class IWLADisplayFeeds(IPlugin):
             path = self.iwla.getCurDisplayPath(filename)
 
             page = display.createPage(title, path, self.iwla.getConfValue('css_path', []))
-            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits')
+            table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'All feeds parsers'), [self.iwla._(u'Host'), self.iwla._(u'Pages'), self.iwla._(u'Hits'), self.iwla._(u'Last Access')])
-                                                                                                   , self.iwla._(u'Domain'), self.iwla._(u'Subscribers'), self.iwla._(u'Last Access')])
+            table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', ''])
-            table.setColsCSSClass(['', 'iwla_page', 'iwla_hit', '', '', ''])
-            rows = []
             for super_hit in hits.values():
-                if super_hit.get('feed_parser', None) not in (IWLAPostAnalysisFeeds.FEED_PARSER,\
+                if not super_hit.get('feed_parser', False): continue
-                                                              IWLAPostAnalysisFeeds.MERGED_FEED_PARSER):
+                if super_hit['feed_parser'] == IWLAPostAnalysisFeeds.BAD_FEED_PARSER:
                     continue
                 nb_feeds_parsers += 1
                 address = super_hit['remote_addr']
@@ -86,21 +84,11 @@ class IWLADisplayFeeds(IPlugin):
                     address += ' *'
                 pages = super_hit['not_viewed_pages'][0] + super_hit['viewed_pages'][0]
                 hits = super_hit['not_viewed_hits'][0] + super_hit['viewed_hits'][0]
-                last_access = super_hit.get('feed_parser_last_access', super_hit['last_access'])
+                last_access = super_hit.get('feed_parser_last_access', None)
-                feed_domain = super_hit.get('feed_domain', '')
+                if not last_access:
-                if feed_domain:
+                    last_access = super_hit['last_access']
-                    link = '<a href=\'https://%s/%s\'>%s</a>' % (feed_domain, super_hit.get('feed_uri', ''), feed_domain)
+                row = [address, pages, hits, time.asctime(last_access)]
-                else:
+                table.appendRow(row, super_hit['remote_ip'])
-                    link = ''
-                subscribers = super_hit.get('feed_subscribers', '')
-                # Don't overload interface
-                if subscribers <= 1: subscribers = ''
-                row = [address, pages, hits, link, subscribers, time.asctime(last_access),
-                       super_hit['remote_ip'], last_access]
-                rows.append(row)
-            rows = sorted(rows, key=lambda t: t[7], reverse=True)
-            for row in rows:
-                table.appendRow(row[:6], row[6])
             page.appendBlock(table)
             note = DisplayHTMLRaw(self.iwla, ('<small>*%s</small>' % (self.iwla._(u'Merged feeds parsers'))))
             page.appendBlock(note)

plugins/post_analysis/feeds.py

@@ -19,7 +19,6 @@
 #
 
 import re
-import time
 
 from iwla import IWLA
 from iplugin import IPlugin
@@ -48,11 +47,8 @@ Output files :
 Statistics creation :
    remote_ip =>
        feed_parser
-       feed_name_analyzed
+       feed_name_analysed
        feed_parser_last_access (for merged parser)
-       feed_domain
-       feed_uri
-       feed_subscribers
 
 Statistics update :
     None
@@ -95,21 +91,15 @@ class IWLAPostAnalysisFeeds(IPlugin):
         for f in feeds_agents:
             self.user_agents_re.append(re.compile(f))
 
-        self.bad_user_agents_re = []
-        self.bad_user_agents_re.append(re.compile(r'.*feedback.*'))
-
-        self.subscribers_re = re.compile(r'.* ([0-9]+) subscriber.*')
-        
         self.merge_feeds_parsers_list = []
         for f in _merge_feeds_parsers_list:
             self.merge_feeds_parsers_list.append(re.compile(f))
-
+            
         self.merged_feeds = {}
 
         return True
 
     def _appendToMergeCache(self, isFeedParser, key, hit):
-        hit['feed_parser'] = isFeedParser
         # First time, register into dict
         if self.merged_feeds.get(key, None) is None:
             # Merged
@@ -118,27 +108,21 @@ class IWLAPostAnalysisFeeds(IPlugin):
             # Next time
             # Current must be ignored
             hit['feed_parser'] = self.NOT_A_FEED_PARSER
-            merged_hit = hit
             last_access = hit['last_access']
             # Previous matched hit must be set as merged
+            isFeedParser = self.MERGED_FEED_PARSER
             hit = self.merged_feeds[key]
-            hit['feed_parser'] = self.MERGED_FEED_PARSER
+            if hit['last_access'] < last_access:
-            hit['viewed_pages'][0]     += merged_hit['viewed_pages'][0]
+                hit['feed_parser_last_access'] = last_access
-            hit['viewed_hits'][0]      += merged_hit['viewed_hits'][0]
-            hit['not_viewed_pages'][0] += merged_hit['not_viewed_pages'][0]
-            hit['not_viewed_hits'][0]  += merged_hit['not_viewed_hits'][0]
-            if hit['last_access'] < merged_hit['last_access']:
-                hit['feed_parser_last_access'] = merged_hit['last_access']
             else:
                 hit['feed_parser_last_access'] = hit['last_access']
+        hit['feed_parser'] = isFeedParser
     
     def mergeFeedsParsers(self, isFeedParser, hit):
-        if isFeedParser in (self.FEED_PARSER, self.MERGED_FEED_PARSER):
+        if isFeedParser:
             for r in self.merge_feeds_parsers_list:
                 if r.match(hit['remote_addr']) or r.match(hit['remote_ip']):
-                    # One group can view multiple different feeds
+                    self._appendToMergeCache(isFeedParser, r, hit)
-                    key = r.pattern + hit.get('feed_domain', '') + hit.get('feed_uri', '')
-                    self._appendToMergeCache(isFeedParser, key, hit)
                     return
             #print("No match for %s : %d" % (hit['remote_addr'], hit['viewed_hits'][0] + hit['not_viewed_hits'][0]))
             # Other cases, look for user agent
@@ -150,27 +134,22 @@ class IWLAPostAnalysisFeeds(IPlugin):
         for hit in hits.values():
             isFeedParser = hit.get('feed_parser', None)
 
-            if isFeedParser == self.NOT_A_FEED_PARSER:
+            # Register already tagged feed parser in merged_feeds
+            if self.merge_feeds_parsers and\
+               not isFeedParser in (None, self.BAD_FEED_PARSER):
+                self.mergeFeedsParsers(isFeedParser, hit)
                 continue
-            
+
-            # Second time
             if isFeedParser:
-                # Update last access time
+                if hit['feed_parser'] == self.BAD_FEED_PARSER: continue
-                if hit['last_access'] > hit.get('feed_parser_last_access', time.gmtime(0)):
+                if not hit.get('feed_name_analysed', False) and\
-                    hit['feed_parser_last_access'] = hit['last_access']
-                    
-                if not hit.get('feed_name_analyzed', False) and\
                    hit.get('dns_name_replaced', False):
-                    hit['feed_name_analyzed'] = True
+                    hit['feed_name_analysed'] = True
                     addr = hit.get('remote_addr', None)
                     for r in self.bad_feeds_re:
                         if r.match(addr):
-                            hit['feed_parser'] = self.NOT_A_FEED_PARSER
+                            hit['feed_parser'] = self.BAD_FEED_PARSER
                             break
-                        
-                # Register already tagged feed parser in merged_feeds
-                if self.merge_feeds_parsers:
-                    self.mergeFeedsParsers(isFeedParser, hit)
                 continue
 
             request = hit['requests'][0]
@@ -185,38 +164,14 @@ class IWLAPostAnalysisFeeds(IPlugin):
                             isFeedParser = self.NOT_A_FEED_PARSER
                     break
 
-            user_agent = request['http_user_agent'].lower()
-                    
             if isFeedParser == self.NOT_A_FEED_PARSER:
+                user_agent = request['http_user_agent'].lower()
                 for regexp in self.user_agents_re:
                     if regexp.match(user_agent):
                         isFeedParser = self.FEED_PARSER
                         break
-                  
+                
-            if isFeedParser == self.FEED_PARSER:
-                for regexp in self.bad_user_agents_re:
-                    if regexp.match(user_agent):
-                        isFeedParser = self.NOT_A_FEED_PARSER
-                        break
-
-            if not hit.get('feed_name_analyzed', False) and\
-               hit.get('dns_name_replaced', False):
-                hit['feed_name_analyzed'] = True
-                addr = hit.get('remote_addr', None)
-                for r in self.bad_feeds_re:
-                    if r.match(addr):
-                        isFeedParser = hit['feed_parser'] = self.NOT_A_FEED_PARSER
-                        break
-
-            if isFeedParser == self.FEED_PARSER:
-                hit['feed_domain'] = request['server_name']
-                hit['feed_uri'] = uri
-                hit['feed_subscribers'] = 0
-
-                subscribers = self.subscribers_re.match(user_agent)
-                if subscribers:
-                    hit['feed_subscribers'] = int(subscribers.groups()[0])
-            
-            hit['feed_parser'] = isFeedParser
             if self.merge_feeds_parsers:
                 self.mergeFeedsParsers(isFeedParser, hit)
+            else:
+                hit['feed_parser'] = isFeedParser

plugins/pre_analysis/robots.py

@@ -61,11 +61,7 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.awstats_robots = list(map(lambda x : re.compile(('.*%s.*') % (x), re.IGNORECASE), awstats_data.robots))
         self.robot_re = re.compile(r'.*bot.*', re.IGNORECASE)
         self.crawl_re = re.compile(r'.*crawl.*', re.IGNORECASE)
-        self.compatible_re = []
+        self.compatible_re = re.compile(r'.*\(.*compatible; (.*); \+.*\)*')
-        self.compatible_re.append(re.compile(r'.*\(.*compatible; ([^;]+);.*\).*'))
-        self.compatible_re.append(re.compile(r'.*\(.*compatible; (.*)\).*'))
-        self.compatible_re.append(re.compile(r'.*\(([^;]+); \+.*\).*'))
-        self.compatible_re.append(re.compile(r'(.*); \(\+.*\)*'))
         self.logger = logging.getLogger(self.__class__.__name__)
         self.one_hit_only = self.iwla.getConfValue('count_hit_only_visitors', False)
         self.no_referrer_domains = self.iwla.getConfValue('no_referrer_domains', [])
@@ -80,14 +76,12 @@ class IWLAPreAnalysisRobots(IPlugin):
         self.logger.debug('%s is a robot (caller %s:%d)' % (k, info.function, info.lineno))
         super_hit['robot'] = True
         super_hit['keep_requests'] = False
-
+        for hit in super_hit['requests']:
-        agent = super_hit['requests'][0]['http_user_agent']
+            robot_name = self.compatible_re.match(hit['http_user_agent'])
-        for compatible_re in self.compatible_re:
-            robot_name = compatible_re.match(agent)
             if robot_name:
                 super_hit['robot_name'] = robot_name[1]
                 break
-            
+
 # Basic rule to detect robots
     def hook(self):
         hits = self.iwla.getCurrentVisits()
@@ -107,9 +101,10 @@ class IWLAPreAnalysisRobots(IPlugin):
             referers = 0
 
             first_page = super_hit['requests'][0]
-                
+
             if self.robot_re.match(first_page['http_user_agent']) or\
-                    self.crawl_re.match(first_page['http_user_agent']):
+                    self.crawl_re.match(first_page['http_user_agent']) or\
+                    self.compatible_re.match(first_page['http_user_agent']):
                 self.logger.debug(first_page['http_user_agent'])
                 self._setRobot(k, super_hit)
                 continue
@@ -152,8 +147,7 @@ class IWLAPreAnalysisRobots(IPlugin):
 
                 # Exception for favicon.png and all apple-*icon*
                 if int(hit['status']) >= 400 and int(hit['status']) <= 499 and\
-                   'icon' not in hit['extract_request']['http_uri'] and\
+                   'icon' not in hit['extract_request']['http_uri']:
-                   hit['server_name'] != 'forge.soutade.fr':
                     error_codes += 1
                 elif int(hit['status']) in (304,):
                     not_modified_pages += 1