Add hasBeenViewed() function

iwla.py

@@ -79,6 +79,9 @@ class IWLA(object):
         cur_time = self.meta_infos['start_analysis_time']
         return (time.mktime(cur_time) <= time.mktime(request['time_decoded']))
 
+    def hasBeenViewed(self, request):
+        return int(request['status']) in conf.viewed_http_codes
+
     def _clearMeta(self):
         self.meta_infos = {
             'last_time' : None

plugins/post_analysis/top_downloads.py

@@ -21,9 +21,11 @@ class IWLAPostAnalysisTopDownloads(IPlugin):
         for (k, super_hit) in stats.items():
             if super_hit['robot']: continue
             for r in super_hit['requests']:
+                if not self.iwla.isValidForCurrentAnalysis(r) or\
+                        not self.iwla.hasBeenViewed(r):
+                    continue
                 if r['is_page']: continue
 
-                if not self.iwla.isValidForCurrentAnalysis(r): continue
 
                 if not int(r['status']) in viewed_http_codes: continue
 

plugins/pre_analysis/page_to_hit.py

@@ -10,7 +10,6 @@ class IWLAPreAnalysisPageToHit(IPlugin):
     def __init__(self, iwla):
         super(IWLAPreAnalysisPageToHit, self).__init__(iwla)
         self.API_VERSION = 1
-        self.conf_requires = ['viewed_http_codes']
 
     def load(self):
 # Remove logo from indefero
@@ -22,14 +21,15 @@ class IWLAPreAnalysisPageToHit(IPlugin):
 
     def hook(self):
         hits = self.iwla.getCurrentVisists()
-        viewed_http_codes = self.iwla.getConfValue('viewed_http_codes')
 
         for (k, super_hit) in hits.items():
             if super_hit['robot']: continue
 
             for request in super_hit['requests']:
+                if not self.iwla.isValidForCurrentAnalysis(request) or\
+                        not self.iwla.hasBeenViewed(request):
+                    continue
                 if not request['is_page']: continue
-                if not self.iwla.isValidForCurrentAnalysis(request): continue
                 uri = request['extract_request']['extract_uri']
                 for regexp in self.regexps:
                     if regexp.match(uri):