From 519033e474800f6d93ae54ccac299268f67a4edd Mon Sep 17 00:00:00 2001 From: Gregory Soutade Date: Mon, 18 Jan 2016 07:33:48 +0100 Subject: [PATCH] Do reverse DNS on feeds parsers --- ChangeLog | 6 ++++-- iwla.py | 14 +++++++++----- plugins/post_analysis/reverse_dns.py | 5 ++++- tools/iwla_convert.pl | 2 ++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8b5a4e5..2af6636 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -v0.3 (20/12/2015) +v0.3 (17/01/2016) ** User ** Add referers_diff display plugin Add year statistics in month details @@ -15,13 +15,15 @@ v0.3 (20/12/2015) Add gz files support Add -z option (don't compress databases) Add own search enfines files + Do reverse DNS on feeds parsers ** Dev ** Add istats_diff interface Sort documentation output Add debug traces in robots plugin Update awstats data - Dont count 'uri' and 'uri/' as differents uri + Remove doule // at the end of URL + Remove final slashes for referrers ** Bugs ** Forgot tag diff --git a/iwla.py b/iwla.py index c72c5e1..9453a74 100755 --- a/iwla.py +++ b/iwla.py @@ -289,6 +289,13 @@ class IWLA(object): self.logger.debug("False") return False + def isValidVisitor(self, hit): + if hit['robot']: return False + if not (conf.count_hit_only_visitors or\ + hit['viewed_pages']): + return False + return True + def _appendHit(self, hit): remote_addr = hit['remote_addr'] @@ -596,11 +603,8 @@ class IWLA(object): self.valid_visitors = {} for (k,v) in visits.items(): - if v['robot']: continue - if not (conf.count_hit_only_visitors or\ - v['viewed_pages']): - continue - self.valid_visitors[k] = v + if self.isValidVisitor(v): + self.valid_visitors[k] = v duplicated_stats['nb_visitors'] = stats['nb_visitors'] = len(self.valid_visitors.keys()) diff --git a/plugins/post_analysis/reverse_dns.py b/plugins/post_analysis/reverse_dns.py index c63965f..b94a8cb 100644 --- a/plugins/post_analysis/reverse_dns.py +++ b/plugins/post_analysis/reverse_dns.py @@ -64,9 +64,12 @@ class IWLAPostAnalysisReverseDNS(IPlugin): return True def hook(self): - hits = self.iwla.getValidVisitors() + hits = self.iwla.getCurrentVisists() for (k, hit) in hits.items(): if hit.get('dns_analysed', False): continue + if not hit['feed_parser'] and\ + not self.iwla.isValidVisitor(hit): + continue try: name, _, _ = socket.gethostbyaddr(k) hit['remote_addr'] = name.lower() diff --git a/tools/iwla_convert.pl b/tools/iwla_convert.pl index 696e230..2df1928 100755 --- a/tools/iwla_convert.pl +++ b/tools/iwla_convert.pl @@ -107,3 +107,5 @@ dumpHash(\%BrowsersHashIcon, $FIC, 1); print $FIC "}\n\n"; close($FIC); + +print "Done !\n";