import re # Remove logo from indefero logo_re = re.compile(r'^.+/logo/$') # Basic rule to detect robots def hook(hits): for k in hits.keys(): super_hit = hits[k] if super_hit['robot']: continue for p in super_hit['pages']: if not p['is_page']: continue if int(p['status']) != 200: continue if logo_re.match(p['extract_request']['extract_uri']): p['is_page'] = False if super_hit['viewed_pages']: super_hit['viewed_pages'] -= 1 super_hit['viewed_hits'] += 1