Basically seems to work
This commit is contained in:
@@ -13,6 +13,8 @@ def hook(hits):
|
||||
|
||||
for p in super_hit['pages']:
|
||||
if not p['is_page']: continue
|
||||
if int(p['status']) != 200: continue
|
||||
|
||||
if logo_re.match(p['extract_request']['extract_uri']):
|
||||
p['is_page'] = False
|
||||
if super_hit['viewed_pages']:
|
||||
|
@@ -19,7 +19,10 @@ def hook(hits):
|
||||
if not super_hit['viewed_hits']:
|
||||
super_hit['robot'] = 1
|
||||
continue
|
||||
|
||||
elif not super_hit['viewed_pages']:
|
||||
# Hit only
|
||||
super_hit['hit_only'] = 1
|
||||
|
||||
for hit in super_hit['pages']:
|
||||
# 3) /robots.txt read
|
||||
if hit['extract_request']['http_uri'] == '/robots.txt':
|
||||
|
Reference in New Issue
Block a user