diff --git a/awstats_data.py b/awstats_data.py index 8789384..1c4a0d4 100644 --- a/awstats_data.py +++ b/awstats_data.py @@ -1,3 +1,5 @@ +#This file was automatically generated by iwla_convert.pl. Do not edit manually. + robots = ['appie', 'architext', 'jeeves', 'bjaaland', 'contentmatch', 'ferret', 'googlebot', 'google\-sitemaps', 'gulliver', 'virus[_+ ]detector', 'harvest', 'htdig', 'linkwalker', 'lilina', 'lycos[_+ ]', 'moget', 'muscatferret', 'myweb', 'nomad', 'scooter', 'slurp', '^voyager\/', 'weblayers', 'antibot', 'bruinbot', 'digout4u', 'echo!', 'fast\-webcrawler', 'ia_archiver\-web\.archive\.org', 'ia_archiver', 'jennybot', 'mercator', 'netcraft', 'msnbot\-media', 'msnbot', 'petersnews', 'relevantnoise\.com', 'unlost_web_crawler', 'voila', 'webbase', 'webcollage', 'cfetch', 'zyborg', 'wisenutbot', '[^a]fish', 'abcdatos', 'acme\.spider', 'ahoythehomepagefinder', 'alkaline', 'anthill', 'arachnophilia', 'arale', 'araneo', 'aretha', 'ariadne', 'powermarks', 'arks', 'aspider', 'atn\.txt', 'atomz', 'auresys', 'backrub', 'bbot', 'bigbrother', 'blackwidow', 'blindekuh', 'bloodhound', 'borg\-bot', 'brightnet', 'bspider', 'cactvschemistryspider', 'calif[^r]', 'cassandra', 'cgireader', 'checkbot', 'christcrawler', 'churl', 'cienciaficcion', 'collective', 'combine', 'conceptbot', 'coolbot', 'core', 'cosmos', 'cruiser', 'cusco', 'cyberspyder', 'desertrealm', 'deweb', 'dienstspider', 'digger', 'diibot', 'direct_hit', 'dnabot', 'download_express', 'dragonbot', 'dwcp', 'e\-collector', 'ebiness', 'elfinbot', 'emacs', 'emcspider', 'esther', 'evliyacelebi', 'fastcrawler', 'feedcrawl', 'fdse', 'felix', 'fetchrover', 'fido', 'finnish', 'fireball', 'fouineur', 'francoroute', 'freecrawl', 'funnelweb', 'gama', 'gazz', 'gcreep', 'getbot', 'geturl', 'golem', 'gougou', 'grapnel', 'griffon', 'gromit', 'gulperbot', 'hambot', 'havindex', 'hometown', 'htmlgobble', 'hyperdecontextualizer', 'iajabot', 'iaskspider', 'hl_ftien_spider', 'sogou', 'iconoclast', 'ilse', 'imagelock', 'incywincy', 'informant', 'infoseek', 'infoseeksidewinder', 'infospider', 'inspectorwww', 'intelliagent', 'irobot', 'iron33', 'israelisearch', 'javabee', 'jbot', 'jcrawler', 'jobo', 'jobot', 'joebot', 'jubii', 'jumpstation', 'kapsi', 'katipo', 'kilroy', 'ko[_+ ]yappo[_+ ]robot', 'kummhttp', 'labelgrabber\.txt', 'larbin', 'legs', 'linkidator', 'linkscan', 'lockon', 'logo_gif', 'macworm', 'magpie', 'marvin', 'mattie', 'mediafox', 'merzscope', 'meshexplorer', 'mindcrawler', 'mnogosearch', 'momspider', 'monster', 'motor', 'muncher', 'mwdsearch', 'ndspider', 'nederland\.zoek', 'netcarta', 'netmechanic', 'netscoop', 'newscan\-online', 'nhse', 'northstar', 'nzexplorer', 'objectssearch', 'occam', 'octopus', 'openfind', 'orb_search', 'packrat', 'pageboy', 'parasite', 'patric', 'pegasus', 'perignator', 'perlcrawler', 'phantom', 'phpdig', 'piltdownman', 'pimptrain', 'pioneer', 'pitkow', 'pjspider', 'plumtreewebaccessor', 'poppi', 'portalb', 'psbot', 'python', 'raven', 'rbse', 'resumerobot', 'rhcs', 'road_runner', 'robbie', 'robi', 'robocrawl', 'robofox', 'robozilla', 'roverbot', 'rules', 'safetynetrobot', 'search\-info', 'search_au', 'searchprocess', 'senrigan', 'sgscout', 'shaggy', 'shaihulud', 'sift', 'simbot', 'site\-valet', 'sitetech', 'skymob', 'slcrawler', 'smartspider', 'snooper', 'solbot', 'speedy', 'spider[_+ ]monkey', 'spiderbot', 'spiderline', 'spiderman', 'spiderview', 'spry', 'sqworm', 'ssearcher', 'suke', 'sunrise', 'suntek', 'sven', 'tach_bw', 'tagyu_agent', 'tailrank', 'tarantula', 'tarspider', 'techbot', 'templeton', 'titan', 'titin', 'tkwww', 'tlspider', 'ucsd', 'udmsearch', 'universalfeedparser', 'urlck', 'valkyrie', 'verticrawl', 'victoria', 'visionsearch', 'voidbot', 'vwbot', 'w3index', 'w3m2', 'wallpaper', 'wanderer', 'wapspIRLider', 'webbandit', 'webcatcher', 'webcopy', 'webfetcher', 'webfoot', 'webinator', 'weblinker', 'webmirror', 'webmoose', 'webquest', 'webreader', 'webreaper', 'websnarf', 'webspider', 'webvac', 'webwalk', 'webwalker', 'webwatch', 'whatuseek', 'whowhere', 'wired\-digital', 'wmir', 'wolp', 'wombat', 'wordpress', 'worm', 'woozweb', 'wwwc', 'wz101', 'xget', '1\-more_scanner', 'accoona\-ai\-agent', 'activebookmark', 'adamm_bot', 'almaden', 'aipbot', 'aleadsoftbot', 'alpha_search_agent', 'allrati', 'aport', 'archive\.org_bot', 'argus', 'arianna\.libero\.it', 'aspseek', 'asterias', 'awbot', 'baiduspider', 'becomebot', 'bender', 'betabot', 'biglotron', 'bittorrent_bot', 'biz360[_+ ]spider', 'blogbridge[_+ ]service', 'bloglines', 'blogpulse', 'blogsearch', 'blogshares', 'blogslive', 'blogssay', 'bncf\.firenze\.sbn\.it\/raccolta\.txt', 'bobby', 'boitho\.com\-dc', 'bookmark\-manager', 'boris', 'bumblebee', 'candlelight[_+ ]favorites[_+ ]inspector', 'cbn00glebot', 'cerberian_drtrs', 'cfnetwork', 'cipinetbot', 'checkweb_link_validator', 'commons\-httpclient', 'computer_and_automation_research_institute_crawler', 'converamultimediacrawler', 'converacrawler', 'cscrawler', 'cse_html_validator_lite_online', 'cuasarbot', 'cursor', 'custo', 'datafountains\/dmoz_downloader', 'daviesbot', 'daypopbot', 'deepindex', 'dipsie\.bot', 'dnsgroup', 'domainchecker', 'domainsdb\.net', 'dulance', 'dumbot', 'dumm\.de\-bot', 'earthcom\.info', 'easydl', 'edgeio\-retriever', 'ets_v', 'exactseek', 'extreme[_+ ]picture[_+ ]finder', 'eventax', 'everbeecrawler', 'everest\-vulcan', 'ezresult', 'enteprise', 'facebook', 'fast_enterprise_crawler.*crawleradmin\.t\-info@telekom\.de', 'fast_enterprise_crawler.*t\-info_bi_cluster_crawleradmin\.t\-info@telekom\.de', 'matrix_s\.p\.a\._\-_fast_enterprise_crawler', 'fast_enterprise_crawler', 'fast\-search\-engine', 'favicon', 'favorg', 'favorites_sweeper', 'feedburner', 'feedfetcher\-google', 'feedflow', 'feedster', 'feedsky', 'feedvalidator', 'filmkamerabot', 'findlinks', 'findexa_crawler', 'fooky\.com\/ScorpionBot', 'g2crawler', 'gaisbot', 'geniebot', 'gigabot', 'girafabot', 'global_fetch', 'gnodspider', 'goforit\.com', 'goforitbot', 'gonzo', 'grub', 'gpu_p2p_crawler', 'henrythemiragorobot', 'heritrix', 'holmes', 'hoowwwer', 'hpprint', 'htmlparser', 'html[_+ ]link[_+ ]validator', 'httrack', 'hundesuche\.com\-bot', 'ichiro', 'iltrovatore\-setaccio', 'infobot', 'infociousbot', 'infomine', 'insurancobot', 'internet[_+ ]ninja', 'internetarchive', 'internetseer', 'internetsupervision', 'irlbot', 'isearch2006', 'iupui_research_bot', 'jrtwine[_+ ]software[_+ ]check[_+ ]favorites[_+ ]utility', 'justview', 'kalambot', 'kamano\.de_newsfeedverzeichnis', 'kazoombot', 'kevin', 'keyoshid', 'kinjabot', 'kinja\-imagebot', 'knowitall', 'knowledge\.com', 'kouaa_krawler', 'krugle', 'ksibot', 'kurzor', 'lanshanbot', 'letscrawl\.com', 'libcrawl', 'linkbot', 'link_valet_online', 'metager\-linkchecker', 'linkchecker', 'livejournal\.com', 'lmspider', 'lwp\-request', 'lwp\-trivial', 'magpierss', 'mail\.ru', 'mapoftheinternet\.com', 'mediapartners\-google', 'megite', 'metaspinner', 'microsoft[_+ ]url[_+ ]control', 'mini\-reptile', 'minirank', 'missigua_locator', 'misterbot', 'miva', 'mizzu_labs', 'mj12bot', 'mojeekbot', 'msiecrawler', 'ms_search_4\.0_robot', 'msrabot', 'msrbot', 'mt::telegraph::agent', 'nagios', 'nasa_search', 'mydoyouhike', 'netluchs', 'netsprint', 'newsgatoronline', 'nicebot', 'nimblecrawler', 'noxtrumbot', 'npbot', 'nutchcvs', 'nutchosu\-vlib', 'nutch', 'ocelli', 'octora_beta_bot', 'omniexplorer[_+ ]bot', 'onet\.pl[_+ ]sa', 'onfolio', 'opentaggerbot', 'openwebspider', 'oracle_ultra_search', 'orbiter', 'yodaobot', 'qihoobot', 'passwordmaker\.org', 'pear_http_request_class', 'peerbot', 'perman', 'php[_+ ]version[_+ ]tracker', 'pictureofinternet', 'ping\.blo\.gs', 'plinki', 'pluckfeedcrawler', 'pogodak', 'pompos', 'popdexter', 'port_huron_labs', 'postfavorites', 'projectwf\-java\-test\-crawler', 'proodlebot', 'pyquery', 'rambler', 'redalert', 'rojo', 'rssimagesbot', 'ruffle', 'rufusbot', 'sandcrawler', 'sbider', 'schizozilla', 'scumbot', 'searchguild[_+ ]dmoz[_+ ]experiment', 'seekbot', 'sensis_web_crawler', 'seznambot', 'shim\-crawler', 'shoutcast', 'slysearch', 'snap\.com_beta_crawler', 'sohu\-search', 'sohu', 'snappy', 'sphere_scout', 'spip', 'sproose_crawler', 'steeler', 'steroid__download', 'suchfin\-bot', 'superbot', 'surveybot', 'susie', 'syndic8', 'syndicapi', 'synoobot', 'tcl_http_client_package', 'technoratibot', 'teragramcrawlersurf', 'test_crawler', 'testbot', 't\-h\-u\-n\-d\-e\-r\-s\-t\-o\-n\-e', 'topicblogs', 'turnitinbot', 'turtlescanner', 'turtle', 'tutorgigbot', 'twiceler', 'ubicrawler', 'ultraseek', 'unchaos_bot_hybrid_web_search_engine', 'unido\-bot', 'updated', 'ustc\-semantic\-group', 'vagabondo\-wap', 'vagabondo', 'vermut', 'versus_crawler_from_eda\.baykan@epfl\.ch', 'vespa_crawler', 'vortex', 'vse\/', 'w3c\-checklink', 'w3c[_+ ]css[_+ ]validator[_+ ]jfouffa', 'w3c_validator', 'watchmouse', 'wavefire', 'webclipping\.com', 'webcompass', 'webcrawl\.net', 'web_downloader', 'webdup', 'webfilter', 'webindexer', 'webminer', 'website[_+ ]monitoring[_+ ]bot', 'webvulncrawl', 'wells_search', 'wonderer', 'wume_crawler', 'wwweasel', 'xenu\'s_link_sleuth', 'xenu_link_sleuth', 'xirq', 'y!j', 'yacy', 'yahoo\-blogs', 'yahoo\-verticalcrawler', 'yahoofeedseeker', 'yahooseeker\-testing', 'yahooseeker', 'yahoo\-mmcrawler', 'yahoo!_mindset', 'yandex', 'flexum', 'yanga', 'yooglifetchagent', 'z\-add_link_checker', 'zealbot', 'zhuaxia', 'zspider', 'zeus', 'ng\/1\.', 'ng\/2\.', 'exabot', 'wget', 'libwww', 'java\/[0-9]'] search_engines = ['google\.[\w.]+/products', 'base\.google\.', 'froogle\.google\.', 'groups\.google\.', 'images\.google\.', 'google\.', 'googlee\.', 'googlecom\.com', 'goggle\.co\.hu', '216\.239\.(35|37|39|51)\.100', '216\.239\.(35|37|39|51)\.101', '216\.239\.5[0-9]\.104', '64\.233\.1[0-9]{2}\.104', '66\.102\.[1-9]\.104', '66\.249\.93\.104', '72\.14\.2[0-9]{2}\.104', 'msn\.', 'live\.com', 'bing\.', 'voila\.', 'mindset\.research\.yahoo', 'yahoo\.', '(66\.218\.71\.225|216\.109\.117\.135|216\.109\.125\.130|66\.218\.69\.11)', 'search\.aol\.co', 'tiscali\.', 'lycos\.', 'alexa\.com', 'alltheweb\.com', 'altavista\.', 'a9\.com', 'dmoz\.org', 'netscape\.', 'search\.terra\.', 'www\.search\.com', 'search\.sli\.sympatico\.ca', 'excite\.'] @@ -10,3 +12,15 @@ search_engines_hashid = {'search\.sli\.sympatico\.ca' : 'sympatico', 'mywebsearc search_engines_knwown_url = {'dmoz' : 'search=', 'google' : '(p|q|as_p|as_q)=', 'searchalot' : 'q=', 'teoma' : 'q=', 'looksmartuk' : 'key=', 'polymeta_hu' : '', 'google_groups' : 'group\/', 'iune' : '(keywords|q)=', 'chellosk' : 'q1=', 'eniro' : 'q=', 'msn' : 'q=', 'webcrawler' : 'searchText=', 'mirago' : '(txtsearch|qry)=', 'enirose' : 'q=', 'miragobe' : '(txtsearch|qry)=', 'netease' : 'q=', 'netluchs' : 'query=', 'google_products' : '(p|q|as_p|as_q)=', 'jyxo' : '(s|q)=', 'origo' : '(q|search)=', 'ilse' : 'search_for=', 'chellocom' : 'q1=', 'goodsearch' : 'Keywords=', 'ledix' : 'q=', 'mozbot' : 'q=', 'chellocz' : 'q1=', 'webde' : 'su=', 'biglotron' : 'question=', 'metacrawler_de' : 'qry=', 'finddk' : 'words=', 'start' : 'q=', 'sagool' : 'q=', 'miragoch' : '(txtsearch|qry)=', 'google_base' : '(p|q|as_p|as_q)=', 'aliceit' : 'qs=', 'shinyseek\.it' : 'KEY=', 'onetpl' : 'qt=', 'clusty' : 'query=', 'chellonl' : 'q1=', 'miragode' : '(txtsearch|qry)=', 'miragose' : '(txtsearch|qry)=', 'o2pl' : 'qt=', 'goliat' : 'KERESES=', 'kvasir' : 'q=', 'askfr' : '(ask|q)=', 'infoseek' : 'qt=', 'yahoo_mindset' : 'p=', 'comettoolbar' : 'qry=', 'alltheweb' : 'q(|uery)=', 'miner' : 'q=', 'aol' : 'query=', 'rambler' : 'words=', 'scroogle' : 'Gw=', 'chellose' : 'q1=', 'ineffabile' : '', 'miragoit' : '(txtsearch|qry)=', 'yandex' : 'text=', 'segnalo' : '', 'dodajpl' : 'keyword=', 'avantfind' : 'keywords=', 'nusearch' : 'nusearch_terms=', 'bbc' : 'q=', 'supereva' : 'q=', 'atomz' : 'sp-q=', 'searchy' : 'search_term=', 'dogpile' : 'q(|kw)=', 'chellohu' : 'q1=', 'vnet' : 'kw=', '1klik' : 'query=', 't-online' : 'q=', 'hogapl' : 'qt=', 'stumbleupon' : '', 'soso' : 'q=', 'zhongsou' : '(word|w)=', 'a9' : 'a9\.com\/', 'centraldatabase' : 'query=', 'mamma' : 'query=', 'icerocket' : 'q=', 'ask' : '(ask|q)=', 'chellobe' : 'q1=', 'altavista' : 'q=', 'vindex' : 'in=', 'miragodk' : '(txtsearch|qry)=', 'chelloat' : 'q1=', 'digg' : 's=', 'metacrawler' : 'general=', 'nbci' : 'keyword=', 'chellono' : 'q1=', 'icq' : 'q=', 'arianna' : 'query=', 'miragocouk' : '(txtsearch|qry)=', '3721' : '(p|name)=', 'pogodak' : 'q=', 'ukdirectory' : 'k=', 'overture' : 'keywords=', 'heureka' : 'heureka=', 'teecnoit' : 'q=', 'miragoes' : '(txtsearch|qry)=', 'haku' : 'w=', 'go' : 'qt=', 'fireball' : 'q=', 'wisenut' : 'query=', 'sify' : 'keyword=', 'ixquick' : 'query=', 'anzwers' : 'search=', 'quick' : 'query=', 'jubii' : 'soegeord=', 'questionanswering' : '', 'asknl' : '(ask|q)=', 'askde' : '(ask|q)=', 'att' : 'qry=', 'terra' : 'query=', 'bing' : 'q=', 'wowpl' : 'q=', 'freeserve' : 'q=', 'atlas' : '(searchtext|q)=', 'askuk' : '(ask|q)=', 'godado' : 'Keywords=', 'northernlight' : 'qr=', 'answerbus' : '', 'search.com' : 'q=', 'google_image' : '(p|q|as_p|as_q)=', 'jumpy\.it' : 'searchWord=', 'gazetapl' : 'slowo=', 'yahoo' : 'p=', 'hotbot' : 'mt=', 'metabot' : 'st=', 'copernic' : 'web\/', 'kartoo' : '', 'metaspinner' : 'qry=', 'toile' : 'q=', 'aolde' : 'q=', 'blingo' : 'q=', 'askit' : '(ask|q)=', 'netscape' : 'search=', 'splut' : 'pattern=', 'looksmart' : 'key=', 'sphere' : 'q=', 'sol' : 'q=', 'miragono' : '(txtsearch|qry)=', 'kataweb' : 'q=', 'ofir' : 'querytext=', 'aliceitmaster' : 'qs=', 'miragofr' : '(txtsearch|qry)=', 'spray' : 'string=', 'seznam' : '(w|q)=', 'interiapl' : 'q=', 'euroseek' : 'query=', 'schoenerbrausen' : 'q=', 'centrum' : 'q=', 'netsprintpl' : 'q=', 'go2net' : 'general=', 'katalogonetpl' : 'qt=', 'ukindex' : 'stext=', 'shawca' : 'q=', 'szukaczpl' : 'q=', 'accoona' : 'qt=', 'live' : 'q=', 'google4counter' : '(p|q|as_p|as_q)=', 'iask' : '(w|k)=', 'earthlink' : 'q=', 'tiscali' : 'key=', 'askes' : '(ask|q)=', 'gotuneed' : '', 'clubinternet' : 'q=', 'redbox' : 'srch=', 'delicious' : 'all=', 'chellofr' : 'q1=', 'lycos' : 'query=', 'sympatico' : 'query=', 'vivisimo' : 'query=', 'bluewin' : 'qry=', 'mysearch' : 'searchfor=', 'google_cache' : '(p|q|as_p|as_q)=cache:[0-9A-Za-z]{12}:', 'ukplus' : 'search=', 'gerypl' : 'q=', 'keresolap_hu' : 'q=', 'abacho' : 'q=', 'engine' : 'p1=', 'opasia' : 'q=', 'wp' : 'szukaj=', 'steadysearch' : 'w=', 'chellopl' : 'q1=', 'voila' : '(kw|rdata)=', 'aport' : 'r=', 'internetto' : 'searchstr=', 'passagen' : 'q=', 'wwweasel' : 'q=', 'najdi' : 'dotaz=', 'alexa' : 'q=', 'baidu' : '(wd|word)=', 'spotjockey' : 'Search_Keyword=', 'virgilio' : 'qs=', 'orbis' : 'search_field=', 'tango_hu' : 'q=', 'askjp' : '(ask|q)=', 'bungeebonesdotcom' : 'query=', 'francite' : 'name=', 'searchch' : 'q=', 'google_froogle' : '(p|q|as_p|as_q)=', 'excite' : 'search=', 'infospace' : 'qkw=', 'polskapl' : 'qt=', 'swik' : 'swik\.net/', 'edderkoppen' : 'query=', 'mywebsearch' : 'searchfor=', 'danielsen' : 'q=', 'wahoo' : 'q=', 'sogou' : 'query=', 'miragonl' : '(txtsearch|qry)=', 'findarticles' : 'key='} +operating_systems = ['windows[_+ ]?2005', 'windows[_+ ]nt[_+ ]6\.0', 'windows[_+ ]?2008', 'windows[_+ ]nt[_+ ]6\.1', 'windows[_+ ]?vista', 'windows[_+ ]nt[_+ ]6', 'windows[_+ ]?2003', 'windows[_+ ]nt[_+ ]5\.2', 'windows[_+ ]xp', 'windows[_+ ]nt[_+ ]5\.1', 'windows[_+ ]me', 'win[_+ ]9x', 'windows[_+ ]?2000', 'windows[_+ ]nt[_+ ]5', 'winnt', 'windows[_+ \-]?nt', 'win32', 'win(.*)98', 'win(.*)95', 'win(.*)16', 'windows[_+ ]3', 'win(.*)ce', 'mac[_+ ]os[_+ ]x', 'mac[_+ ]?p', 'mac[_+ ]68', 'macweb', 'macintosh', 'linux(.*)android', 'linux(.*)asplinux', 'linux(.*)centos', 'linux(.*)debian', 'linux(.*)fedora', 'linux(.*)gentoo', 'linux(.*)mandr', 'linux(.*)momonga', 'linux(.*)pclinuxos', 'linux(.*)red[_+ ]hat', 'linux(.*)suse', 'linux(.*)ubuntu', 'linux(.*)vector', 'linux(.*)vine', 'linux(.*)white\sbox', 'linux(.*)zenwalk', 'linux', 'gnu.hurd', 'bsdi', 'gnu.kfreebsd', 'freebsd', 'openbsd', 'netbsd', 'dragonfly', 'aix', 'sunos', 'irix', 'osf', 'hp\-ux', 'unix', 'x11', 'gnome\-vfs', 'beos', 'os/2', 'amiga', 'atari', 'vms', 'commodore', 'qnx', 'inferno', 'palmos', 'syllable', 'blackberry', 'cp/m', 'crayos', 'dreamcast', 'iphone[_+ ]os', 'risc[_+ ]?os', 'symbian', 'webtv', 'playstation', 'xbox', 'wii', 'vienna', 'newsfire', 'applesyndication', 'akregator', 'plagger', 'syndirella', 'j2me', 'java', 'microsoft', 'msie[_+ ]', 'ms[_+ ]frontpage', 'windows'] + +operating_systems_hashid = {'crayos' : 'crayos', 'linux(.*)white\sbox' : 'linuxwhitebox', 'windows[_+ \-]?nt' : 'winnt', 'windows[_+ ]?2003' : 'win2003', 'mac[_+ ]?p' : 'macintosh', 'netbsd' : 'bsdnetbsd', 'win[_+ ]9x' : 'winme', 'vms' : 'vms', 'gnome\-vfs' : 'unix', 'windows[_+ ]nt[_+ ]5' : 'win2000', 'windows[_+ ]nt[_+ ]6\.0' : 'winlong', 'dragonflybsd' : 'bsddflybsd', 'wii' : 'wii', 'linux(.*)vector' : 'linuxvector', 'microsoft' : 'winunknown', 'plagger' : 'unix', 'amiga' : 'amigaos', 'windows[_+ ]me' : 'winme', 'irix' : 'irix', 'linux(.*)android' : 'linuxandroid', 'linux(.*)suse' : 'linuxsuse', 'java' : 'java', 'win(.*)ce' : 'wince', 'cp/m' : 'cp/m', 'windows[_+ ]3' : 'win16', 'win(.*)98' : 'win98', 'windows' : 'winunknown', 'os/2' : 'os/2', 'syndirella' : 'winxp', 'osf' : 'osf', 'macweb' : 'macintosh', 'linux(.*)centos' : 'linuxcentos', 'gnu.hurd' : 'gnu', 'dreamcast' : 'dreamcast', 'linux' : 'linux', 'win(.*)16' : 'win16', 'freebsd' : 'bsdfreebsd', 'windows[_+ ]xp' : 'winxp', 'blackberry' : 'blackberry', 'macintosh' : 'macintosh', 'symbian' : 'symbian', 'linux(.*)debian' : 'linuxdebian', 'windows[_+ ]?2005' : 'winlong', 'linux(.*)red[_+ ]hat' : 'linuxredhat', 'x11' : 'unix', 'windows[_+ ]nt[_+ ]5\.2' : 'win2003', 'j2me' : 'j2me', 'sunos' : 'sunos', 'linux(.*)vine' : 'linuxvine', 'mac[_+ ]os[_+ ]x' : 'macosx', 'unix' : 'unix', 'windows[_+ ]?2000' : 'win2000', 'inferno' : 'inferno', 'aix' : 'aix', 'akregator' : 'linux', 'atari' : 'atari', 'linux(.*)asplinux' : 'linuxasplinux', 'linux(.*)ubuntu' : 'linuxubuntu', 'win(.*)95' : 'win95', 'xbox' : 'winxbox', 'applesyndication' : 'macosx', 'risc[_+ ]?os' : 'riscos', 'playstation' : 'psp', 'winnt' : 'winnt', 'windows[_+ ]nt[_+ ]5\.1' : 'winxp', 'palmos' : 'palmos', 'windows[_+ ]nt[_+ ]6\.1' : 'win7', 'syllable' : 'syllable', 'commodore' : 'commodore', 'vienna' : 'macosx', 'linux(.*)gentoo' : 'linuxgentoo', 'hp\-ux' : 'hp\-ux', 'linux(.*)fedora' : 'linuxfedora', 'linux(.*)pclinuxos' : 'linuxpclinuxos', 'openbsd' : 'bsdopenbsd', 'windows[_+ ]nt[_+ ]6' : 'winvista', 'mac[_+ ]68' : 'macintosh', 'windows[_+ ]?vista' : 'winvista', 'newsfire' : 'macosx', 'windows[_+ ]?2008' : 'win2008', 'linux(.*)mandr' : 'linuxmandr', 'gnu.kfreebsd' : 'bsdkfreebsd', 'bsdi' : 'bsdi', 'win32' : 'winnt', 'webtv' : 'webtv', 'linux(.*)momonga' : 'linuxmomonga', 'msie[_+ ]' : 'winunknown', 'qnx' : 'qnx', 'iphone[_+ ]os' : 'ios', 'linux(.*)zenwalk' : 'linuxzenwalk', 'beos' : 'beos', 'ms[_+ ]frontpage' : 'winunknown'} + +operating_systems_family = {'mac' : 'Macintosh', 'linux' : 'Linux', 'bsd' : 'BSD', 'win' : 'Windows'} + +browsers = ['elinks', 'firebird', 'go!zilla', 'icab', 'links', 'lynx', 'omniweb', '22acidownload', 'abrowse', 'aol\-iweng', 'amaya', 'amigavoyager', 'arora', 'aweb', 'charon', 'donzilla', 'seamonkey', 'flock', 'minefield', 'bonecho', 'granparadiso', 'songbird', 'strata', 'sylera', 'kazehakase', 'prism', 'icecat', 'iceape', 'iceweasel', 'w3clinemode', 'bpftp', 'camino', 'chimera', 'cyberdog', 'dillo', 'xchaos_arachne', 'doris', 'dreamcast', 'xbox', 'downloadagent', 'ecatch', 'emailsiphon', 'encompass', 'epiphany', 'friendlyspider', 'fresco', 'galeon', 'flashget', 'freshdownload', 'getright', 'leechget', 'netants', 'headdump', 'hotjava', 'ibrowse', 'intergo', 'k\-meleon', 'k\-ninja', 'linemodebrowser', 'lotus\-notes', 'macweb', 'multizilla', 'ncsa_mosaic', 'netcaptor', 'netpositive', 'nutscrape', 'msfrontpageexpress', 'contiki', 'emacs\-w3', 'phoenix', 'shiira', 'tzgeturl', 'viking', 'webfetcher', 'webexplorer', 'webmirror', 'webvcr', 'qnx\svoyager', 'teleport', 'webcapture', 'webcopier', 'real', 'winamp', 'windows\-media\-player', 'audion', 'freeamp', 'itunes', 'jetaudio', 'mint_audio', 'mpg123', 'mplayer', 'nsplayer', 'qts', 'quicktime', 'sonique', 'uplayer', 'xaudio', 'xine', 'xmms', 'gstreamer', 'abilon', 'aggrevator', 'aiderss', 'akregator', 'applesyndication', 'betanews_reader', 'blogbridge', 'cyndicate', 'feeddemon', 'feedreader', 'feedtools', 'greatnews', 'gregarius', 'hatena_rss', 'jetbrains_omea', 'liferea', 'netnewswire', 'newsfire', 'newsgator', 'newzcrawler', 'plagger', 'pluck', 'potu', 'pubsub\-rss\-reader', 'pulpfiction', 'rssbandit', 'rssreader', 'rssowl', 'rss\sxpress', 'rssxpress', 'sage', 'sharpreader', 'shrook', 'straw', 'syndirella', 'vienna', 'wizz\srss\snews\sreader', 'alcatel', 'lg\-', 'mot\-', 'nokia', 'panasonic', 'philips', 'sagem', 'samsung', 'sie\-', 'sec\-', 'sonyericsson', 'ericsson', 'mmef', 'mspie', 'vodafone', 'wapalizer', 'wapsilon', 'wap', 'webcollage', 'up\.', 'android', 'blackberry', 'cnf2', 'docomo', 'ipcheck', 'iphone', 'portalmmm', 'webtv', 'democracy', 'cjb\.net', 'ossproxy', 'smallproxy', 'adobeair', 'apt', 'analogx_proxy', 'gnome\-vfs', 'neon', 'curl', 'csscheck', 'httrack', 'fdm', 'javaws', 'wget', 'fget', 'chilkat', 'webdownloader\sfor\sx', 'w3m', 'wdg_validator', 'w3c_validator', 'jigsaw', 'webreaper', 'webzip', 'staroffice', 'gnus', 'nikto', 'download\smaster', 'microsoft\-webdav\-miniredir', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav', 'POE\-Component\-Client\-HTTP', 'mozilla', 'libwww', 'lwp', 'WebSec'] + +browsers_hashid = {'jetbrains_omea' : 'Omea (RSS Reader)', 'aol\-iweng' : 'AOL-Iweng', 'webcapture' : 'Acrobat Webcapture', 'winamp' : 'WinAmp (media player)', 'chrome' : 'Google Chrome', 'analogx_proxy' : 'AnalogX Proxy', 'sylera' : 'Sylera', 'rss\sxpress' : 'RSS Xpress (RSS Reader)', 'xchaos_arachne' : 'Arachne', 'mspie' : 'MS Pocket Internet Explorer (PDA/Phone browser)', 'lynx' : 'Lynx', 'alcatel' : 'Alcatel Browser (PDA/Phone browser)', 'emailsiphon' : 'EmailSiphon', 'POE\-Component\-Client\-HTTP' : 'HTTP user-agent for POE (portable networking framework for Perl)', 'javaws' : 'Java Web Start', 'ecatch' : 'eCatch', 'aggrevator' : 'Aggrevator (RSS Reader)', 'mmef' : 'Microsoft Mobile Explorer (PDA/Phone browser)', 'qts' : 'QuickTime (media player)', 'linemodebrowser' : 'W3C Line Mode Browser', 'wizz\srss\snews\sreader' : 'Wizz RSS News Reader (RSS Reader)', 'xine' : 'Xine, a free multimedia player (media player)', 'gnome\-vfs' : 'Gnome FileSystem Abstraction library', 'flock' : 'Flock', 'audion' : 'Audion (media player)', 'icecat' : 'GNU IceCat', 'webfetcher' : 'WebFetcher', 'flashget' : 'FlashGet', 'docomo' : 'I-Mode phone (PDA/Phone browser)', 'friendlyspider' : 'FriendlySpider', 'wapalizer' : 'WAPalizer (PDA/Phone browser)', 'ipcheck' : 'Supervision IP Check (phone)', 'wapsilon' : 'WAPsilon (PDA/Phone browser)', 'svn' : 'Subversion client', 'lwp' : 'LibWWW-perl', 'plagger' : 'Plagger (RSS Reader)', 'shrook' : 'Shrook (RSS Reader)', 'mplayer' : 'The Movie Player (media player)', 'nsplayer' : 'NetShow Player (media player)', 'mint_audio' : 'Mint Audio (media player)', 'fget' : 'FGet', 'panasonic' : 'Panasonic Browser (PDA/Phone browser)', 'rssreader' : 'RssReader (RSS Reader)', 'download\smaster' : 'Download Master', 'itunes' : 'Apple iTunes (media player)', 'arora' : 'Arora', 'contiki' : 'Contiki', 'mot\-' : 'Motorola Browser (PDA/Phone browser)', 'nutscrape' : 'Nutscrape', 'fdm' : 'FDM Free Download Manager', 'prism' : 'Prism', 'safari' : 'Safari', 'encompass' : 'Encompass', 'feedreader' : 'FeedReader (RSS Reader)', 'newsgator' : 'NewsGator (RSS Reader)', 'microsoft\-webdav\-miniredir' : 'Microsoft Data Access Component Internet Publishing Provider', 'android' : 'Android browser (PDA/Phone browser)', 'strata' : 'Strata', 'teleport' : 'TelePort Pro', 'songbird' : 'Songbird', 'syndirella' : 'Syndirella (RSS Reader)', 'epiphany' : 'Epiphany', 'minefield' : 'Minefield (Firefox 3.0 development)', 'ncsa_mosaic' : 'NCSA Mosaic', 'links' : 'Links', 'macweb' : 'MacWeb', 'iphone' : 'IPhone (PDA/Phone browser)', 'cjb\.net' : 'CJB.NET Proxy', 'bpftp' : 'BPFTP', 'netcaptor' : 'NetCaptor', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav' : 'Microsoft Data Access Component Internet Publishing Provider DAV', 'dreamcast' : 'Dreamcast', 'straw' : 'Straw (RSS Reader)', 'windows\-media\-player' : 'Windows Media Player (media player)', 'philips' : 'Philips Browser (PDA/Phone browser)', 'netpositive' : 'NetPositive', 'doris' : 'Doris (for Symbian)', 'gstreamer' : 'GStreamer (media library)', 'intergo' : 'InterGO', 'shiira' : 'Shiira', 'gregarius' : 'Gregarius (RSS Reader)', 'potu' : 'Potu (RSS Reader)', 'blackberry' : 'BlackBerry (PDA/Phone browser)', 'smallproxy' : 'SmallProxy', 'galeon' : 'Galeon', 'iceweasel' : 'Iceweasel', 'leechget' : 'LeechGet', 'opera' : 'Opera', 'pubsub\-rss\-reader' : 'PubSub (RSS Reader)', 'vodafone' : 'Vodaphone browser (PDA/Phone browser)', 'rssbandit' : 'RSS Bandit (RSS Reader)', 'samsung' : 'Samsung (PDA/Phone browser)', 'charon' : 'Charon', 'democracy' : 'Democracy', 'freshdownload' : 'FreshDownload', 'freeamp' : 'FreeAmp (media player)', 'nokia' : 'Nokia Browser (PDA/Phone browser)', 'elinks' : 'ELinks', 'multizilla' : 'MultiZilla', 'ericsson' : 'Ericsson Browser (PDA/Phone browser)', 'nikto' : 'Nikto Web Scanner', 'mpg123' : 'mpg123 (media player)', 'gnus' : 'Gnus Network User Services', 'firefox' : 'Firefox', 'msie' : 'MS Internet Explorer', 'betanews_reader' : 'Betanews Reader (RSS Reader)', 'akregator' : 'Akregator (RSS Reader)', 'hatena_rss' : 'Hatena (RSS Reader)', 'iceape' : 'GNU IceApe', 'viking' : 'Viking', 'k\-ninja' : 'K-Ninja', 'ibrowse' : 'iBrowse', 'sonyericsson' : 'Sony/Ericsson Browser (PDA/Phone browser)', 'portalmmm' : 'I-Mode phone (PDA/Phone browser)', 'apt' : 'Debian APT', 'curl' : 'Curl', 'xbox' : 'XBoX', 'aweb' : 'AWeb', 'WebSec' : 'Web Secretary', 'applesyndication' : 'AppleSyndication (RSS Reader)', 'qnx\svoyager' : 'QNX Voyager', 'netnewswire' : 'NetNewsWire (RSS Reader)', 'cnf2' : 'Supervision I-Mode ByTel (phone)', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager' : 'Microsoft Data Access Component Internet Publishing Provider Cache Manager', 'go!zilla' : 'Go!Zilla', 'cyndicate' : 'Cyndicate (RSS Reader)', 'wget' : 'Wget', 'jetaudio' : 'JetAudio (media player)', 'sharpreader' : 'SharpReader (RSS Reader)', 'w3c_validator' : 'W3C Validator', 'netscape' : 'Netscape', 'webcollage' : 'WebCollage (PDA/Phone browser)', 'feeddemon' : 'FeedDemon (RSS Reader)', 'wap' : 'Unknown WAP browser (PDA/Phone browser)', 'aiderss' : 'AideRSS (RSS Reader)', 'lg\-' : 'LG (PDA/Phone browser)', 'webzip' : 'WebZIP', 'pulpfiction' : 'PulpFiction (RSS Reader)', 'webreaper' : 'WebReaper', 'k\-meleon' : 'K-Meleon', 'pluck' : 'Pluck (RSS Reader)', 'msfrontpageexpress' : 'MS FrontPage Express', 'fresco' : 'ANT Fresco', 'httrack' : 'HTTrack', 'real' : 'Real player or compatible (media player)', 'quicktime' : 'QuickTime (media player)', 'konqueror' : 'Konqueror', 'jigsaw' : 'W3C Validator', 'sie\-' : 'SIE (PDA/Phone browser)', 'mozilla' : 'Mozilla', '22acidownload' : '22AciDownload', 'netants' : 'NetAnts', 'csscheck' : 'WDG CSS Validator', 'newzcrawler' : 'NewzCrawler (RSS Reader)', 'sagem' : 'Sagem (PDA/Phone browser)', 'xmms' : 'XMMS (media player)', 'rssxpress' : 'RSSXpress (RSS Reader)', 'wdg_validator' : 'WDG HTML Validator', 'amigavoyager' : 'AmigaVoyager', 'vienna' : 'Vienna (RSS Reader)', 'feedtools' : 'FeedTools (RSS Reader)', 'camino' : 'Camino', 'blogbridge' : 'BlogBridge (RSS Reader)', 'bonecho' : 'BonEcho (Firefox 2.0 development)', 'granparadiso' : 'GranParadiso (Firefox 3.0 development)', 'hotjava' : 'Sun HotJava', 'up\.' : 'UP.Browser (PDA/Phone browser)', 'w3m' : 'w3m', 'dillo' : 'Dillo', 'liferea' : 'Liferea (RSS Reader)', 'getright' : 'GetRight', 'kazehakase' : 'Kazehakase', 'lotus\-notes' : 'Lotus Notes web client', 'tzgeturl' : 'TzGetURL', 'sage' : 'Sage (RSS Reader)', 'webcopier' : 'WebCopier', 'phoenix' : 'Phoenix', 'abrowse' : 'ABrowse', 'xaudio' : 'Some XAudio Engine based MPEG player (media player)', 'sec\-' : 'Sony/Ericsson (PDA/Phone browser)', 'w3clinemode' : 'W3CLineMode', 'chimera' : 'Chimera (Old Camino)', 'headdump' : 'HeadDump', 'abilon' : 'Abilon (RSS Reader)', 'downloadagent' : 'DownloadAgent', 'cyberdog' : 'Cyberdog', 'rssowl' : 'RSSOwl (RSS Reader)', 'newsfire' : 'NewsFire (RSS Reader)', 'webdownloader\sfor\sx' : 'Downloader for X', 'sonique' : 'Sonique (media player)', 'webmirror' : 'WebMirror', 'webexplorer' : 'IBM-WebExplorer', 'chilkat' : 'Chilkat', 'ossproxy' : 'OSSProxy', 'libwww' : 'LibWWW', 'adobeair' : 'AdobeAir', 'uplayer' : 'Ultra Player (media player)', 'amaya' : 'Amaya', 'webtv' : 'WebTV browser', 'neon' : 'Neon HTTP and WebDAV client library', 'greatnews' : 'GreatNews (RSS Reader)', 'seamonkey' : 'SeaMonkey', 'omniweb' : 'OmniWeb', 'donzilla' : 'Donzilla', 'webvcr' : 'WebVCR', 'icab' : 'iCab', 'firebird' : 'Firebird (Old Firefox)', 'staroffice' : 'StarOffice', 'emacs\-w3' : 'Emacs/w3s'} + +browsers_icons = {'jetbrains_omea' : 'rss', 'webcapture' : 'adobe', 'winamp' : 'mediaplayer', 'chrome' : 'chrome', 'analogx_proxy' : 'analogx', 'sylera' : 'mozilla', 'rss\sxpress' : 'rss', 'mspie' : 'pdaphone', 'lynx' : 'lynx', 'alcatel' : 'pdaphone', 'javaws' : 'java', 'ecatch' : 'ecatch', 'aggrevator' : 'rss', 'mmef' : 'pdaphone', 'qts' : 'mediaplayer', 'wizz\srss\snews\sreader' : 'wizz', 'xine' : 'mediaplayer', 'gnome\-vfs' : 'gnome', 'flock' : 'flock', 'audion' : 'mediaplayer', 'icecat' : 'icecat', 'flashget' : 'flashget', 'docomo' : 'pdaphone', 'avantbrowser' : 'avant', 'wapalizer' : 'pdaphone', 'wapsilon' : 'pdaphone', 'svn' : 'subversion', 'plagger' : 'rss', 'shrook' : 'rss', 'mplayer' : 'mediaplayer', 'nsplayer' : 'netshow', 'mint_audio' : 'mediaplayer', 'panasonic' : 'pdaphone', 'rssreader' : 'rss', 'itunes' : 'mediaplayer', 'microsoft\soffice\sprotocol\sdiscovery' : 'frontpage', 'mot\-' : 'pdaphone', 'prism' : 'mozilla', 'safari' : 'safari', 'encompass' : 'encompass', 'feedreader' : 'rss', 'newsgator' : 'rss', 'microsoft\-webdav\-miniredir' : 'frontpage', 'android' : 'android', 'teleport' : 'teleport', 'strata' : 'mozilla', 'songbird' : 'mozilla', 'syndirella' : 'rss', 'epiphany' : 'epiphany', 'minefield' : 'firefox', 'ncsa_mosaic' : 'ncsa_mosaic', 'macweb' : 'macweb', 'iphone' : 'pdaphone', 'cjb\.net' : 'cjbnet', 'bpftp' : 'bpftp', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sdav' : 'frontpage', 'dreamcast' : 'dreamcast', 'straw' : 'rss', 'windows\-media\-player' : 'mplayer', 'philips' : 'pdaphone', 'netpositive' : 'netpositive', 'doris' : 'doris', 'gregarius' : 'rss', 'potu' : 'rss', 'blackberry' : 'pdaphone', 'galeon' : 'galeon', 'iceweasel' : 'iceweasel', 'leechget' : 'leechget', 'opera' : 'opera', 'pubsub\-rss\-reader' : 'rss', 'vodafone' : 'pdaphone', 'rssbandit' : 'rss', 'samsung' : 'pdaphone', 'freshdownload' : 'freshdownload', 'freeamp' : 'mediaplayer', 'nokia' : 'pdaphone', 'multizilla' : 'multizilla', 'ericsson' : 'pdaphone', 'mpg123' : 'mediaplayer', 'gnus' : 'gnus', 'firefox' : 'firefox', 'msie' : 'msie', 'betanews_reader' : 'rss', 'akregator' : 'rss', 'hatena_rss' : 'rss', 'iceape' : 'mozilla', 'ibrowse' : 'ibrowse', 'sonyericsson' : 'pdaphone', 'portalmmm' : 'pdaphone', 'apt' : 'apt', 'xbox' : 'winxbox', 'aweb' : 'aweb', 'applesyndication' : 'rss', 'netnewswire' : 'rss', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\scache\smanager' : 'frontpage', 'go!zilla' : 'gozilla', 'microsoft\sdata\saccess\sinternet\spublishing\sprovider\sprotocol\sdiscovery' : 'frontpage', 'jetaudio' : 'mediaplayer', 'sharpreader' : 'rss', 'netscape' : 'netscape', 'webcollage' : 'pdaphone', 'feeddemon' : 'rss', 'wap' : 'pdaphone', 'aiderss' : 'rss', 'lg\-' : 'pdaphone', 'webzip' : 'webzip', 'pulpfiction' : 'rss', 'webreaper' : 'webreaper', 'pluck' : 'rss', 'k\-meleon' : 'kmeleon', 'msfrontpageexpress' : 'fpexpress', 'fresco' : 'fresco', 'httrack' : 'httrack', 'real' : 'real', 'konqueror' : 'konqueror', 'sie\-' : 'pdaphone', 'mozilla' : 'mozilla', 'sagem' : 'pdaphone', 'newzcrawler' : 'rss', 'rssxpress' : 'rss', 'xmms' : 'mediaplayer', 'vienna' : 'rss', 'amigavoyager' : 'amigavoyager', 'feedtools' : 'rss', 'camino' : 'chimera', 'blogbridge' : 'rss', 'bonecho' : 'firefox', 'granparadiso' : 'firefox', 'hotjava' : 'hotjava', 'up\.' : 'pdaphone', 'dillo' : 'dillo', 'liferea' : 'rss', 'getright' : 'getright', 'kazehakase' : 'mozilla', 'lotus\-notes' : 'lotusnotes', 'sage' : 'rss', 'webcopier' : 'webcopier', 'phoenix' : 'phoenix', 'sec\-' : 'pdaphone', 'xaudio' : 'mediaplayer', 'microsoft\soffice\sexistence\sdiscovery' : 'frontpage', 'chimera' : 'chimera', 'abilon' : 'abilon', 'rssowl' : 'rss', 'cyberdog' : 'cyberdog', 'newsfire' : 'rss', 'sonique' : 'mediaplayer', 'adobeair' : 'adobe', 'uplayer' : 'mediaplayer', 'amaya' : 'amaya', 'webtv' : 'webtv', 'neon' : 'neon', 'greatnews' : 'rss', 'seamonkey' : 'seamonkey', 'omniweb' : 'omniweb', 'donzilla' : 'mozilla', 'icab' : 'icab', 'firebird' : 'phoenix', 'staroffice' : 'staroffice'} + diff --git a/conf.py b/conf.py index 71f89a3..aeb6ad1 100644 --- a/conf.py +++ b/conf.py @@ -10,8 +10,8 @@ display_visitor_ip = True # Hooks used pre_analysis_hooks = ['page_to_hit', 'robots'] -post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'reverse_dns'] -display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff'] +post_analysis_hooks = ['referers', 'top_pages', 'top_downloads', 'operating_systems', 'browsers'] #, 'reverse_dns'] +display_hooks = ['top_visitors', 'all_visits', 'referers', 'top_pages', 'top_downloads', 'referers_diff', 'operating_systems', 'browsers'] # Reverse DNS timeout reverse_dns_timeout = 0.2 diff --git a/plugins/display/operating_systems.py b/plugins/display/operating_systems.py new file mode 100644 index 0000000..47ddd87 --- /dev/null +++ b/plugins/display/operating_systems.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2015 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +from iwla import IWLA +from iplugin import IPlugin +from display import * + +import awstats_data + +""" +Display hook + +Add operating systems statistics + +Plugin requirements : + post_analysis/operating_systems + +Conf values needed : + create_families_page* + +Output files : + OUTPUT_ROOT/year/month/index.html + +Statistics creation : + None + +Statistics update : + None + +Statistics deletion : + None +""" + +class IWLADisplayTopOperatingSystems(IPlugin): + def __init__(self, iwla): + super(IWLADisplayTopOperatingSystems, self).__init__(iwla) + self.API_VERSION = 1 + self.requires = ['IWLAPostAnalysisOperatingSystems'] + + def load(self): + self.icon_path = self.iwla.getConfValue('icon_path', '/') + self.create_families_pages = self.iwla.getConfValue('create_families_pages_page', True) + self.icon_names = {v:k for (k, v) in awstats_data.operating_systems_family.items()} + + return True + + def hook(self): + display = self.iwla.getDisplay() + os_families = self.iwla.getMonthStats()['os_families'] + os_families = sorted(os_families.items(), key=lambda t: t[1], reverse=True) + operating_systems = self.iwla.getMonthStats()['operating_systems'] + operating_systems = sorted(operating_systems.items(), key=lambda t: t[1], reverse=True) + + # All in a page + if self.create_families_pages: + title = createCurTitle(self.iwla, u'All Operating Systems') + filename = 'operating_systems.html' + path = self.iwla.getCurDisplayPath(filename) + + page = display.createPage(title, path, self.iwla.getConfValue('css_path', [])) + table = display.createBlock(DisplayHTMLBlockTable, self.iwla._(u'Operating Systems'), ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (os_name, entrance) in operating_systems: + icon = '' % (self.icon_path, os_name) + table.appendRow([icon, os_name, entrance]) + page.appendBlock(table) + + display.addPage(page) + + # Families in index + title = self.iwla._(u'Operating Systems') + if self.create_families_pages: + link = '%s' % (filename, self.iwla._(u'Details')) + title = '%s - %s' % (title, link) + + index = self.iwla.getDisplayIndex() + + table = display.createBlock(DisplayHTMLBlockTable, title, ['', self.iwla._(u'Operating System'), self.iwla._(u'Entrance')]) + table.setColsCSSClass(['', '', 'iwla_hit']) + for (family, entrance) in os_families: + icon = '' % (self.icon_path, self.icon_names[family]) + table.appendRow([icon, family, entrance]) + index.appendBlock(table) diff --git a/plugins/post_analysis/operating_systems.py b/plugins/post_analysis/operating_systems.py new file mode 100644 index 0000000..4fc1f73 --- /dev/null +++ b/plugins/post_analysis/operating_systems.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# +# Copyright Grégory Soutadé 2015 + +# This file is part of iwla + +# iwla is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# iwla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with iwla. If not, see . +# + +import re + +from iwla import IWLA +from iplugin import IPlugin + +import awstats_data + +""" +Post analysis hook + +Detect operating systems from requests + +Plugin requirements : + None + +Conf values needed : + None + +Output files : + None + +Statistics creation : +visits : + remote_addr => + operating_system + +month_stats : + operating_systems => + operating_system => count + + os_families => + family => count + +Statistics update : + None + +Statistics deletion : + None +""" + +class IWLAPostAnalysisOperatingSystems(IPlugin): + def __init__(self, iwla): + super(IWLAPostAnalysisOperatingSystems, self).__init__(iwla) + self.API_VERSION = 1 + + def load(self): + self.operating_systems = [] + self.os_family = {} + + for hashid in awstats_data.operating_systems: + hashid_re = re.compile(r'.*%s.*' % (hashid), re.IGNORECASE) + + if hashid in awstats_data.operating_systems_hashid.keys(): + self.operating_systems.append((hashid_re, awstats_data.operating_systems_hashid[hashid])) + + for (name, family) in awstats_data.operating_systems_family.items(): + name_re = re.compile(r'.*%s.*' % (name)) + self.os_family[name_re] = family + + return True + + def hook(self): + stats = self.iwla.getValidVisitors() + month_stats = self.iwla.getMonthStats() + + operating_systems = month_stats.get('operating_systems', {}) + + os_stats = {} + family_stats = {} + + for (k, super_hit) in stats.items(): + if not 'operating_system' in super_hit: + for r in super_hit['requests'][::-1]: + user_agent = r['http_user_agent'] + if not user_agent: continue + + os_name = 'unknown' + for (hashid_re, operating_system) in self.operating_systems: + if hashid_re.match(user_agent): + os_name = operating_system + break + super_hit['operating_system'] = os_name + break + else: + os_name = super_hit['operating_system'] + + os_family = '' + if os_name != 'unknown': + for (name_re, family) in self.os_family.items(): + if name_re.match(os_name): + os_family = family + break + + if not os_name in os_stats.keys(): + os_stats[os_name] = 1 + else: + os_stats[os_name] += 1 + + if os_family: + if not os_family in family_stats.keys(): + family_stats[os_family] = 1 + else: + family_stats[os_family] += 1 + + month_stats['operating_systems'] = os_stats + month_stats['os_families'] = family_stats diff --git a/tools/iwla_convert.pl b/tools/iwla_convert.pl index b5c9587..970ce4d 100755 --- a/tools/iwla_convert.pl +++ b/tools/iwla_convert.pl @@ -1,9 +1,9 @@ #!/usr/bin/perl -my $awstats_lib_root = './'; -my @awstats_libs = ('search_engines.pm', 'robots.pm'); +my $awstats_lib_root = '/usr/share/awstats/lib/'; +# my $awstats_lib_root = './'; +my @awstats_libs = ('search_engines.pm', 'robots.pm', 'operating_systems.pm', 'browsers.pm'); -# my $awstats_lib_root = '/usr/share/awstats/lib/'; # my @awstats_libs = ('browsers.pm', 'browsers_phone.pm', 'mime.pm', 'referer_spam.pm', 'search_engines.pm', 'operating_systems.pm', 'robots.pm', 'worms.pm'); foreach $lib (@awstats_libs) {require $awstats_lib_root . $lib;} @@ -51,6 +51,8 @@ sub dumpHash { # Robots open($FIC,">", "awstats_data.py") or die $!; +print $FIC "#This file was automatically generated by iwla_convert.pl. Do not edit manually.\n\n"; + print $FIC "robots = ["; dumpList(\@RobotsSearchIDOrder_list1, $FIC, 1); dumpList(\@RobotsSearchIDOrder_list2, $FIC, 0); @@ -76,4 +78,28 @@ print $FIC "search_engines_knwown_url = {"; dumpHash(\%SearchEnginesKnownUrl, $FIC, 1); print $FIC "}\n\n"; +print $FIC "operating_systems = ["; +dumpList(\@OSSearchIDOrder, $FIC, 1); +print $FIC "]\n\n"; + +print $FIC "operating_systems_hashid = {"; +dumpHash(\%OSHashID, $FIC, 1); +print $FIC "}\n\n"; + +print $FIC "operating_systems_family = {"; +dumpHash(\%OSFamily, $FIC, 1); +print $FIC "}\n\n"; + +print $FIC "browsers = ["; +dumpList(\@BrowsersSearchIDOrder, $FIC, 1); +print $FIC "]\n\n"; + +print $FIC "browsers_hashid = {"; +dumpHash(\%BrowsersHashIDLib, $FIC, 1); +print $FIC "}\n\n"; + +print $FIC "browsers_icons = {"; +dumpHash(\%BrowsersHashIcon, $FIC, 1); +print $FIC "}\n\n"; + close($FIC);