iwla/docs/modules.md

  * iwla.py
  * plugins/display/all_visits_enlight.py
  * plugins/display/all_visits.py
  * plugins/display/browsers.py
  * plugins/display/feeds.py
  * plugins/display/filter_users.py
  * plugins/display/hours_stats.py
  * plugins/display/ip_to_geo.py
  * plugins/display/ip_type.py
  * plugins/display/istats_diff.py
  * plugins/display/operating_systems.py
  * plugins/display/referers_diff.py
  * plugins/display/referers.py
  * plugins/display/robot_bandwidth.py
  * plugins/display/subdomains.py
  * plugins/display/top_downloads_diff.py
  * plugins/display/top_downloads.py
  * plugins/display/top_hits.py
  * plugins/display/top_pages_diff.py
  * plugins/display/top_pages.py
  * plugins/display/top_visitors.py
  * plugins/post_analysis/anonymize_ip.py
  * plugins/post_analysis/browsers.py
  * plugins/post_analysis/feeds.py
  * plugins/post_analysis/filter_users.py
  * plugins/post_analysis/hours_stats.py
  * plugins/post_analysis/ip_to_geo.py
  * plugins/post_analysis/ip_type.py
  * plugins/post_analysis/operating_systems.py
  * plugins/post_analysis/referers.py
  * plugins/post_analysis/reverse_dns.py
  * plugins/post_analysis/subdomains.py
  * plugins/post_analysis/top_downloads.py
  * plugins/post_analysis/top_hits.py
  * plugins/post_analysis/top_pages.py
  * plugins/pre_analysis/page_to_hit.py
  * plugins/pre_analysis/robots.py


iwla
----

    Main class IWLA
    Parse Log, compute them, call plugins and produce output
    For now, only HTTP log are valid
    
    Plugin requirements :
        None
    
    Conf values needed :
        analyzed_filename
        domain_name
        locales_path
        compress_output_files
        excluded_ip
    
    Output files :
        DB_ROOT/meta.db
        DB_ROOT/year/month/iwla.db
        OUTPUT_ROOT/index.html
        OUTPUT_ROOT/year/_stats.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
    
    meta :
        last_time
        start_analysis_time
        stats =>
            year =>
                month =>
                    viewed_bandwidth
                    not_viewed_bandwidth
                    viewed_pages
                    viewed_hits
                    nb_visits
                    nb_visitors
    
    month_stats :
        viewed_bandwidth
        not_viewed_bandwidth
        viewed_pages
        viewed_hits
        nb_visits
    
    days_stats :
        day =>
            viewed_bandwidth
            not_viewed_bandwidth
            viewed_pages
            viewed_hits
            nb_visits
            nb_visitors
    
    visits :
       remote_addr =>
           remote_addr
           remote_ip
           viewed_pages{0..31} # 0 contains total
           viewed_hits{0..31} # 0 contains total
           not_viewed_pages{0..31}
           not_viewed_hits{0..31}
           bandwidth{0..31}
           last_access
           requests =>
               [fields_from_format_log]
               extract_request =>
                   http_method
                   http_uri
                   http_version
                   extract_uri
                   extract_parameters*
               extract_referer* =>
                   extract_uri
                   extract_parameters*
           robot
           hit_only
           is_page
           keep_requests
    
    valid_visitors:
        month_stats without robot and hit only visitors (if not conf.count_hit_only_visitors)
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.all_visits_enlight
----------------------------------

    Display hook
    
    Enlight users in all visits page if visitor['enlight'] property is True
    or user is filtered
    Can be set in filter users callback
    
    Plugin requirements :
        IWLADisplayAllVisits
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.all_visits
--------------------------

    Display hook
    
    Create All visits page
    
    Plugin requirements :
        None
    
    Conf values needed :
        display_visitor_ip*
    
    Output files :
        OUTPUT_ROOT/year/month/all_visits.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.browsers
------------------------

    Display hook
    
    Create browsers page
    
    Plugin requirements :
        post_analysis/browsers
    
    Conf values needed :
        max_browsers_displayed*
        create_browsers_page*
    
    Output files :
        OUTPUT_ROOT/year/month/browsers.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.feeds
---------------------

    Display hook
    
    Display feeds parsers
    
    Plugin requirements :
        post_analysis/feeds
    
    Conf values needed :
        create_all_feeds_page*
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
        OUTPUT_ROOT/year/month/all_feeds.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.filter_users
----------------------------

    Display hook
    
    Filter users
    
    Plugin requirements :
        None
    
    Conf values needed :
        create_filtered_page*
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
        OUTPUT_ROOT/year/month/filtered_users.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.hours_stats
---------------------------

    Display hook
    
    Display statistics by hour/week day
    
    Plugin requirements :
        post_analysis/hours_stats
    
    Conf values needed :
        None
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.ip_to_geo
-------------------------

    Display hook
    
    Add geo statistics
    
    Plugin requirements :
        post_analysis/ip_to_geo
    
    Conf values needed :
        create_geo_page*
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.ip_type
-----------------------

    Display hook
    
    Add IPv4/IPv6 statistics
    
    Plugin requirements :
        post_analysis/ip_type
    
    Conf values needed :
        None
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.istats_diff
---------------------------

    Display hook interface
    
    Enlight new and updated statistics
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.operating_systems
---------------------------------

    Display hook
    
    Add operating systems statistics
    
    Plugin requirements :
        post_analysis/operating_systems
    
    Conf values needed :
        create_families_page*
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.referers_diff
-----------------------------

    Display hook
    
    Enlight new and updated key phrases in in all_key_phrases.html
    
    Plugin requirements :
        display/referers
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.referers
------------------------

    Display hook
    
    Create Referers page
    
    Plugin requirements :
        post_analysis/referers
    
    Conf values needed :
        max_referers_displayed*
        create_all_referers_page*
        max_key_phrases_displayed*
        create_all_key_phrases_page*
    
    Output files :
        OUTPUT_ROOT/year/month/referers.html
        OUTPUT_ROOT/year/month/key_phrases.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.robot_bandwidth
-------------------------------

    Display hook
    
    Display top 10 robot bandwidth use
    
    Plugin requirements :
        None
    
    Conf values needed :
        display_visitor_ip*
        create_all_robot_bandwidth_page*
    
    Output files :
        OUTPUT_ROOT/year/month/top_robots_bandwidth.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.subdomains
--------------------------

    Display hook
    
    Add subdomains statistics
    
    Plugin requirements :
        post_analysis/subdomains
    
    Conf values needed :
        None
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_downloads_diff
----------------------------------

    Display hook
    
    Enlight new and updated downloads in in top_downloads.html
    
    Plugin requirements :
        display/top_downloads
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_downloads
-----------------------------

    Display hook
    
    Create TOP downloads page
    
    Plugin requirements :
        post_analysis/top_downloads
    
    Conf values needed :
        max_downloads_displayed*
        create_all_downloads_page*
    
    Output files :
        OUTPUT_ROOT/year/month/top_downloads.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_hits
------------------------

    Display hook
    
    Create TOP hits page
    
    Plugin requirements :
        post_analysis/top_hits
    
    Conf values needed :
        max_hits_displayed*
        create_all_hits_page*
    
    Output files :
        OUTPUT_ROOT/year/month/top_hits.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_pages_diff
------------------------------

    Display hook
    
    Enlight new and updated pages in in top_pages.html
    
    Plugin requirements :
        display/top_pages
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_pages
-------------------------

    Display hook
    
    Create TOP pages page
    
    Plugin requirements :
        post_analysis/top_pages
    
    Conf values needed :
        max_pages_displayed*
        create_all_pages_page*
    
    Output files :
        OUTPUT_ROOT/year/month/top_pages.html
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.display.top_visitors
----------------------------

    Display hook
    
    Create TOP visitors block
    
    Plugin requirements :
        None
    
    Conf values needed :
        display_visitor_ip*
    
    Output files :
        OUTPUT_ROOT/year/month/index.html
    
    Statistics creation :
        None
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.anonymize_ip
----------------------------------

    Post analysis hook
    
    Replace remote_addr by a SHA1
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    valid_visitors:
       remote_addr
    
    Statistics deletion :
        None


plugins.post_analysis.browsers
------------------------------

    Post analysis hook
    
    Detect browser information from requests
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
    visits :
       remote_addr =>
           browser
    
    month_stats :
        browsers =>
            browser => count
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.feeds
---------------------------

    Post analysis hook
    
    Find feeds parsers (first hit in feeds conf value and no viewed pages if it's a robot)
    If merge_feeds_parsers is set to True, merge feeds parsers with the same user agent
    as it must be the same person with a different IP address.
    
    Warning : When merge_feeds_parsers is activated, last access display date is the more
    recent date of all merged parsers found
    
    Plugin requirements :
        None
    
    Conf values needed :
        feeds
        feeds_agents*
        merge_feeds_parsers*
    
    Output files :
        None
    
    Statistics creation :
       remote_addr =>
           feed_parser
           feed_name_analysed
           feed_parser_last_access (for merged parser)
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.filter_users
----------------------------------

    Post analysis hook
    
    Filter users with given user conditions
    
    Plugin requirements :
        None
    
    Conf values needed :
        filtered_users : list of filters
        filtered_ip : list of ip (string)
        create_filtered_page*
    
    Filter can be a function or a list of filter description combined by AND operator
    Filter description can be a function or a list of 3 elements :
    
      * Field to match in visits
      * Operator '=', '==', '!=', '>', '>=', '<', '<=' for int value
      * Operator '=', '==', '!=', 'in', 'match' for str value
      * Target value
    
    For easiest config, you can indicate both 'remote_addr' or 'ip' in field element
    
    function prototype is func(iwla, hit) and must return True or False
    
    Example :
    
    def my_filter(iwla, hit):
        return True
    
    filtered_users = [
        [['viewed_pages', '>=', '5'], ['viewed_hits', '>=', '5']],
        [['viewed_hits', '>=', '5'], my_filter],
        my_filter,
    ]
    
    Output files :
        None
    
    Statistics creation :
    visits :
       remote_addr =>
           filtered
           geo_location
    
    Statistics update :
    visits :
       remote_addr =>
           keep_requests
    
    Statistics deletion :
        None


plugins.post_analysis.hours_stats
---------------------------------

    Post analysis hook
    
    Count pages, hits and bandwidth by hour/week day
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
    month_stats:
        hours_stats =>
            00 .. 23 =>
                pages
                hits
                bandwidth
    
        days_stats =>
            0 .. 6 =>
                pages
                hits
                bandwidth
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.ip_to_geo
-------------------------------

    Post analysis hook
    
    Get country code from IP address
    
    Plugin requirements :
        None
    
    Conf values needed :
        iptogeo_remote_addr*
        iptogeo_remote_port*
    
    Output files :
        None
    
    Statistics creation :
        geo =>
            country_code => count
        None
    
    Statistics update :
    valid_visitors:
       country_code
    
    Statistics deletion :
        None


plugins.post_analysis.ip_type
-----------------------------

    Post analysis hook
    
    Detect if IP is IPv4 or IPv6
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
    visits :
       remote_addr =>
           ip_type
    
    month_stats :
        ip_type : {4: XXX, 6: XXX}
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.operating_systems
---------------------------------------

    Post analysis hook
    
    Detect operating systems from requests
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
    visits :
       remote_addr =>
           operating_system
    
    month_stats :
        operating_systems =>
            operating_system => count
    
        os_families =>
            family => count
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.referers
------------------------------

    Post analysis hook
    
    Extract referers and key phrases from requests
    
    Plugin requirements :
        None
    
    Conf values needed :
        domain_name
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    month_stats :
       referers =>
           pages => count
           hits  => count
       robots_referers =>
           pages => count
           hits  => count
       search_engine_referers =>
           pages => count
           hits  => count
       key_phrases =>
           phrase => count
    
    Statistics deletion :
        None


plugins.post_analysis.reverse_dns
---------------------------------

    Post analysis hook
    
    Replace IP by reverse DNS names
    
    Plugin requirements :
        None
    
    Conf values needed :
        reverse_dns_timeout*
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    valid_visitors:
       remote_addr
       dns_name_replaced
       dns_analyzed
    
    Statistics deletion :
        None


plugins.post_analysis.subdomains
--------------------------------

    Post analysis hook
    
    Group top pages by subdomains
    
    Plugin requirements :
        post_analysis/top_pages
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
    month_stats:
       subdomains =>
           domain => count
    
    Statistics update :
        None
    
    Statistics deletion :
        None


plugins.post_analysis.top_downloads
-----------------------------------

    Post analysis hook
    
    Count TOP downloads
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    month_stats:
       top_downloads =>
           uri => count
    
    Statistics deletion :
        None


plugins.post_analysis.top_hits
------------------------------

    Post analysis hook
    
    Count TOP hits
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    month_stats:
       top_hits =>
           uri => count
    
    Statistics deletion :
        None


plugins.post_analysis.top_pages
-------------------------------

    Post analysis hook
    
    Count TOP pages
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    month_stats:
       top_pages =>
           uri => count
    
    Statistics deletion :
        None


plugins.pre_analysis.page_to_hit
--------------------------------

    Pre analysis hook
    Change page into hit and hit into page into statistics
    
    Plugin requirements :
        None
    
    Conf values needed :
        page_to_hit_conf*
        hit_to_page_conf*
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    visits :
       remote_addr =>
           is_page
    
    Statistics deletion :
        None


plugins.pre_analysis.robots
---------------------------

    Pre analysis hook
    
    Filter robots
    
    Plugin requirements :
        None
    
    Conf values needed :
        None
    
    Output files :
        None
    
    Statistics creation :
        None
    
    Statistics update :
    visits :
       remote_addr =>
           robot
           keep_requests
    
    Statistics deletion :
        None