import_logs.py

#!/usr/bin/python
# vim: et sw=4 ts=4:
# -*- coding: utf-8 -*-
#
# Piwik - free/libre analytics platform
#
# @link http://piwik.org
# @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
# @version $Id$
#
# For more info see: http://piwik.org/log-analytics/ and http://piwik.org/docs/log-analytics-tool-how-to/
#
# Requires Python 2.6 or greater.
#

import base64
import bz2
import ConfigParser
import datetime
import fnmatch
import gzip
import hashlib
import httplib
import inspect
import itertools
import logging
import optparse
import os
import os.path
import Queue
import re
import sys
import threading
import time
import urllib
import urllib2
import urlparse
import subprocess
import functools

try:
    import json
except ImportError:
    try:
        import simplejson as json
    except ImportError:
        if sys.version_info < (2, 6):
            print >> sys.stderr, 'simplejson (http://pypi.python.org/pypi/simplejson/) is required.'
            sys.exit(1)



##
## Constants.
##

STATIC_EXTENSIONS = set((
    'gif jpg jpeg png bmp ico svg ttf eot woff class swf css js xml robots.txt'
).split())

DOWNLOAD_EXTENSIONS = set((
    '7z aac arc arj asf asx avi bin csv deb dmg doc docx exe flv gz gzip hqx '
    'jar mpg mp2 mp3 mp4 mpeg mov movie msi msp odb odf odg odp '
    'ods odt ogg ogv pdf phps ppt pptx qt qtm ra ram rar rpm sea sit tar tbz '
    'bz2 tbz tgz torrent txt wav wma wmv wpd xls xlsx xml xsd z zip '
    'azw3 epub mobi apk'
).split())

# A good source is: http://phpbb-bots.blogspot.com/
EXCLUDED_USER_AGENTS = (
    'adsbot-google',
    'ask jeeves',
    'baidubot',
    'bot-',
    'bot/',
    'ccooter/',
    'crawl',
    'curl',
    'echoping',
    'exabot',
    'feed',
    'googlebot',
    'ia_archiver',
    'java/',
    'libwww',
    'mediapartners-google',
    'msnbot',
    'netcraftsurvey',
    'panopta',
    'robot',
    'spider',
    'surveybot',
    'twiceler',
    'voilabot',
    'yahoo',
    'yandex',
)

PIWIK_MAX_ATTEMPTS = 3
PIWIK_DELAY_AFTER_FAILURE = 2

PIWIK_EXPECTED_IMAGE = base64.b64decode(
    'R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='
)

##
## Formats.
##

class BaseFormatException(Exception): pass

class BaseFormat(object):
    def __init__(self, name):
        self.name = name
        self.regex = None
        self.date_format = '%d/%b/%Y:%H:%M:%S'

    def check_format(self, file):
        line = file.readline()
        file.seek(0)
        return self.check_format_line(line)

    def check_format_line(self, line):
        return False

class JsonFormat(BaseFormat):
    def __init__(self, name):
        super(JsonFormat, self).__init__(name)
        self.json = None
        self.date_format = '%Y-%m-%dT%H:%M:%S'

    def check_format_line(self, line):
        try:
            self.json = json.loads(line)
            return True
        except:
            return False

    def match(self, line):
        try:
            self.json = json.loads(line)
            return self
        except:
            self.json = None
            return None

    def get(self, key):
        # Some ugly patchs ...
        if key == 'generation_time_milli':
            self.json[key] =  int(self.json[key] * 1000)
        # Patch date format ISO 8601
        elif key == 'date':
            tz = self.json[key][19:]
            self.json['timezone'] = tz.replace(':', '')
            self.json[key] = self.json[key][:19]

        try:
            return self.json[key]
        except KeyError:
            raise BaseFormatException()

    def get_all(self,):
        return self.json

    def remove_ignored_groups(self, groups):
        for group in groups:
            del self.json[group]

class RegexFormat(BaseFormat):

    def __init__(self, name, regex, date_format=None):
        super(RegexFormat, self).__init__(name)
        if regex is not None:
            self.regex = re.compile(regex)
        if date_format is not None:
            self.date_format = date_format
        self.matched = None

    def check_format_line(self, line):
        return self.match(line)

    def match(self,line):
        match_result = self.regex.match(line)
        if match_result:
            self.matched = match_result.groupdict()
        else:
            self.matched = None
        return match_result

    def get(self, key):
        try:
            return self.matched[key]
        except KeyError:
            raise BaseFormatException()

    def get_all(self,):
        return self.matched

    def remove_ignored_groups(self, groups):
        for group in groups:
            del self.matched[group]

class W3cExtendedFormat(RegexFormat):

    FIELDS_LINE_PREFIX = '#Fields: '

    fields = {
        'date': '(?P<date>^\d+[-\d+]+',
        'time': '[\d+:]+)[.\d]*?', # TODO should not assume date & time will be together not sure how to fix ATM.
        'cs-uri-stem': '(?P<path>/\S*)',
        'cs-uri-query': '(?P<query_string>\S*)',
        'c-ip': '"?(?P<ip>[\d*.]*)"?',
        'cs(User-Agent)': '(?P<user_agent>".*?"|\S+)',
        'cs(Referer)': '(?P<referrer>\S+)',
        'sc-status': '(?P<status>\d+)',
        'sc-bytes': '(?P<length>\S+)',
        'cs-host': '(?P<host>\S+)',
        'cs-username': '(?P<userid>\S+)',
        'time-taken': '(?P<generation_time_secs>[.\d]+)'
    }

    def __init__(self):
        super(W3cExtendedFormat, self).__init__('w3c_extended', None, '%Y-%m-%d %H:%M:%S')

    def check_format(self, file):
        self.create_regex(file)

        # if we couldn't create a regex, this file does not follow the W3C extended log file format
        if not self.regex:
            file.seek(0)
            return

        first_line = file.readline()

        file.seek(0)
        return self.check_format_line(first_line)

    def create_regex(self, file):
        fields_line = None
        if config.options.w3c_fields:
            fields_line = config.options.w3c_fields

        # collect all header lines up until the Fields: line
        # if we're reading from stdin, we can't seek, so don't read any more than the Fields line
        header_lines = []
        while fields_line is None:
            line = file.readline()

            if not line.startswith('#'):
                break

            if line.startswith(W3cExtendedFormat.FIELDS_LINE_PREFIX):
                fields_line = line
            else:
                header_lines.append(line)

        if not fields_line:
            return

        # store the header lines for a later check for IIS
        self.header_lines = header_lines

        # Parse the 'Fields: ' line to create the regex to use
        full_regex = []

        expected_fields = type(self).fields.copy() # turn custom field mapping into field => regex mapping

        # if the --w3c-time-taken-millisecs option is used, make sure the time-taken field is interpreted as milliseconds
        if config.options.w3c_time_taken_in_millisecs:
            expected_fields['time-taken'] = '(?P<generation_time_milli>[\d.]+)'

        for mapped_field_name, field_name in config.options.custom_w3c_fields.iteritems():
            expected_fields[mapped_field_name] = expected_fields[field_name]
            del expected_fields[field_name]

        # add custom field regexes supplied through --w3c-field-regex option
        for field_name, field_regex in config.options.w3c_field_regexes.iteritems():
            expected_fields[field_name] = field_regex

        # Skip the 'Fields: ' prefix.
        fields_line = fields_line[9:]
        for field in fields_line.split():
            try:
                regex = expected_fields[field]
            except KeyError:
                regex = '\S+'
            full_regex.append(regex)
        full_regex = '\s+'.join(full_regex)
        self.regex = re.compile(full_regex)

    def check_for_iis_option(self):
        if not config.options.w3c_time_taken_in_millisecs and self._is_time_taken_milli() and self._is_iis():
            logging.info("WARNING: IIS log file being parsed without --w3c-time-taken-milli option. IIS"
                         " stores millisecond values in the time-taken field. If your logfile does this, the aforementioned"
                         " option must be used in order to get accurate generation times.")

    def _is_iis(self):
        return len([line for line in self.header_lines if 'internet information services' in line.lower() or 'iis' in line.lower()]) > 0

    def _is_time_taken_milli(self):
        return 'generation_time_milli' not in self.regex.pattern

class IisFormat(W3cExtendedFormat):

    fields = W3cExtendedFormat.fields.copy()
    fields.update({
        'time-taken': '(?P<generation_time_milli>[.\d]+)',
        'sc-win32-status': '(?P<__win32_status>\S+)' # this group is useless for log importing, but capturing it
                                                     # will ensure we always select IIS for the format instead of
                                                     # W3C logs when detecting the format. This way there will be
                                                     # less accidental importing of IIS logs w/o --w3c-time-taken-milli.
    })

    def __init__(self):
        super(IisFormat, self).__init__()

        self.name = 'iis'

class AmazonCloudFrontFormat(W3cExtendedFormat):

    fields = W3cExtendedFormat.fields.copy()
    fields.update({
        'x-event': '(?P<event_action>\S+)',
        'x-sname': '(?P<event_name>\S+)',
        'cs-uri-stem': '(?:rtmp:/)?(?P<path>/\S*)',
        'c-user-agent': '(?P<user_agent>".*?"|\S+)'
    })

    def __init__(self):
        super(AmazonCloudFrontFormat, self).__init__()

        self.name = 'amazon_cloudfront'

    def get(self, key):
        if key == 'event_category' and 'event_category' not in self.matched:
            return 'cloudfront_rtmp'
        elif key == 'status' and 'status' not in self.matched:
            return '200'
        else:
            return super(AmazonCloudFrontFormat, self).get(key)

_HOST_PREFIX = '(?P<host>[\w\-\.]*)(?::\d+)? '
_COMMON_LOG_FORMAT = (
    '(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?) (?P<timezone>.*?)\] '
    '"\S+ (?P<path>.*?) \S+" (?P<status>\S+) (?P<length>\S+)'
)
_NCSA_EXTENDED_LOG_FORMAT = (_COMMON_LOG_FORMAT +
    ' "(?P<referrer>.*?)" "(?P<user_agent>.*?)"'
)
_S3_LOG_FORMAT = (
    '\S+ (?P<host>\S+) \[(?P<date>.*?) (?P<timezone>.*?)\] (?P<ip>\S+) '
    '\S+ \S+ \S+ \S+ "\S+ (?P<path>.*?) \S+" (?P<status>\S+) \S+ (?P<length>\S+) '
    '\S+ \S+ \S+ "(?P<referrer>.*?)" "(?P<user_agent>.*?)"'
)
_ICECAST2_LOG_FORMAT = ( _NCSA_EXTENDED_LOG_FORMAT +
    ' (?P<session_time>\S+)'
)

FORMATS = {
    'common': RegexFormat('common', _COMMON_LOG_FORMAT),
    'common_vhost': RegexFormat('common_vhost', _HOST_PREFIX + _COMMON_LOG_FORMAT),
    'ncsa_extended': RegexFormat('ncsa_extended', _NCSA_EXTENDED_LOG_FORMAT),
    'common_complete': RegexFormat('common_complete', _HOST_PREFIX + _NCSA_EXTENDED_LOG_FORMAT),
    'w3c_extended': W3cExtendedFormat(),
    'amazon_cloudfront': AmazonCloudFrontFormat(),
    'iis': IisFormat(),
    's3': RegexFormat('s3', _S3_LOG_FORMAT),
    'icecast2': RegexFormat('icecast2', _ICECAST2_LOG_FORMAT),
    'nginx_json': JsonFormat('nginx_json'),
}

##
## Code.
##

class Configuration(object):
    """
    Stores all the configuration options by reading sys.argv and parsing,
    if needed, the config.inc.php.

    It has 2 attributes: options and filenames.
    """

    class Error(Exception):
        pass

    def _create_parser(self):
        """
        Initialize and return the OptionParser instance.
        """
        option_parser = optparse.OptionParser(
            usage='Usage: %prog [options] log_file [ log_file [...] ]',
            description="Import HTTP access logs to Piwik. "
                         "log_file is the path to a server access log file (uncompressed, .gz, .bz2, or specify - to read from stdin). "
                         " By default, the script will try to produce clean reports and will exclude bots, static files, discard http error and redirects, etc. This is customizable, see below.",
            epilog="About Piwik Server Log Analytics: http://piwik.org/log-analytics/ "
                   "              Found a bug? Please create a ticket in http://dev.piwik.org/ "
                   "              Please send your suggestions or successful user story to hello@piwik.org "
        )

        option_parser.add_option(
            '--debug', '-d', dest='debug', action='count', default=0,
            help="Enable debug output (specify multiple times for more verbose)",
        )
        option_parser.add_option(
            '--url', dest='piwik_url',
            help="REQUIRED Your Piwik server URL, eg. http://example.com/piwik/ or http://analytics.example.net",
        )
        option_parser.add_option(
            '--dry-run', dest='dry_run',
            action='store_true', default=False,
            help="Perform a trial run with no tracking data being inserted into Piwik",
        )
        option_parser.add_option(
            '--show-progress', dest='show_progress',
            action='store_true', default=os.isatty(sys.stdout.fileno()),
            help="Print a progress report X seconds (default: 1, use --show-progress-delay to override)"
        )
        option_parser.add_option(
            '--show-progress-delay', dest='show_progress_delay',
            type='int', default=1,
            help="Change the default progress delay"
        )
        option_parser.add_option(
            '--add-sites-new-hosts', dest='add_sites_new_hosts',
            action='store_true', default=False,
            help="When a hostname is found in the log file, but not matched to any website "
            "in Piwik, automatically create a new website in Piwik with this hostname to "
            "import the logs"
        )
        option_parser.add_option(
            '--idsite', dest='site_id',
            help= ("When specified, "
                   "data in the specified log files will be tracked for this Piwik site ID."
                   " The script will not auto-detect the website based on the log line hostname (new websites will not be automatically created).")
        )
        option_parser.add_option(
            '--idsite-fallback', dest='site_id_fallback',
            help="Default Piwik site ID to use if the hostname doesn't match any "
            "known Website's URL. New websites will not be automatically created. "
            "                         Used only if --add-sites-new-hosts or --idsite are not set",
        )
        default_config = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
            '../../config/config.ini.php'),
        )
        option_parser.add_option(
            '--config', dest='config_file', default=default_config,
            help=(
                "This is only used when --login and --password is not used. "
                "Piwik will read the configuration file (default: %default) to "
                "fetch the Super User token_auth from the config file. "
            )
        )
        option_parser.add_option(
            '--login', dest='login',
            help="You can manually specify the Piwik Super User login"
        )
        option_parser.add_option(
            '--password', dest='password',
            help="You can manually specify the Piwik Super User password"
        )
        option_parser.add_option(
            '--token-auth', dest='piwik_token_auth',
            help="Piwik Super User token_auth, 32 characters hexadecimal string, found in Piwik > API",
        )

        option_parser.add_option(
            '--hostname', dest='hostnames', action='append', default=[],
            help="Accepted hostname (requests with other hostnames will be excluded). "
            "Can be specified multiple times"
        )
        option_parser.add_option(
            '--exclude-path', dest='excluded_paths', action='append', default=[],
            help="Any URL path matching this exclude-path will not be imported in Piwik. Can be specified multiple times"
        )
        option_parser.add_option(
            '--exclude-path-from', dest='exclude_path_from',
            help="Each line from this file is a path to exclude (see: --exclude-path)"
        )
        option_parser.add_option(
            '--include-path', dest='included_paths', action='append', default=[],
            help="Paths to include. Can be specified multiple times. If not specified, all paths are included."
        )
        option_parser.add_option(
            '--include-path-from', dest='include_path_from',
            help="Each line from this file is a path to include"
        )
        option_parser.add_option(
            '--useragent-exclude', dest='excluded_useragents',
            action='append', default=[],
            help="User agents to exclude (in addition to the standard excluded "
            "user agents). Can be specified multiple times",
        )
        option_parser.add_option(
            '--enable-static', dest='enable_static',
            action='store_true', default=False,
            help="Track static files (images, css, js, ico, ttf, etc.)"
        )
        option_parser.add_option(
            '--enable-bots', dest='enable_bots',
            action='store_true', default=False,
            help="Track bots. All bot visits will have a Custom Variable set with name='Bot' and value='$Bot_user_agent_here$'"
        )
        option_parser.add_option(
            '--enable-http-errors', dest='enable_http_errors',
            action='store_true', default=False,
            help="Track HTTP errors (status code 4xx or 5xx)"
        )
        option_parser.add_option(
            '--enable-http-redirects', dest='enable_http_redirects',
            action='store_true', default=False,
            help="Track HTTP redirects (status code 3xx except 304)"
        )
        option_parser.add_option(
            '--enable-reverse-dns', dest='reverse_dns',
            action='store_true', default=False,
            help="Enable reverse DNS, used to generate the 'Providers' report in Piwik. "
                 "Disabled by default, as it impacts performance"
        )
        option_parser.add_option(
            '--strip-query-string', dest='strip_query_string',
            action='store_true', default=False,
            help="Strip the query string from the URL"
        )
        option_parser.add_option(
            '--query-string-delimiter', dest='query_string_delimiter', default='?',
            help="The query string delimiter (default: %default)"
        )
        option_parser.add_option(
            '--log-format-name', dest='log_format_name', default=None,
            help=("Access log format to detect (supported are: %s). "
                  "When not specified, the log format will be autodetected by trying all supported log formats."
                  % ', '.join(sorted(FORMATS.iterkeys())))
        )
        available_regex_groups = ['date', 'path', 'query_string', 'ip', 'user_agent', 'referrer', 'status',
                                  'length', 'host', 'userid', 'generation_time_milli', 'event_action',
                                  'event_name', 'timezone', 'session_time']
        option_parser.add_option(
            '--log-format-regex', dest='log_format_regex', default=None,
            help="Regular expression used to parse log entries. Regexes must contain named groups for different log fields. "
                 "Recognized fields include: %s. For an example of a supported Regex, see the source code of this file. "
                 "Overrides --log-format-name." % (', '.join(available_regex_groups))
        )
        option_parser.add_option(
            '--log-hostname', dest='log_hostname', default=None,
            help="Force this hostname for a log format that doesn't incldude it. All hits "
            "will seem to came to this host"
        )
        option_parser.add_option(
            '--skip', dest='skip', default=0, type='int',
            help="Skip the n first lines to start parsing/importing data at a given line for the specified log file",
        )
        option_parser.add_option(
            '--recorders', dest='recorders', default=1, type='int',
            help="Number of simultaneous recorders (default: %default). "
            "It should be set to the number of CPU cores in your server. "
            "You can also experiment with higher values which may increase performance until a certain point",
        )
        option_parser.add_option(
            '--recorder-max-payload-size', dest='recorder_max_payload_size', default=200, type='int',
            help="Maximum number of log entries to record in one tracking request (default: %default). "
        )
        option_parser.add_option(
            '--replay-tracking', dest='replay_tracking',
            action='store_true', default=False,
            help="Replay piwik.php requests found in custom logs (only piwik.php requests expected). \nSee http://piwik.org/faq/how-to/faq_17033/"
        )
        option_parser.add_option(
            '--output', dest='output',
            help="Redirect output (stdout and stderr) to the specified file"
        )
        option_parser.add_option(
            '--encoding', dest='encoding', default='utf8',
            help="Log files encoding (default: %default)"
        )
        option_parser.add_option(
            '--disable-bulk-tracking', dest='use_bulk_tracking',
            default=True, action='store_false',
            help="Disables use of bulk tracking so recorders record one hit at a time."
        )
        option_parser.add_option(
            '--debug-force-one-hit-every-Ns', dest='force_one_action_interval', default=False, type='float',
            help="Debug option that will force each recorder to record one hit every N secs."
        )
        option_parser.add_option(
            '--invalidate-dates', dest='invalidate_dates', default=None,
            help="Invalidate reports for the specified dates (format: YYYY-MM-DD,YYYY-MM-DD,...). "
                 "By default, all dates found in the logs will be invalidated.",
        )
        option_parser.add_option(
            '--force-lowercase-path', dest='force_lowercase_path', default=False, action='store_true',
            help="Make URL path lowercase so paths with the same letters but different cases are "
                 "treated the same."
        )
        option_parser.add_option(
            '--enable-testmode', dest='enable_testmode', default=False, action='store_true',
            help="If set, it will try to get the token_auth from the piwik_tests directory"
        )
        option_parser.add_option(
            '--download-extensions', dest='download_extensions', default=None,
            help="By default Piwik tracks as Downloads the most popular file extensions. If you set this parameter (format: pdf,doc,...) then files with an extension found in the list will be imported as Downloads, other file extensions downloads will be skipped."
        )
        option_parser.add_option(
            '--w3c-map-field', action='callback', callback=functools.partial(self._set_option_map, 'custom_w3c_fields'), type='string',
            help="Map a custom log entry field in your W3C log to a default one. Use this option to load custom log "
                 "files that use the W3C extended log format such as those from the Advanced Logging W3C module. Used "
                 "as, eg, --w3c-map-field my-date=date. Recognized default fields include: %s\n\n"
                 "Formats that extend the W3C extended log format (like the cloudfront RTMP log format) may define more "
                 "fields that can be mapped."
                     % (', '.join(W3cExtendedFormat.fields.keys()))
        )
        option_parser.add_option(
            '--w3c-time-taken-millisecs', action='store_true', default=False, dest='w3c_time_taken_in_millisecs',
            help="If set, interprets the time-taken W3C log field as a number of milliseconds. This must be set for importing"
                 " IIS logs."
        )
        option_parser.add_option(
            '--w3c-fields', dest='w3c_fields', default=None,
            help="Specify the '#Fields:' line for a log file in the W3C Extended log file format. Use this option if "
                 "your log file doesn't contain the '#Fields:' line which is required for parsing. This option must be used "
                 "in conjuction with --log-format-name=w3c_extended.\n"
                 "Example: --w3c-fields='#Fields: date time c-ip ...'"
        )
        option_parser.add_option(
            '--w3c-field-regex', action='callback', callback=functools.partial(self._set_option_map, 'w3c_field_regexes'), type='string',
            help="Specify a regex for a field in your W3C extended log file. You can use this option to parse fields the "
                 "importer does not natively recognize and then use one of the --regex-group-to-XXX-cvar options to track "
                 "the field in a custom variable. For example, specifying --w3c-field-regex=sc-win32-status=(?P<win32_status>\\S+) "
                 "--regex-group-to-page-cvar=\"win32_status=Windows Status Code\" will track the sc-win32-status IIS field "
                 "in the 'Windows Status Code' custom variable. Regexes must contain a named group."
        )
        option_parser.add_option(
            '--title-category-delimiter', dest='title_category_delimiter', default='/',
            help="If --enable-http-errors is used, errors are shown in the page titles report. If you have "
            "changed General.action_title_category_delimiter in your Piwik configuration, you need to set this "
            "option to the same value in order to get a pretty page titles report."
        )
        option_parser.add_option(
            '--dump-log-regex', dest='dump_log_regex', action='store_true', default=False,
            help="Prints out the regex string used to parse log lines and exists. Can be useful for using formats "
                 "in newer versions of the script in older versions of the script. The output regex can be used with "
                 "the --log-format-regex option."
        )

        option_parser.add_option(
            '--ignore-groups', dest='regex_groups_to_ignore', default=None,
            help="Comma separated list of regex groups to ignore when parsing log lines. Can be used to, for example, "
                 "disable normal user id tracking. See documentation for --log-format-regex for list of available "
                 "regex groups."
        )

        option_parser.add_option(
            '--regex-group-to-visit-cvar', action='callback', callback=functools.partial(self._set_option_map, 'regex_group_to_visit_cvars_map'), type='string',
            help="Track an attribute through a custom variable with visit scope instead of through Piwik's normal "
                 "approach. For example, to track usernames as a custom variable instead of through the uid tracking "
                 "parameter, supply --regex-group-to-visit-cvar=\"userid=User Name\". This will track usernames in a "
                 "custom variable named 'User Name'. See documentation for --log-format-regex for list of available "
                 "regex groups."
        )
        option_parser.add_option(
            '--regex-group-to-page-cvar', action='callback', callback=functools.partial(self._set_option_map, 'regex_group_to_page_cvars_map'), type='string',
            help="Track an attribute through a custom variable with page scope instead of through Piwik's normal "
                 "approach. For example, to track usernames as a custom variable instead of through the uid tracking "
                 "parameter, supply --regex-group-to-page-cvar=\"userid=User Name\". This will track usernames in a "
                 "custom variable named 'User Name'. See documentation for --log-format-regex for list of available "
                 "regex groups."
        )
        return option_parser

    def _set_option_map(self, option_attr_name, option, opt_str, value, parser):
        """
        Sets a key-value mapping in a dict that is built from command line options. Options that map
        string keys to string values (like --w3c-map-field) can set the callback to a bound partial
        of this method to handle the option.
        """

        parts = value.split('=')

        if len(parts) != 2:
            fatal_error("Invalid %s option: '%s'" % (opt_str, value))

        key, value = parts

        if not hasattr(parser.values, option_attr_name):
            setattr(parser.values, option_attr_name, {})

        getattr(parser.values, option_attr_name)[key] = value

    def _parse_args(self, option_parser):
        """
        Parse the command line args and create self.options and self.filenames.
        """
        self.options, self.filenames = option_parser.parse_args(sys.argv[1:])

        if self.options.output:
            sys.stdout = sys.stderr = open(self.options.output, 'a+', 0)

        if not self.filenames:
            print(option_parser.format_help())
            sys.exit(1)

        # Configure logging before calling logging.{debug,info}.
        logging.basicConfig(
            format='%(asctime)s: [%(levelname)s] %(message)s',
            level=logging.DEBUG if self.options.debug >= 1 else logging.INFO,
        )

        self.options.excluded_useragents = set([s.lower() for s in self.options.excluded_useragents])

        if self.options.exclude_path_from:
            paths = [path.strip() for path in open(self.options.exclude_path_from).readlines()]
            self.options.excluded_paths.extend(path for path in paths if len(path) > 0)
        if self.options.excluded_paths:
            self.options.excluded_paths = set(self.options.excluded_paths)
            logging.debug('Excluded paths: %s', ' '.join(self.options.excluded_paths))

        if self.options.include_path_from:
            paths = [path.strip() for path in open(self.options.include_path_from).readlines()]
            self.options.included_paths.extend(path for path in paths if len(path) > 0)
        if self.options.included_paths:
            self.options.included_paths = set(self.options.included_paths)
            logging.debug('Included paths: %s', ' '.join(self.options.included_paths))

        if self.options.hostnames:
            logging.debug('Accepted hostnames: %s', ', '.join(self.options.hostnames))
        else:
            logging.debug('Accepted hostnames: all')

        if self.options.log_format_regex:
            self.format = RegexFormat('custom', self.options.log_format_regex)
        elif self.options.log_format_name:
            try:
                self.format = FORMATS[self.options.log_format_name]
            except KeyError:
                fatal_error('invalid log format: %s' % self.options.log_format_name)
        else:
            self.format = None

        if not hasattr(self.options, 'custom_w3c_fields'):
            self.options.custom_w3c_fields = {}
        elif self.format is not None:
            # validate custom field mappings
            for custom_name, default_name in self.options.custom_w3c_fields.iteritems():
                if default_name not in type(format).fields:
                    fatal_error("custom W3C field mapping error: don't know how to parse and use the '%' field" % default_name)
                    return

        if not hasattr(self.options, 'regex_group_to_visit_cvars_map'):
            self.options.regex_group_to_visit_cvars_map = {}

        if not hasattr(self.options, 'regex_group_to_page_cvars_map'):
            self.options.regex_group_to_page_cvars_map = {}

        if not hasattr(self.options, 'w3c_field_regexes'):
            self.options.w3c_field_regexes = {}
        else:
            # make sure each custom w3c field regex has a named group
            for field_name, field_regex in self.options.w3c_field_regexes.iteritems():
                if '(?P<' not in field_regex:
                    fatal_error("cannot find named group in custom w3c field regex '%s' for field '%s'" % (field_regex, field_name))
                    return

        if not self.options.piwik_url:
            fatal_error('no URL given for Piwik')

        if not (self.options.piwik_url.startswith('http://') or self.options.piwik_url.startswith('https://')):
            self.options.piwik_url = 'http://' + self.options.piwik_url
        logging.debug('Piwik URL is: %s', self.options.piwik_url)

        if not self.options.piwik_token_auth:
            try:
                self.options.piwik_token_auth = self._get_token_auth()
            except Piwik.Error, e:
                fatal_error(e)
        logging.debug('Authentication token token_auth is: %s', self.options.piwik_token_auth)

        if self.options.recorders < 1:
            self.options.recorders = 1

        if self.options.download_extensions:
            self.options.download_extensions = set(self.options.download_extensions.split(','))
        else:
            self.options.download_extensions = DOWNLOAD_EXTENSIONS

        if self.options.regex_groups_to_ignore:
            self.options.regex_groups_to_ignore = set(self.options.regex_groups_to_ignore.split())

    def __init__(self):
        self._parse_args(self._create_parser())

    def _get_token_auth(self):
        """
        If the token auth is not specified in the options, get it from Piwik.
        """
        # Get superuser login/password from the options.
        logging.debug('No token-auth specified')

        if self.options.login and self.options.password:
            piwik_login = self.options.login
            piwik_password = hashlib.md5(self.options.password).hexdigest()

            logging.debug('Using credentials: (login = %s, password = %s)', piwik_login, piwik_password)
            try:
                api_result = piwik.call_api('UsersManager.getTokenAuth',
                    userLogin=piwik_login,
                    md5Password=piwik_password,
                    _token_auth='',
                    _url=self.options.piwik_url,
                )
            except urllib2.URLError, e:
                fatal_error('error when fetching token_auth from the API: %s' % e)

            try:
                return api_result['value']
            except KeyError:
                # Happens when the credentials are invalid.
                message = api_result.get('message')
                fatal_error(
                    'error fetching authentication token token_auth%s' % (
                    ': %s' % message if message else '')
                )
        else:
            # Fallback to the given (or default) configuration file, then
            # get the token from the API.
            logging.debug(
                'No credentials specified, reading them from "%s"',
                self.options.config_file,
            )
            config_file = ConfigParser.RawConfigParser()
            success = len(config_file.read(self.options.config_file)) > 0
            if not success:
                fatal_error(
                    "the configuration file" + self.options.config_file + " could not be read. Please check permission. This file must be readable to get the authentication token"
                )

            updatetokenfile = os.path.abspath(
                os.path.join(os.path.dirname(__file__),
                    '../../misc/cron/updatetoken.php'),
            )

            phpBinary = 'php'

            is_windows = sys.platform.startswith('win')
            if is_windows:
                try:
                    processWin = subprocess.Popen('where php.exe', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    [stdout, stderr] = processWin.communicate()
                    if processWin.returncode == 0:
                        phpBinary = stdout.strip()
                    else:
                        fatal_error("We couldn't detect PHP. It might help to add your php.exe to the path or alternatively run the importer using the --login and --password option")
                except:
                    fatal_error("We couldn't detect PHP. You can run the importer using the --login and --password option to fix this issue")

            command = [phpBinary, updatetokenfile]
            if self.options.enable_testmode:
                command.append('--testmode')

            hostname = urlparse.urlparse( self.options.piwik_url ).hostname
            command.append('--piwik-domain=' + hostname )

            command = subprocess.list2cmdline(command)
            process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            [stdout, stderr] = process.communicate()
            if process.returncode != 0:
                fatal_error("`" + command + "` failed with error: " + stderr + ".\nReponse code was: " + str(process.returncode) + ". You can alternatively run the importer using the --login and --password option")

            filename = stdout
            credentials = open(filename, 'r').readline()
            credentials = credentials.split('\t')
            return credentials[1]

    def get_resolver(self):
        if self.options.site_id:
            logging.debug('Resolver: static')
            return StaticResolver(self.options.site_id)
        else:
            logging.debug('Resolver: dynamic')
            return DynamicResolver()

class Statistics(object):
    """
    Store statistics about parsed logs and recorded entries.
    Can optionally print statistics on standard output every second.
    """

    class Counter(object):
        """
        Simple integers cannot be used by multithreaded programs. See:
        http://stackoverflow.com/questions/6320107/are-python-ints-thread-safe
        """
        def __init__(self):
            # itertools.count's implementation in C does not release the GIL and
            # therefore is thread-safe.
            self.counter = itertools.count(1)
            self.value = 0

        def increment(self):
            self.value = self.counter.next()

        def advance(self, n):
            for i in range(n):
                self.increment()

        def __str__(self):
            return str(int(self.value))

    def __init__(self):
        self.time_start = None
        self.time_stop = None

        self.piwik_sites = set()                # sites ID
        self.piwik_sites_created = []           # (hostname, site ID)
        self.piwik_sites_ignored = set()        # hostname

        self.count_lines_parsed = self.Counter()
        self.count_lines_recorded = self.Counter()

        # Do not match the regexp.
        self.count_lines_invalid = self.Counter()
        # No site ID found by the resolver.
        self.count_lines_no_site = self.Counter()
        # Hostname filtered by config.options.hostnames
        self.count_lines_hostname_skipped = self.Counter()
        # Static files.
        self.count_lines_static = self.Counter()
        # Ignored user-agents.
        self.count_lines_skipped_user_agent = self.Counter()
        # Ignored HTTP erors.
        self.count_lines_skipped_http_errors = self.Counter()
        # Ignored HTTP redirects.
        self.count_lines_skipped_http_redirects = self.Counter()
        # Downloads
        self.count_lines_downloads = self.Counter()
        # Ignored downloads when --download-extensions is used
        self.count_lines_skipped_downloads = self.Counter()

        # Misc
        self.dates_recorded = set()
        self.monitor_stop = False

    def set_time_start(self):
        self.time_start = time.time()

    def set_time_stop(self):
        self.time_stop = time.time()

    def _compute_speed(self, value, start, end):
        delta_time = end - start
        if value == 0:
            return 0
        if delta_time == 0:
            return 'very high!'
        else:
            return value / delta_time

    def _round_value(self, value, base=100):
        return round(value * base) / base

    def _indent_text(self, lines, level=1):
        """
        Return an indented text. 'lines' can be a list of lines or a single
        line (as a string). One level of indentation is 4 spaces.
        """
        prefix = ' ' * (4 * level)
        if isinstance(lines, basestring):
            return prefix + lines
        else:
            return '\n'.join(
                prefix + line
                for line in lines
            )

    def print_summary(self):
        print '''
Logs import summary
-------------------

    %(count_lines_recorded)d requests imported successfully
    %(count_lines_downloads)d requests were downloads
    %(total_lines_ignored)d requests ignored:
        %(count_lines_skipped_http_errors)d HTTP errors
        %(count_lines_skipped_http_redirects)d HTTP redirects
        %(count_lines_invalid)d invalid log lines
        %(count_lines_no_site)d requests did not match any known site
        %(count_lines_hostname_skipped)d requests did not match any --hostname
        %(count_lines_skipped_user_agent)d requests done by bots, search engines...
        %(count_lines_static)d requests to static resources (css, js, images, ico, ttf...)
        %(count_lines_skipped_downloads)d requests to file downloads did not match any --download-extensions

Website import summary
----------------------

    %(count_lines_recorded)d requests imported to %(total_sites)d sites
        %(total_sites_existing)d sites already existed
        %(total_sites_created)d sites were created:
%(sites_created)s
    %(total_sites_ignored)d distinct hostnames did not match any existing site:
%(sites_ignored)s