import-logs.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
if sys.version_info < (2, 6):
    print >> sys.stderr, 'You must use Python 2.6 or greater'
    sys.exit(1)


import base64
import ConfigParser
import datetime
import fnmatch
import gzip
import hashlib
import inspect
import itertools
import json
import logging
import optparse
import os
import os.path
import Queue
import re
import threading
import time
import urllib
import urllib2



##
## Constants.
##

_COMMON_LOG_FORMAT = (
    '(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?)\] '
    '"\S+ (?P<path>.*?) \S+" (?P<status>\S+) (?P<length>\S+)'
)
_NSCA_EXTENDED_LOG_FORMAT = (
    '(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?)\] '
    '"\S+ (?P<path>.*?) \S+" (?P<status>\S+) (?P<length>\S+) '
    '"(?P<referrer>.*?)" "(?P<user_agent>.*?)"'
)
_COMMON_COMPLETE_LOG_FORMAT = (
    '(?P<host>[\w\-\.]*)(?::\d+)? '
    '(?P<ip>\S+) \S+ \S+ \[(?P<date>.*?)\] '
    '"\S+ (?P<path>.*?) \S+" (?P<status>\S+) (?P<length>\S+) '
    '"(?P<referrer>.*?)" "(?P<user_agent>.*?)"'
)

FORMATS = {
    'common': _COMMON_LOG_FORMAT,
    'common_vhost': '(?P<host>[\w\-\.]*)(?::\d+)? ' + _COMMON_LOG_FORMAT,
    'nsca_extended': _NSCA_EXTENDED_LOG_FORMAT,
    'common_complete': _COMMON_COMPLETE_LOG_FORMAT,
}

DATE_FORMAT = '%d/%b/%Y:%H:%M:%S'


EXCLUDED_EXTENSIONS = (
    # Images
    '.gif', '.jpg', '.jpeg', '.png', '.bmp', '.ico', '.svg',
    # Fonts
    '.ttf', '.eot', '.woff',
    # Plugins
    '.class', '.swf',
    # Misc
    '.css', '.js', '.xml', 'robots.txt',
)

DOWNLOAD_EXTENSIONS = (
    '7z aac arc arj asf asx avi bin csv deb dmg doc exe flv gif gz gzip hqx '
    'jar jpg jpeg js mpg mp2 mp3 mp4 mpeg mov movie msi msp odb odf odg odp '
    'ods odt ogg ogv pdf phps png ppt qt qtm ra ram rar rpm sea sit tar tbz '
    'bz2 tbz tgz torrent txt wav wma wmv wpd xls xml z zip'
).split()

# A good source is: http://phpbb-bots.blogspot.com/
EXCLUDED_USER_AGENTS = (
    'AdsBot-Google',
    'ia_archiver',
    'Scooter/',
    'Ask Jeeves',
    'Baiduspider+(',
    'Exabot',
    'Googlebot',
    'Mediapartners-Google',
    'msnbot',
    'Sosospider+',
    'SurveyBot',
    'Twiceler',
    'VoilaBot',
    'Yahoo',
    'Yandex',
)


PIWIK_MAX_ATTEMPTS = 3
PIWIK_DELAY_AFTER_FAILURE = 2

PIWIK_EXPECTED_IMAGE = base64.b64decode(
    'R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=='
)




##
## Code.
##


class Configuration(object):
    """
    Stores all the configuration options by reading sys.argv and parsing,
    if needed, the config.inc.php.

    It has 2 attributes: options and filenames.
    """

    class Error(Exception):
        pass

    def __init__(self):
        option_parser = optparse.OptionParser(
            usage='Usage: %prog [options] log_file [ log_file [...] ]',
            description="Import HTTP access logs to Piwik."
        )
        option_parser.add_option(
            '-d', '--debug', dest='debug', action='count', default=0,
            help="Enable debug output (specify multiple times for more verbose)",
        )
        option_parser.add_option(
            '-n', '--dry-run', dest='dry_run',
            action='store_true', default=False,
            help="Perform a trial run with nothing being really inserted into Piwik",
        )
        option_parser.add_option(
            '-u', '--url', dest='piwik_url',
            help="Piwik install URL",
        )
        default_config = os.path.abspath(
            os.path.join(os.path.dirname(__file__),
            '../../config/config.ini.php'),
        )
        option_parser.add_option(
            '-c', '--config', dest='config_file', default=default_config,
            help=(
                "Piwik configuration file (default: %default). This file is used to "
                "get superuser credentials unless they are themselves given as "
                "options"
            )
        )
        option_parser.add_option(
            '-l', '--login', dest='login',
            help="Piwik superuser login"
        )
        option_parser.add_option(
            '-p', '--password', dest='password',
            help="Piwik superuser password"
        )
        option_parser.add_option(
            '-t', '--token-auth', dest='piwik_token_auth',
            help="Piwik token_auth",
        )
        option_parser.add_option(
            '-f', '--format', dest='format', default=None,
            help="Access log format. If not specified, the format will be autodetected",
        )
        option_parser.add_option(
            '-i', '--idsite', dest='site_id',
            help="Piwik site ID to use",
        )
        option_parser.add_option(
            '--idsite-fallback', dest='site_id_fallback',
            help="Default Piwik site ID to use if the hostname doesn't match any "
            "known Piwik URL",
        )
        option_parser.add_option(
            '--hostnames', dest='hostnames', action='append',
            help="Accepted hostnames (others will be excluded)"
        )
        option_parser.add_option(
            '-s', '--skip', dest='skip', default=0, type='int',
            help="Skip the n first lines",
        )
        option_parser.add_option(
            '-r', '--recorders', dest='recorders', default=1, type='int',
            help="Number of simultaneous recorders (default: %default)",
        )
        option_parser.add_option(
            '--add-sites-new-hosts', dest='add_sites_new_hosts',
            action='store_true', default=False,
            help="When a hostname is found in the log file, but not matched to any website "
            "in Piwik, automatically create a new website in Piwik with this hostname to "
            "import the logs"
        )
        option_parser.add_option(