import_logs.py

            if hit.path.lower().endswith(extension):
                stats.count_lines_downloads.increment()
                hit.is_download = True
        return True

    def check_user_agent(self, hit):
        user_agent = hit.user_agent.lower()
        for s in itertools.chain(EXCLUDED_USER_AGENTS, config.options.excluded_useragents):
            if s in user_agent:
                if config.options.enable_bots:
                    hit.is_robot = True
                    return True
                else:
                    stats.count_lines_skipped_user_agent.increment()
                    return False
        return True

    def check_http_error(self, hit):
        if hit.status.startswith('4') or hit.status.startswith('5'):
            if config.options.enable_http_errors:
                hit.is_error = True
                return True
            else:
                stats.count_lines_skipped_http_errors.increment()
                return False
        return True

    def check_http_redirect(self, hit):
        if hit.status.startswith('3') and hit.status != '304':
            if config.options.enable_http_redirects:
                hit.is_redirect = True
                return True
            else:
                stats.count_lines_skipped_http_redirects.increment()
                return False
        return True

    def check_path(self, hit):
        for excluded_path in config.options.excluded_paths:
            if fnmatch.fnmatch(hit.path, excluded_path):
                return False
        return True


    @staticmethod
    def detect_format(line):
        """
        Return the format matching this line, or None if none was found.
        """
        logging.debug('Detecting the log format...')
        for name, format in FORMATS.iteritems():
            if re.match(format, line):
                logging.debug('Format %s matches', name)
                return name
            else:
                logging.debug('Format %s does not match', name)

    def parse(self, filename):
        """
        Parse the specified filename and insert hits in the queue.
        """
        def invalid_line(line):
            stats.count_lines_invalid.increment()
            if config.options.debug >= 2:
                logging.debug('Invalid line detected: ' + line)

        if filename == '-':
            filename = '(stdin)'
            file = sys.stdin
        else:
            if not os.path.exists(filename):
                print >> sys.stderr, 'File %s does not exist' % filename
                return
            else:
                if filename.endswith('.bz2'):
                    open_func = bz2.BZ2File
                elif filename.endswith('.gz'):
                    open_func = gzip.open
                else:
                    open_func = open
                file = open_func(filename, 'r')

        if config.options.show_progress:
            print 'Parsing log %s...' % filename

        for lineno, line in enumerate(file):
            line = line.decode(config.options.encoding)
            # Guess the format if needed.
            if not config.format_regexp:
                logging.debug('Guessing the log format...')
                format_name = self.detect_format(line)
                if not format_name:
                    return fatal_error(
                        'Cannot guess the logs format. Please give one using '
                        'either the --log-format-name or --log-format-regex option'
                    )
                format = FORMATS[format_name]
                config.format = format
                config.format_regexp = re.compile(format)
                # Make sure the format is compatible with the resolver.
                resolver.check_format(format)

            stats.count_lines_parsed.increment()
            if stats.count_lines_parsed.value <= config.options.skip:
                continue

            match = config.format_regexp.match(line)
            if not match:
                invalid_line(line)
                continue

            hit = Hit(
                filename=filename,
                lineno=lineno,
                status=match.group('status'),
                full_path=match.group('path'),
                is_download=False,
                is_robot=False,
                is_error=False,
                is_redirect=False,
            )

            # Strip query string
            if config.options.strip_query_string:
                hit.path = hit.full_path.split(config.options.query_string_delimiter, 1)[0]
            else:
                hit.path = hit.full_path

            # Parse date _with_ timezone to get an UTC timestamp.
            date_string = match.group('date')
            try:
                tz = float(date_string[-5:])
                hit.date = datetime.datetime.strptime(date_string[:-6], '%d/%b/%Y:%H:%M:%S')
            except ValueError:
                # Date format is incorrect, the line is probably badly formatted.
                invalid_line(line)
                continue
            hit.date -= datetime.timedelta(hours=tz/100)

            try:
                hit.referrer = match.group('referrer')
            except IndexError:
                hit.referrer = ''
            if hit.referrer == '-':
                hit.referrer = ''

            try:
                hit.user_agent = match.group('user_agent')
            except IndexError:
                hit.user_agent = ''

            hit.ip = match.group('ip')
            try:
                hit.length = int(match.group('length'))
            except ValueError:
                # Not all lines have a length (e.g. 304 redirects)
                hit.length = 0
            try:
                hit.host = match.group('host')
            except IndexError:
                # Some formats have no host.
                pass

            # Check if the hit must be excluded.
            check_methods = inspect.getmembers(self, predicate=inspect.ismethod)
            if all((method(hit) for name, method in check_methods if name.startswith('check_'))):
                Recorder.add_hit(hit)




def main():
    """
    Start the importing process.
    """
    if config.options.show_progress:
        stats.start_monitor()

    stats.set_time_start()

    recorders = Recorder.launch(config.options.recorders)

    try:
        for filename in config.filenames:
            parser.parse(filename)

        Recorder.wait_empty()
    except KeyboardInterrupt:
        pass

    stats.set_time_stop()

    if config.options.show_progress:
        stats.stop_monitor()

    try:
        Recorder.invalidate_reports()
    except Piwik.Error, e:
        pass
    stats.print_summary()



def fatal_error(error, filename=None, lineno=None):
    print >> sys.stderr, 'Fatal error: %s' % error
    if filename and lineno is not None:
        print >> sys.stderr, (
            'You can restart the import of "%s" from the point it failed by '
            'specifying --skip=%d on the command line.\n' % (filename, lineno)
        )
    os._exit(1)


if __name__ == '__main__':
    try:
        piwik = Piwik()
        config = Configuration()
        stats = Statistics()
        resolver = config.get_resolver()
        parser = Parser()
        main()
        sys.exit(0)
    except KeyboardInterrupt:
        pass