IntegrationTestCase.php

<?php
/**
 * Piwik - Open source web analytics
 *
 * @link http://piwik.org
 * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
 */
require_once PIWIK_INCLUDE_PATH . '/libs/PiwikTracker/PiwikTracker.php';

/**
 * Base class for Integration tests.
 *
 * Provides helpers to track data and then call API get* methods to check outputs automatically.
 *
 */
abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase
{
    /**
     * Identifies the last language used in an API/Controller call.
     *
     * @var string
     */
    protected $lastLanguage;
	
	/**
	 * Creates a config object for use w/ tests.
	 */
	public static function createTestConfig()
	{
        Piwik::createConfigObject();
        Piwik_Config::getInstance()->setTestEnvironment();
	}
	
	/**
	 * Connects to MySQL w/o specifying a database.
	 */
	public static function connectWithoutDatabase()
	{
        $dbConfig = Piwik_Config::getInstance()->database;
        $oldDbName = $dbConfig['dbname'];
        $dbConfig['dbname'] = null;
        
        Piwik::createDatabaseObject($dbConfig);
        
        $dbConfig['dbname'] = $oldDbName;
	}

	public static function setUpBeforeClass()
	{
		self::_setUpBeforeClass();
	}

	// this function
    public static function _setUpBeforeClass( $dbName = false, $createEmptyDatabase = true, $createConfig = true )
    {
        try {
        	Piwik::$piwikUrlCache = '';
        	
        	if ($createConfig)
        	{
	        	self::createTestConfig();
        	}
        	
        	if ($dbName === false) // must be after test config is created
        	{
        		$dbName = Piwik_Config::getInstance()->database['dbname'];
        	}
        	
        	self::connectWithoutDatabase();
            if ($createEmptyDatabase)
            {
                Piwik::dropDatabase();
            }
            Piwik::createDatabase($dbName);
            Piwik::disconnectDatabase();
			
			// reconnect once we're sure the database exists
			Piwik_Config::getInstance()->database['dbname'] = $dbName;
            Piwik::createDatabaseObject();
            
            Piwik::createTables();
            Piwik::createLogObject();

            Piwik_PluginsManager::getInstance()->loadPlugins(array());

        } catch(Exception $e) {
            self::fail("TEST INITIALIZATION FAILED: " .$e->getMessage());
        }

        include "DataFiles/SearchEngines.php";
        include "DataFiles/Languages.php";
        include "DataFiles/Countries.php";
        include "DataFiles/Currencies.php";
        include "DataFiles/LanguageToCountry.php";

        Piwik::createAccessObject();
        Piwik_PostEvent('FrontController.initAuthenticationObject');

        // We need to be SU to create websites for tests
        Piwik::setUserIsSuperUser();
	    Piwik_Common::deleteTrackerCache();

        // Load and install plugins
        $pluginsManager = Piwik_PluginsManager::getInstance();
        $plugins = $pluginsManager->readPluginsDirectory();

        $pluginsManager->loadPlugins( $plugins );
        if ($createEmptyDatabase) // only install if database is empty
        {
	        $pluginsManager->installLoadedPlugins();
        }

        $_GET = $_REQUEST = array();
        $_SERVER['HTTP_REFERER'] = '';

        // Make sure translations are loaded to check messages in English
        Piwik_Translate::getInstance()->loadEnglishTranslation();
        Piwik_LanguagesManager_API::getInstance()->setLanguageForUser('superUserLogin', 'en');

        // List of Modules, or Module.Method that should not be called as part of the XML output compare
        // Usually these modules either return random changing data, or are already tested in specific unit tests.
        self::setApiNotToCall(self::$defaultApiNotToCall);
        self::setApiToCall( array());
    }

	public static function tearDownAfterClass()
	{
		self::_tearDownAfterClass();
	}

    public static function _tearDownAfterClass( $dropDatabase = true )
    {
    	Piwik::$piwikUrlCache = null;
    	
        try {
            $plugins = Piwik_PluginsManager::getInstance()->getLoadedPlugins();
            foreach($plugins AS $plugin) {
            	if ($dropDatabase)
            	{
	                $plugin->uninstall();
                }
            }
            Piwik_PluginsManager::getInstance()->unloadPlugins();
        } catch (Exception $e) {}
        if ($dropDatabase)
        {
            Piwik::dropDatabase();
        }
        Piwik_DataTable_Manager::getInstance()->deleteAll();
        Piwik_Option::getInstance()->clearCache();
        Piwik_Site::clearCache();
        Piwik_Common::deleteTrackerCache();
        Piwik_Config::getInstance()->clear();
        Piwik_TablePartitioning::$tablesAlreadyInstalled = null;
        Piwik_PDFReports_API::$cache = array();
        Zend_Registry::_unsetInstance();

        $_GET = $_REQUEST = array();
        Piwik_Translate::getInstance()->unloadEnglishTranslation();

        // re-enable tag cloud shuffling
        Piwik_Visualization_Cloud::$debugDisableShuffle = true;
    }

    public function setUp()
    {
	    // Make sure the browser running the test does not influence the Country detection code
	    $_SERVER['HTTP_ACCEPT_LANGUAGE'] = 'en';

        $this->changeLanguage('en');
    }

    protected static $apiToCall = array();
    protected static $apiNotToCall = array();

    public static $defaultApiNotToCall = array(
        'LanguagesManager',
        'DBStats',
        'UsersManager',
        'SitesManager',
        'ExampleUI',
	    'Overlay',
        'Live',
        'SEO',
        'ExampleAPI',
        'PDFReports',
        'MobileMessaging',
        'Transitions',
        'API',
        'ImageGraph',
        'Annotations',
        'UserCountry.getLocationFromIP',
    );

    const DEFAULT_USER_PASSWORD = 'nopass';
    
    protected $missingExpectedFiles = array();
    protected $comparisonFailures = array();

    /**
     * Forces the test to only call and fetch XML for the specified plugins,
     * or exact API methods.
     * If not called, all default tests will be executed.
     *
     * @param array $apiToCall array( 'ExampleAPI', 'Plugin.getData' )
     *
     * @throws Exception
     * @return void
     */
    protected static function setApiToCall( $apiToCall )
    {
        if(func_num_args() != 1)
        {
            throw new Exception('setApiToCall expects an array');
        }
        if(!is_array($apiToCall))
        {
            $apiToCall = array($apiToCall);
        }
        self::$apiToCall = $apiToCall;
    }

    /**
     * Sets a list of API methods to not call during the test
     *
     * @param string $apiNotToCall eg. 'ExampleAPI.getPiwikVersion'
     *
     * @return void
     */
    protected static function setApiNotToCall( $apiNotToCall )
    {
        if(!is_array($apiNotToCall))
        {
            $apiNotToCall = array($apiNotToCall);
        }
        self::$apiNotToCall = $apiNotToCall;
    }

    /**
     * Returns a PiwikTracker object that you can then use to track pages or goals.
     *
     * @param         $idSite
     * @param         $dateTime
     * @param boolean $defaultInit If set to true, the tracker object will have default IP, user agent, time, resolution, etc.
     *
     * @return PiwikTracker
     */
    public static function getTracker($idSite, $dateTime, $defaultInit = true, $useLocal = false )
    {
    	if ($useLocal)
    	{
			require_once PIWIK_INCLUDE_PATH . '/tests/LocalTracker.php';
    		$t = new Piwik_LocalTracker($idSite, self::getTrackerUrl());
    	}
    	else
    	{
	        $t = new PiwikTracker( $idSite, self::getTrackerUrl());
        }
        $t->setForceVisitDateTime($dateTime);

        if($defaultInit)
        {
            $t->setIp('156.5.3.2');

            // Optional tracking
            $t->setUserAgent( "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.2.6) Gecko/20100625 Firefox/3.6.6 (.NET CLR 3.5.30729)");
            $t->setBrowserLanguage('fr');
            $t->setLocalTime( '12:34:06' );
            $t->setResolution( 1024, 768 );
            $t->setBrowserHasCookies(true);
            $t->setPlugins($flash = true, $java = true, $director = false);
        }
        return $t;
    }

    /**
     * Creates a website, then sets its creation date to a day earlier than specified dateTime
     * Useful to create a website now, but force data to be archived back in the past.
     *
     * @param string  $dateTime eg '2010-01-01 12:34:56'
     * @param int     $ecommerce
     * @param string  $siteName
     *
     * @return int    idSite of website created
     */
    public static function createWebsite( $dateTime, $ecommerce = 0, $siteName = 'Piwik test', $siteUrl = false, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null )
    {
        $idSite = Piwik_SitesManager_API::getInstance()->addSite(
            $siteName,
            $siteUrl === false ? "http://piwik.net/" : $siteUrl,
            $ecommerce,
	        $siteSearch , $searchKeywordParameters, $searchCategoryParameters,
            $ips = null,
            $excludedQueryParameters = null,
            $timezone = null,
            $currency = null
        );

        // Manually set the website creation date to a day earlier than the earliest day we record stats for
        Zend_Registry::get('db')->update(Piwik_Common::prefixTable("site"),
            array('ts_created' => Piwik_Date::factory($dateTime)->subDay(1)->getDatetime()),
            "idsite = $idSite"
        );

        // Clear the memory Website cache
        Piwik_Site::clearCache();

        return $idSite;
    }

	/**
	 * Create one MAIL and two MOBILE scheduled reports
	 *
	 * Reports sent by mail can contain PNG graphs when the user specifies it.
	 * Depending on the system under test, generated images differ slightly.
	 * Because of this discrepancy, PNG graphs are only tested if the system under test
	 * has the characteristics described in 'canImagesBeIncludedInScheduledReports'
	 *
	 * @see canImagesBeIncludedInScheduledReports
	 * @param int $idSite id of website created
	 */
	protected static function setUpScheduledReports($idSite)
	{
		// fake access is needed so API methods can call Piwik::getCurrentUserLogin(), e.g: 'PDFReports.addReport'
		$pseudoMockAccess = new FakeAccess;
		FakeAccess::$superUser = true;
		Zend_Registry::set('access', $pseudoMockAccess);

		// retrieve available reports
		$availableReportMetadata = Piwik_PDFReports_API::getReportMetadata($idSite, Piwik_PDFReports::EMAIL_TYPE);

		$availableReportIds = array();
		foreach($availableReportMetadata as $reportMetadata)
		{
			$availableReportIds[] = $reportMetadata['uniqueId'];
		}

		//@review should we also test evolution graphs?
		// set-up mail report
		Piwik_PDFReports_API::getInstance()->addReport(
			$idSite,
			'Mail Test report',
			'day', // overridden in getApiForTestingScheduledReports()
			0,
			Piwik_PDFReports::EMAIL_TYPE,
			Piwik_ReportRenderer::HTML_FORMAT, // overridden in getApiForTestingScheduledReports()
			$availableReportIds,
			array("displayFormat" => Piwik_PDFReports::DISPLAY_FORMAT_TABLES_ONLY)
		);

		// set-up sms report for one website
		Piwik_PDFReports_API::getInstance()->addReport(
			$idSite,
			'SMS Test report, one website',
			'day', // overridden in getApiForTestingScheduledReports()
			0,
			Piwik_MobileMessaging::MOBILE_TYPE,
			Piwik_MobileMessaging::SMS_FORMAT,
			array("MultiSites_getOne"),
			array("phoneNumbers"=>array())
		);

		// set-up sms report for all websites
		Piwik_PDFReports_API::getInstance()->addReport(
			$idSite,
			'SMS Test report, all websites',
			'day', // overridden in getApiForTestingScheduledReports()
			0,
			Piwik_MobileMessaging::MOBILE_TYPE,
			Piwik_MobileMessaging::SMS_FORMAT,
			array("MultiSites_getAll"),
			array("phoneNumbers"=>array())
		);

		if(self::canImagesBeIncludedInScheduledReports())
		{
			// set-up mail report with images
			Piwik_PDFReports_API::getInstance()->addReport(
				$idSite,
				'Mail Test report',
				'day', // overridden in getApiForTestingScheduledReports()
				0,
				Piwik_PDFReports::EMAIL_TYPE,
				Piwik_ReportRenderer::HTML_FORMAT, // overridden in getApiForTestingScheduledReports()
				$availableReportIds,
				array("displayFormat" => Piwik_PDFReports::DISPLAY_FORMAT_TABLES_AND_GRAPHS)
			);
		}
	}

	protected function alertWhenImagesExcludedFromTests()
	{
		if(!self::canImagesBeIncludedInScheduledReports())
		{
			$this->markTestSkipped(
			'Do take note that scheduled reports are not being tested with images. ' .
			'If images contained in scheduled reports have been altered, tests will fail on the Piwik QA Server. ' .
			'To include images in the test suite, please use a machine with the following specifications : OS = Linux precise32, PHP Version = 5.3.10 and GD Version = 2.0'.
			'Ignore this message if you\'re running on your dev machine, but pay attention when it comes from Jenkins.'
			);
		}
	}

	/**
	 * Return true if system under test has the following characteristics :
	 *  - php_uname() contains 'precise32' or 'ubuntu'
	 *  - phpversion() contains '5.3.10'
	 *  - 'GD Version' equals '2.0'
	 */
	private static function canImagesBeIncludedInScheduledReports()
	{
		$gdInfo = gd_info();
		return
			(stristr(php_uname(), 'precise32') || stristr(php_uname(), 'ubuntu')) &&
			stristr(phpversion(), '5.3.10') &&
			$gdInfo['GD Version'] == '2.0';
	}

	/**
	 * Return 4 Api Urls for testing scheduled reports :
	 * - one in HTML format with all available reports
	 * - one in PDF format with all available reports
	 * - two in SMS (one for each available report: MultiSites.getOne & MultiSites.getAll)
	 *
	 * @param string $dateTime eg '2010-01-01 12:34:56'
	 * @param string $period eg 'day', 'week', 'month', 'year'
	 */
	protected static function getApiForTestingScheduledReports($dateTime, $period)
	{
		$apiCalls = array();

		// HTML Scheduled Report
		array_push(
			$apiCalls,
			array(
				'PDFReports.generateReport',
				array(
					'testSuffix' => '_scheduled_report_in_html_tables_only',
					'date' => $dateTime,
					'periods' => array($period),
					'format' => 'original',
					'fileExtension' => 'html',
					'otherRequestParameters' => array(
						'idReport' => 1,
						'reportFormat' => Piwik_ReportRenderer::HTML_FORMAT,
						'outputType' => Piwik_PDFReports_API::OUTPUT_RETURN
					)
				)
			)
		);

		// PDF Scheduled Report
		array_push(
			$apiCalls,
			array(
				'PDFReports.generateReport',
				array(
					'testSuffix' => '_scheduled_report_in_pdf_tables_only',
					'date' => $dateTime,
					'periods' => array($period),
					'format' => 'original',
					'fileExtension' => 'pdf',
					'otherRequestParameters' => array(
						'idReport' => 1,
						'reportFormat' => Piwik_ReportRenderer::PDF_FORMAT,
						'outputType' => Piwik_PDFReports_API::OUTPUT_RETURN
					)
				)
			)
		);

		// SMS Scheduled Report, one site
		array_push(
			$apiCalls,
			array(
				'PDFReports.generateReport',
				array(
					'testSuffix' => '_scheduled_report_via_sms_one_site',
					'date' => $dateTime,
					'periods' => array($period),
					'format' => 'original',
					'fileExtension' => 'sms.txt',
					'otherRequestParameters' => array(
						'idReport' => 2,
						'outputType' => Piwik_PDFReports_API::OUTPUT_RETURN
					)
				)
			)
		);

		// SMS Scheduled Report, all sites
		array_push(
			$apiCalls,
			array(
				'PDFReports.generateReport',
				array(
					'testSuffix' => '_scheduled_report_via_sms_all_sites',
					'date' => $dateTime,
					'periods' => array($period),
					'format' => 'original',
					'fileExtension' => 'sms.txt',
					'otherRequestParameters' => array(
						'idReport' => 3,
						'outputType' => Piwik_PDFReports_API::OUTPUT_RETURN
					)
				)
			)
		);

		if(self::canImagesBeIncludedInScheduledReports())
		{
			// HTML Scheduled Report with images
			array_push(
				$apiCalls,
				array(
					'PDFReports.generateReport',
					array(
						'testSuffix' => '_scheduled_report_in_html_tables_and_graph',
						'date' => $dateTime,
						'periods' => array($period),
						'format' => 'original',
						'fileExtension' => 'html',
						'otherRequestParameters' => array(
							'idReport' => 4,
							'reportFormat' => Piwik_ReportRenderer::HTML_FORMAT,
							'outputType' => Piwik_PDFReports_API::OUTPUT_RETURN
						)
					)
				)
			);
		}

		return $apiCalls;
	}

	/**
     * Checks that the response is a GIF image as expected.
     * Will fail the test if the response is not the expected GIF
     *
     * @param $response
     */
    protected static function checkResponse($response)
    {
        $trans_gif_64 = "R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==";
        $expectedResponse = base64_decode($trans_gif_64);
        self::assertEquals($expectedResponse, $response, "Expected GIF beacon, got: <br/>\n"
	        . var_export($response, true)
	        . "\n If you are stuck, you can enable \$GLOBALS['PIWIK_TRACKER_DEBUG']=true; in piwik.php to get more debug info."
//            .base64_encode($response) // uncomment to further debug when the GIF hides the error
        );
    }

	/**
	 * Returns URL to Piwik root.
	 *
	 * @return string
	 */
	public static function getRootUrl()
	{
		$piwikUrl = Piwik_Url::getCurrentUrlWithoutFileName();

		$pathBeforeRoot = 'tests';
		// Running from a plugin
		if(strpos($piwikUrl, 'plugins/') !== false)
		{
			$pathBeforeRoot = 'plugins';
		}

		$testsInPath =  strpos($piwikUrl, $pathBeforeRoot.'/');
		if($testsInPath  !== false) {
			$piwikUrl = substr($piwikUrl, 0, $testsInPath);
		}
		return $piwikUrl;
	}

	/**
	 * Returns URL to the proxy script, used to ensure piwik.php
	 * uses the test environment, and allows variable overwriting
	 *
	 * @return string
	 */
	public static function getTrackerUrl()
	{
		return self::getRootUrl().'tests/PHPUnit/proxy/piwik.php';
	}

	/**
	 * Returns the super user token auth that can be used in tests. Can be used to
	 * do bulk tracking.
	 *
	 * @return string
	 */
	public static function getTokenAuth()
	{
	    return Piwik_UsersManager_API::getInstance()->getTokenAuth(
	    	Piwik_Config::getInstance()->superuser['login'],
	    	Piwik_Config::getInstance()->superuser['password']
    	);
    }

    /**
     * Given a list of default parameters to set, returns the URLs of APIs to call
     * If any API was specified in setApiToCall() we ensure only these are tested.
     * If any API is set as excluded (see list below) then it will be ignored.
     *
     * @param array       $parametersToSet Parameters to set in api call
     * @param array       $formats         Array of 'format' to fetch from API
     * @param array       $periods         Array of 'period' to query API
     * @param bool        $supertableApi
     * @param bool        $setDateLastN    If set to true, the 'date' parameter will be rewritten to query instead a range of dates, rather than one period only.
     * @param bool|string $language        2 letter language code, defaults to default piwik language
     * @param bool|string $segment
     * @param bool|string $fileExtension
     *
     * @throws Exception
     * @return array of API URLs query strings
     */
    protected function generateUrlsApi( $parametersToSet, $formats, $periods, $supertableApi = false, $setDateLastN = false, $language = false, $segment = false, $fileExtension = false )
    {
        // Get the URLs to query against the API for all functions starting with get*
        $skipped = $requestUrls = array();
        $apiMetadata = new Piwik_API_DocumentationGenerator;
        foreach(Piwik_API_Proxy::getInstance()->getMetadata() as $class => $info)
        {
            $moduleName = Piwik_API_Proxy::getInstance()->getModuleNameFromClassName($class);
            foreach($info as $methodName => $infoMethod)
            {
                $apiId = $moduleName.'.'.$methodName;

                // If Api to test were set, we only test these
                if(!empty(self::$apiToCall)
                    && in_array($moduleName, self::$apiToCall) === false
                    && in_array($apiId, self::$apiToCall) === false)
                {
//	                echo "Skipped $apiId... \n";
                    $skipped[] = $apiId;
	                continue;
                }
	            // Excluded modules from test
                elseif(
                    ((strpos($methodName, 'get') !== 0 && $methodName != 'generateReport')
                        || in_array($moduleName, self::$apiNotToCall) === true
                        || in_array($apiId, self::$apiNotToCall) === true
                        || $methodName == 'getLogoUrl'
                        || $methodName == 'getHeaderLogoUrl'
                    )
                )
                {
//	                echo "Skipped $apiId... \n";
                    $skipped[] = $apiId;
                    continue;
                }

                foreach($periods as $period)
                {
                    $parametersToSet['period'] = $period;

                    // If date must be a date range, we process this date range by adding 6 periods to it
                    if($setDateLastN)
                    {
                        if(!isset($parametersToSet['dateRewriteBackup']))
                        {
                            $parametersToSet['dateRewriteBackup'] = $parametersToSet['date'];
                        }

                        $lastCount = (int)$setDateLastN;
                        if($setDateLastN === true)
                        {
                            $lastCount = 6;
                        }
                        $firstDate = $parametersToSet['dateRewriteBackup'];
                        $secondDate = date('Y-m-d', strtotime("+$lastCount " . $period . "s", strtotime($firstDate)));
                        $parametersToSet['date'] = $firstDate . ',' . $secondDate;
                    }

                    // Set response language
                    if($language !== false)
                    {
                        $parametersToSet['language'] = $language;
                    }

                    // set idSubtable if subtable API is set
                    if ($supertableApi !== false)
                    {
                    	$request = new Piwik_API_Request(array(
                    		'module' => 'API',
                    		'method' => $supertableApi,
                    		'idSite' => $parametersToSet['idSite'],
                    		'period' => $parametersToSet['period'],
                    		'date' => $parametersToSet['date'],
                    		'format' => 'php',
                    		'serialize' => 0,
                    	));

                    	// find first row w/ subtable
                    	foreach ($request->process() as $row)
                    	{
                    		if (isset($row['idsubdatatable']))
                    		{
                    			$parametersToSet['idSubtable'] = $row['idsubdatatable'];
                    			break;
                    		}
                    	}

                    	// if no subtable found, throw
                    	if (!isset($parametersToSet['idSubtable']))
                    	{
	                    	throw new Exception(
	                    		"Cannot find subtable to load for $apiId in $supertableApi.");
                    	}
                    }

                    // Generate for each specified format
                    foreach($formats as $format)
                    {
                        $parametersToSet['format'] = $format;
                        $parametersToSet['hideIdSubDatable'] = 1;
                        $parametersToSet['serialize'] = 1;

                        $exampleUrl = $apiMetadata->getExampleUrl($class, $methodName, $parametersToSet);
                        if($exampleUrl === false)
                        {
                            $skipped[] = $apiId;
                            continue;
                        }

                        // Remove the first ? in the query string
                        $exampleUrl = substr($exampleUrl, 1);
                        $apiRequestId = $apiId;
                        if(strpos($exampleUrl, 'period=') !== false)
                        {
                            $apiRequestId .= '_' . $period;
                        }

                        $apiRequestId .= '.' . $format;

						if($fileExtension)
						{
							$apiRequestId .= '.' . $fileExtension;
						}

                        $requestUrls[$apiRequestId] = $exampleUrl;
                    }
                }
            }
        }
        return $requestUrls;
    }

    /**
     * Will return all api urls for the given data
     *
     * @param string|array      $formats        String or array of formats to fetch from API
     * @param int|bool          $idSite         Id site
     * @param string|bool       $dateTime       Date time string of reports to request
     * @param array|bool|string $periods        String or array of strings of periods (day, week, month, year)
     * @param bool              $setDateLastN   When set to true, 'date' parameter passed to API request will be rewritten to query a range of dates rather than 1 date only
     * @param string|bool       $language       2 letter language code to request data in
     * @param string|bool       $segment        Custom Segment to query the data  for
     * @param string|bool       $visitorId      Only used for Live! API testing
     * @param bool              $abandonedCarts Only used in Goals API testing
     * @param bool              $idGoal
     * @param bool              $apiModule
     * @param bool              $apiAction
     * @param array             $otherRequestParameters
     * @param array|bool        $supertableApi
     * @param array|bool        $fileExtension
     *
     * @return array
     */
    protected function _generateApiUrls($formats = 'xml', $idSite = false, $dateTime = false, $periods = false,
										 $setDateLastN = false, $language = false, $segment = false, $visitorId = false,
										 $abandonedCarts = false, $idGoal = false, $apiModule = false, $apiAction = false,
										 $otherRequestParameters = array(), $supertableApi = false, $fileExtension = false)
    {
        list($pathProcessed, $pathExpected) = $this->getProcessedAndExpectedDirs();

        if($periods === false)
        {
            $periods = 'day';
        }
        if(!is_array($periods))
        {
            $periods = array($periods);
        }
        if(!is_array($formats))
        {
            $formats = array($formats);
        }
        if(!is_writable($pathProcessed))
        {
            $this->fail('To run the tests, you need to give write permissions to the following directory (create it if it doesn\'t exist).<code><br/>mkdir '. $pathProcessed.'<br/>chmod 777 '.$pathProcessed.'</code><br/>');
        }
        $parametersToSet = array(
            'idSite'           => $idSite,
            'date'             => $periods == array('range') ? $dateTime : date('Y-m-d', strtotime($dateTime)),
            'expanded'         => '1',
            'piwikUrl'         => 'http://example.org/piwik/',
            // Used in getKeywordsForPageUrl
            'url'              => 'http://example.org/store/purchase.htm',

            // Used in Actions.getPageUrl, .getDownload, etc.
            // tied to Main.test.php doTest_oneVisitorTwoVisits
            // will need refactoring when these same API functions are tested in a new function
            'downloadUrl'      => 'http://piwik.org/path/again/latest.zip?phpsessid=this is ignored when searching',
            'outlinkUrl'       => 'http://dev.piwik.org/svn',
            'pageUrl'          => 'http://example.org/index.htm?sessionid=this is also ignored by default',
            'pageName'         => ' Checkout / Purchasing... ',

            // do not show the millisec timer in response or tests would always fail as value is changing
            'showTimer'        => 0,

            'language'         => $language ? $language : 'en',
            'abandonedCarts'   => $abandonedCarts ? 1 : 0,
            'idSites'          => $idSite,
        );
        $parametersToSet = array_merge($parametersToSet, $otherRequestParameters);
        if(!empty($visitorId ))
        {
            $parametersToSet['visitorId'] = $visitorId;
        }
        if(!empty($apiModule ))
        {
            $parametersToSet['apiModule'] = $apiModule;
        }
        if(!empty($apiAction))
        {
            $parametersToSet['apiAction'] = $apiAction;
        }
        if(!empty($segment))
        {
            $parametersToSet['segment'] = $segment;
        }
        if($idGoal !== false)
        {
            $parametersToSet['idGoal'] = $idGoal;
        }

        $requestUrls = $this->generateUrlsApi($parametersToSet, $formats, $periods, $supertableApi, $setDateLastN, $language, $segment, $fileExtension);
        return $requestUrls;
    }

    protected function _testApiUrl($testName, $apiId, $requestUrl)
    {
        $isLiveMustDeleteDates = strpos($requestUrl, 'Live.getLastVisits') !== false;
        $request               = new Piwik_API_Request($requestUrl);
        $dateTime             = Piwik_Common::getRequestVar('date', '', 'string', Piwik_Common::getArrayFromQueryString($requestUrl));

        list($processedFilePath, $expectedFilePath) = $this->getProcessedAndExpectedPaths($testName, $apiId);

        // Cast as string is important. For example when calling
        // with format=original, objects or php arrays can be returned.
        // we also hide errors to prevent the 'headers already sent' in the ResponseBuilder (which sends Excel headers multiple times eg.)
        $response = (string)$request->process();

        if ($isLiveMustDeleteDates) {
            $response = $this->removeAllLiveDatesFromXml($response);
        }

		// normalize date markups and document ID in pdf files :
		// - /LastModified (D:20120820204023+00'00')
		// - /CreationDate (D:20120820202226+00'00')
		// - /ModDate (D:20120820202226+00'00')
		// - /M (D:20120820202226+00'00')
		// - /ID [ <0f5cc387dc28c0e13e682197f485fe65> <0f5cc387dc28c0e13e682197f485fe65> ]
		$response = preg_replace('/\(D:[0-9]{14}/', '(D:19700101000000', $response);
		$response = preg_replace('/\/ID \[ <.*> ]/', '', $response);

        file_put_contents($processedFilePath, $response);

        $expected = $this->loadExpectedFile($expectedFilePath);
        if (empty($expected)) {
	        print("The expected file is not found at '$expectedFilePath'. The Processed response was:");
	        print("\n----------------------------\n\n");
	        var_dump($response);
	        print("\n----------------------------\n");
	        return;
        }

        // @todo This should not vary between systems AFAIK... "idsubdatatable can differ"
        $expected = $this->removeXmlElement($expected, 'idsubdatatable', $testNotSmallAfter = false);
        $response = $this->removeXmlElement($response, 'idsubdatatable', $testNotSmallAfter = false);

        if ($isLiveMustDeleteDates) {
            $expected = $this->removeAllLiveDatesFromXml($expected);
        } // If date=lastN the <prettyDate> element will change each day, we remove XML element before comparison
        elseif (strpos($dateTime, 'last') !== false
            || strpos($dateTime, 'today') !== false
            || strpos($dateTime, 'now') !== false
        ) {
            if (strpos($requestUrl, 'API.getProcessedReport') !== false) {
                $expected = $this->removePrettyDateFromXml($expected);
                $response = $this->removePrettyDateFromXml($response);
            }
            // avoid build failure when running just before midnight, generating visits in the future
            $expected = $this->removeXmlElement($expected, 'sum_daily_nb_uniq_visitors');
            $response = $this->removeXmlElement($response, 'sum_daily_nb_uniq_visitors');
            $expected = $this->removeXmlElement($expected, 'nb_visits_converted');
            $response = $this->removeXmlElement($response, 'nb_visits_converted');
            
            
             if (strpos($requestUrl, 'date=') !== false)
            {
            	$regex = "/date=[-0-9,%Ca-z]+/"; // need to remove %2C which is encoded ,
            	$expected = preg_replace($regex, 'date=', $expected);
            	$response = preg_replace($regex, 'date=', $response);
            } 
        }

        // if idSubtable is in request URL, make sure idSubtable values are not in any urls
        if (strpos($requestUrl, 'idSubtable=') !== false)
        {
        	$regex = "/idSubtable=[0-9]+/";
        	$expected = preg_replace($regex, 'idSubtable=', $expected);
        	$response = preg_replace($regex, 'idSubtable=', $response);
        }

        // is there a better way to test for the current DB type in use?
        if (Zend_Registry::get('db') instanceof Piwik_Db_Adapter_Mysqli) {
            // Do not test for TRUNCATE(SUM()) returning .00 on mysqli since this is not working
            // http://bugs.php.net/bug.php?id=54508
            $expected = str_replace('.000000</l', '</l', $expected); //lat/long
	        $response = str_replace('.000000</l', '</l', $response); //lat/long
	        $expected = str_replace('.00</revenue>', '</revenue>', $expected);
            $response = str_replace('.00</revenue>', '</revenue>', $response);
	        $response = str_replace('.1</revenue>', '</revenue>', $response);
	        $expected = str_replace('.1</revenue>', '</revenue>', $expected);
	        $expected = str_replace('.11</revenue>', '</revenue>', $expected);
	        $response = str_replace('.11</revenue>', '</revenue>', $response);
        }

		try
		{
		    if (strpos($requestUrl, 'format=xml') !== false) {
		        $this->assertXmlStringEqualsXmlString($expected, $response, "Differences with expected in: $processedFilePath");
		    } else {
		        $this->assertEquals(strlen($expected), strlen($response), "Differences with expected in: $processedFilePath");
		        $this->assertEquals($expected, $response, "Differences with expected in: $processedFilePath");
		    }

			if (trim($response) == trim($expected)) {
				file_put_contents($processedFilePath, $response);
			}
		}
	    catch (Exception $ex)
	    {
	    	$this->comparisonFailures[] = $ex;
	    }
    }

    protected function removeAllLiveDatesFromXml($input)
    {
        $toRemove = array(
            'serverDate',
            'firstActionTimestamp',
            'lastActionTimestamp',
            'lastActionDateTime',
            'serverTimestamp',
            'serverTimePretty',
            'serverDatePretty',
            'serverDatePrettyFirstAction',
            'serverTimePrettyFirstAction',
            'goalTimePretty',
            'serverTimePretty',
            'visitorId'
        );
        foreach($toRemove as $xml) {
            $input = $this->removeXmlElement($input, $xml);
        }
        return $input;
    }

    protected function removePrettyDateFromXml($input)
    {
        return $this->removeXmlElement($input, 'prettyDate');
    }

    protected function removeXmlElement($input, $xmlElement, $testNotSmallAfter = true)
    {
    	// Only raise error if there was some data before
    	$testNotSmallAfter = strlen($input > 100 ) && $testNotSmallAfter;
    	
        $input = preg_replace('/(<'.$xmlElement.'>.+?<\/'.$xmlElement.'>)/', '', $input);
        //check we didn't delete the whole string
        if($testNotSmallAfter)
        {
            $this->assertTrue(strlen($input) > 100);
        }
        return $input;
    }

    private function getProcessedAndExpectedDirs()
    {
        $path = $this->getPathToTestDirectory();
        return array($path . '/processed/', $path . '/expected/');
    }

    private function getProcessedAndExpectedPaths($testName, $testId, $format = null)
    {
        $filename = $testName . '__' . $testId;