Fixes #2902

 * removed ip_address_pre_mask_length setting -- now there is only one setting which anonymises after testing for IP exclusion
  * Reverted the "synchronize" from [5772] since not needed anymore
 * Added integration test to test that IP is indeed anonymized 
 * Added integration test to test that IP is anonymized AFTER testing for ip exclusion 


git-svn-id: http://dev.piwik.org/svn/trunk@5776 59fd770c-687e-43c8-a1e3-f5a4ff64c105

config/global.ini.php

@@ -339,12 +339,7 @@ campaign_keyword_var_name	= "pk_kwd,piwik_kwd,utm_term"
 ; maximum length of a Page Title or a Page URL recorded in the log_action.name table
 page_maximum_length = 1024;
 
-; Anonymize a visitor's IP address before any tracking heuristics or any plugins can access the visitor's IP address;
-; This value is the number of octets in IP address to mask; if the AnonymizeIP plugin is deactivated, this value is ignored.
-; For IPv4 addresses, valid values are 0..4; for IPv6 addresses, valid values are 0..16
-ip_address_pre_mask_length = 0
-
-; Anonymize a visitor's IP address after tracking heuristics but before storing to DB;
+; Anonymize a visitor's IP address after testing for "Ip exclude"
 ; This value is the number of octets in IP address to mask; if the AnonymizeIP plugin is deactivated, this value is ignored.
 ; For IPv4 addresses, valid values are 0..4; for IPv6 addresses, valid values are 0..16
 ip_address_mask_length = 1

core/Tracker.php

@@ -321,11 +321,13 @@ class Piwik_Tracker
 
 	protected function loadTrackerPlugins()
 	{
+		// Adding &dp=1 will disable the provider plugin, if token_auth is used (used to speed up bulk imports)
 		if(isset($this->request['dp'])
 			&& $this->authenticated) 
 		{
 			Piwik_Tracker::setPluginsNotToLoad(array('Provider'));
 		}
+		
 		try{
 			$pluginsTracker = Piwik_Tracker_Config::getInstance()->Plugins_Tracker;
 			if(is_array($pluginsTracker)

core/Tracker/Config.php

@@ -151,8 +151,16 @@ class Piwik_Tracker_Config
 	/**
 	 * Should only be used in tests/
 	 */
-	public function setTestValue($section, $name, $value)
+	public function setTestValue($section, $name, $value = false)
 	{
-		$this->configUser[$section][$name] = $value;
+		// Case setting the whole section at once
+		if($value === false)
+		{
+			$this->configUser[$section] = $name;
+		}
+		else
+		{
+			$this->configUser[$section][$name] = $value;
+		}
 	}
 }

plugins/AnonymizeIP/AnonymizeIP.php

@@ -38,7 +38,6 @@ class Piwik_AnonymizeIP extends Piwik_Plugin
 	{
 		return array(
 			'Tracker.Visit.setVisitorIp' => 'setVisitorIpAddress',
-			'Tracker.saveVisitorInformation' => 'anonymizeVisitorIpAddress',
 		);
 	}
 
@@ -70,15 +69,7 @@ class Piwik_AnonymizeIP extends Piwik_Plugin
 	function setVisitorIpAddress($notification)
 	{
 		$ip =& $notification->getNotificationObject();
-		$ip = self::applyIPMask($ip, Piwik_Tracker_Config::getInstance()->Tracker['ip_address_pre_mask_length']);
+		$ip = self::applyIPMask($ip, Piwik_Tracker_Config::getInstance()->Tracker['ip_address_mask_length']);
 	}
 
-	/**
-	 * Hook on Tracker.saveVisitorInformation to anonymize visitor IP addresses
-	 */
-	function anonymizeVisitorIpAddress($notification)
-	{
-		$visitorInfo =& $notification->getNotificationObject();
-		$visitorInfo['location_ip'] = self::applyIPMask($visitorInfo['location_ip'], Piwik_Tracker_Config::getInstance()->Tracker['ip_address_mask_length']);
-	}
 }

plugins/PrivacyManager/Controller.php

@@ -23,14 +23,17 @@ class Piwik_PrivacyManager_Controller extends Piwik_Controller_Admin
     public function index()
     {
         Piwik::checkUserIsSuperUser();
-        if ($_SERVER["REQUEST_METHOD"] == "POST") {
-            switch (Piwik_Common::getRequestVar('form')) {
+        if ($_SERVER["REQUEST_METHOD"] == "POST") 
+        {
+            switch (Piwik_Common::getRequestVar('form')) 
+            {
                 case("formMaskLength"):
                     $this->handlePluginState(Piwik_Common::getRequestVar("anonymizeIPEnable", 0));
                     $maskLength = Zend_Registry::get('config')->Tracker;
-                    $maskLength->ip_address_mask_length = $maskLength->ip_address_pre_mask_length = Piwik_Common::getRequestVar("maskLength", 1);
+                    $maskLength->ip_address_mask_length = Piwik_Common::getRequestVar("maskLength", 1);
                     Zend_Registry::get('config')->Tracker = $maskLength->toArray();
                     break;
+                    
                 case("formDeleteSettings"):
                     $deleteLogs = Zend_Registry::get('config')->Deletelogs;
                     $deleteLogs->delete_logs_enable = Piwik_Common::getRequestVar("deleteEnable", 0);
@@ -41,6 +44,7 @@ class Piwik_PrivacyManager_Controller extends Piwik_Controller_Admin
 
                     Zend_Registry::get('config')->Deletelogs = $deleteLogs->toArray();
                     break;
+                    
                 default: //do nothing
                     break;
             }
@@ -53,7 +57,8 @@ class Piwik_PrivacyManager_Controller extends Piwik_Controller_Admin
         Piwik::checkUserHasSomeAdminAccess();
         $view = Piwik_View::factory('privacySettings');
 
-        if (Piwik::isUserIsSuperUser()) {
+        if (Piwik::isUserIsSuperUser()) 
+        {
             $deleteLogs = array();
 
             $view->deleteLogs = $this->getDeleteLogsInfo();
@@ -61,7 +66,8 @@ class Piwik_PrivacyManager_Controller extends Piwik_Controller_Admin
         }
         $view->language = Piwik_LanguagesManager::getLanguageCodeForCurrentUser();
 
-        if (!Zend_Registry::get('config')->isFileWritable()) {
+        if (!Zend_Registry::get('config')->isFileWritable()) 
+        {
             $view->configFileNotWritable = true;
         }
 
@@ -81,17 +87,6 @@ class Piwik_PrivacyManager_Controller extends Piwik_Controller_Admin
         $anonymizeIP["name"] = self::ANONYMIZE_IP_PLUGIN_NAME;
         $anonymizeIP["enabled"] = Piwik_PluginsManager::getInstance()->isPluginActivated(self::ANONYMIZE_IP_PLUGIN_NAME);
         $anonymizeIP["maskLength"] = Zend_Registry::get('config')->Tracker->ip_address_mask_length;
-
-        /**
-         * synchronize ip_address_mask_length and ip_address_pre_mask_length (changed in Piwik 1.7)
-         */
-        $maskLength = Zend_Registry::get('config')->Tracker;
-        if (($maskLength->ip_address_mask_length != $maskLength->ip_address_pre_mask_length) &&
-                $anonymizeIP["enabled"] === true) {
-            $maskLength->ip_address_pre_mask_length = $maskLength->ip_address_mask_length;
-            Zend_Registry::get('config')->Tracker = $maskLength->toArray();
-        }
-
         $anonymizeIP["info"] = Piwik_PluginsManager::getInstance()->getLoadedPlugin(self::ANONYMIZE_IP_PLUGIN_NAME)->getInformation();
 
         return $anonymizeIP;

plugins/PrivacyManager/PrivacyManager.php

@@ -151,7 +151,7 @@ class Piwik_PrivacyManager extends Piwik_Plugin
         
         $sql = 'DELETE LOW_PRIORITY QUICK IGNORE FROM ' . $table . ' WHERE `idvisit` <= ? ';
 
-        if(isset($maxRowsPerRun)&&$maxRowsPerRun>0) {
+        if(isset($maxRowsPerRun) && $maxRowsPerRun > 0) {
             $sql .=  ' LIMIT ' . (int)$maxRowsPerRun;
         }
 

tests/integration/NoVisit.test.php

@@ -71,19 +71,26 @@ class Test_Piwik_Integration_NoVisit extends Test_Integration_Facade
 		$t->setUserAgent('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
 		$this->checkResponse($t->doTrackPageView('bot visit, please do not record'));
 		
-		// test with excluded IP
-		$t->setUserAgent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.2.6) Gecko/20100625 Firefox/3.6.6 (.NET CLR 3.5.30729)'); // restore normal user agent	
-		$excludedIp = '154.1.12.34';
-		Piwik_SitesManager_API::getInstance()->updateSite($idSite, 'new site name', $url=array('http://site.com'),$ecommerce = 0, $excludedIp . ',1.2.3.4');
-		$t->setIp($excludedIp);
-		$this->checkResponse($t->doTrackPageView('visit from IP excluded'));
-		
-		// test with global list of excluded IPs 
-		$excludedIpBis = '145.5.3.4';
-		Piwik_SitesManager_API::getInstance()->setGlobalExcludedIps($excludedIpBis);
-		$t->setIp($excludedIpBis);
-		$this->checkResponse($t->doTrackPageView('visit from IP globally excluded'));
-		
+		// Test IP Exclusion works with or without IP exclusion
+		foreach(array(false, true) as $enable)
+		{
+			// 	Enable IP Anonymization
+			$t->DEBUG_APPEND_URL = '&forceIpAnonymization=' . (int)$enable;
+			
+			// test with excluded IP
+			$t->setUserAgent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.2.6) Gecko/20100625 Firefox/3.6.6 (.NET CLR 3.5.30729)'); // restore normal user agent	
+			$excludedIp = '154.1.12.34';
+			Piwik_SitesManager_API::getInstance()->updateSite($idSite, 'new site name', $url=array('http://site.com'),$ecommerce = 0, $excludedIp . ',1.2.3.4');
+			$t->setIp($excludedIp);
+			$this->checkResponse($t->doTrackPageView('visit from IP excluded'));
+			
+			// test with global list of excluded IPs 
+			$excludedIpBis = '145.5.3.4';
+			Piwik_SitesManager_API::getInstance()->setGlobalExcludedIps($excludedIpBis);
+			$t->setIp($excludedIpBis);
+			$this->checkResponse($t->doTrackPageView('visit from IP globally excluded'));
+		}
+				
 		try {
 			@$t->setAttributionInfo(array());
 			$this->fail();

tests/integration/OneVisitor_NoKeywordSpecified.test.php

@@ -7,7 +7,8 @@ if(!defined('PIWIK_CONFIG_TEST_INCLUDED'))
 require_once PIWIK_INCLUDE_PATH . '/tests/integration/Integration.php';
 
 /**
- * Test empty google kwd works nicely in Live! output and Top keywords
+ * 1) Tests empty google kwd works nicely in Live! output and Top keywords
+ * 2) Tests IP anonymization
  * 
  * Also test that Live! will link to the search result page URL rather than the exact referrer URL
  * when the referrer URL is google.XX/url.... which is a redirect to landing page rather than the search result URL 
@@ -50,6 +51,7 @@ class Test_Piwik_Integration_OneVisitor_NoKeywordSpecified extends Test_Integrat
     	$idSite = $this->idSite;
         $t = $this->getTracker($idSite, $dateTime, $defaultInit = true, $useThirdPartyCookie = 1);
         
+		$t->DEBUG_APPEND_URL = '&forceIpAnonymization=1';
         // VISIT 1 = Referrer is "Keyword not defined"
         // Alsotrigger goal to check that attribution goes to this keyword
         $t->setUrlReferrer( 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0CC&url=http%3A%2F%2Fpiwik.org%2F&ei=&usg=');

tests/integration/expected/test_OneVisitor_NoKeywordSpecified__Live.getLastVisitsDetails_day.xml

@@ -3,7 +3,7 @@
 	<row>
 		<idSite>1</idSite>
 		<idVisit>2</idVisit>
-		<visitIp>156.5.3.2</visitIp>
+		<visitIp>156.5.0.0</visitIp>
 		
 		<visitorType>returning</visitorType>
 		<visitorTypeIcon>plugins/Live/templates/images/returningVisitor.gif</visitorTypeIcon>
@@ -82,7 +82,7 @@
 	<row>
 		<idSite>1</idSite>
 		<idVisit>1</idVisit>
-		<visitIp>156.5.3.2</visitIp>
+		<visitIp>156.5.0.0</visitIp>
 		
 		<visitorType>new</visitorType>
 		<visitorTypeIcon />

tests/integration/proxy-piwik.php

@@ -32,6 +32,15 @@ if(Piwik_Common::getRequestVar('forceUseThirdPartyCookie', false) == 1)
 {
 	Piwik_Tracker_Config::getInstance()->setTestValue('Tracker', 'use_third_party_id_cookie', 1);
 }
+
+// Tests can force the enabling of IP anonymization
+if(Piwik_Common::getRequestVar('forceIpAnonymization', false) == 1)
+{
+	Piwik_Tracker_Config::getInstance()->setTestValue('Tracker', 'ip_address_mask_length', 2);
+	$pluginsTracker = Piwik_Tracker_Config::getInstance()->Plugins_Tracker['Plugins_Tracker'];
+	$pluginsTracker[] = "AnonymizeIP";
+	Piwik_Tracker_Config::getInstance()->setTestValue('Plugins_Tracker', array('Plugins_Tracker' => $pluginsTracker));
+}
 // Custom IP to use for this visitor
 $customIp = Piwik_Common::getRequestVar('cip', false);
 if(!empty($customIp))