Finished main algorithm for the Logging script.

Still todo: the "Ip to provider" plugin.

git-svn-id: http://dev.piwik.org/svn/trunk@28 59fd770c-687e-43c8-a1e3-f5a4ff64c105

config/config.ini.php

@@ -18,11 +18,18 @@ tables_prefix	= piwiktests_
 [LogStats]
 ; set to 0 if you want to stop tracking the visitors. Useful if you need to stop all the connections on the DB.
 record_statistics			= 1
+
 default_action_name 		= index
-default_time_one_page_visit = 20
+default_time_one_page_visit = 10
 download_url_var_name 		= download
 outlink_url_var_name		= link
 download_outlink_name_var   = name
+newsletter_var_name			= piwik_nl
+partner_var_name			= piwik_partner
+campaign_var_name			= piwik_campaign
+campaign_keyword_var_name	= piwik_kwd
+
+cookie_name	= piwik_visitor
 
 [log]
 

misc/stress.sh

+echo "
+Stress testing piwik.php
+========================
+"
+ab -n500 -c50 "http://localhost/dev/piwiktrunk/piwik.php"

modules/Common.php

@@ -9,6 +9,15 @@
  */
 class Piwik_Common 
 {
+	const REFERER_TYPE_DIRECT_ENTRY		= 1;
+	const REFERER_TYPE_SEARCH_ENGINE	= 2;
+	const REFERER_TYPE_WEBSITE			= 3;
+	const REFERER_TYPE_PARTNER			= 4;
+	const REFERER_TYPE_NEWSLETTER		= 5;
+	const REFERER_TYPE_CAMPAIGN			= 6;
+	
+	const HTML_ENCODING_QUOTE_STYLE		= ENT_COMPAT;
+	
 	/**
 	 * Returns the variable after cleaning operations.
 	 * NB: The variable still has to be escaped before going into a SQL Query!
@@ -50,7 +59,7 @@ class Piwik_Common
 		}
 		elseif(is_string($value))
 		{
-			$value = htmlspecialchars($value, ENT_COMPAT, 'UTF-8');
+			$value = htmlspecialchars($value, Piwik_Common::HTML_ENCODING_QUOTE_STYLE, 'UTF-8');
 
 			/* Undo the damage caused by magic_quotes */
 			if (get_magic_quotes_gpc()) 
@@ -68,6 +77,12 @@ class Piwik_Common
 		return $value;
     }
 
+
+	static public function getDatetimeFromTimestamp($timestamp)
+	{
+		return date("Y-m-d H:i:s",$timestamp);
+	}
+	
 	/**
 	 * Returns a variable from the $_REQUEST superglobal.
 	 * If the variable doesn't have a value or an empty value, returns the defaultValue if specified.
@@ -388,7 +403,7 @@ class Piwik_Common
 	*/
 	function getContinent($country)
 	{
-		require_once PIWIK_INCLUDE_PATH . "/modules/DataFiles/Countries.php";
+		require_once PIWIK_DATAFILES_INCLUDE_PATH . "/Countries.php";
 		
 		$countryList = $GLOBALS['Piwik_CountryList'];
 		
@@ -411,7 +426,7 @@ class Piwik_Common
 	*/
 	function getCountry( $lang )
 	{
-		require_once PIWIK_INCLUDE_PATH . "/modules/DataFiles/Countries.php";
+		require_once PIWIK_DATAFILES_INCLUDE_PATH . "/Countries.php";
 		
 		$countryList = $GLOBALS['Piwik_CountryList'];
 		
@@ -507,7 +522,37 @@ class Piwik_Common
 		// at this point we really can't guess the country
 		return 'xx';
 	}
-		
+	
+	/**
+	* Returns the value of a GET parameter $parameter in an URL query $urlQuery
+	* 
+	* @param string $urlQuery result of parse_url()['query'] and htmlentitied (& is &)
+	* @param string $param
+	* 
+	* @return string|bool Parameter value if found (can be the empty string!), false if not found
+	*/
+	static public function getParameterFromQueryString( $urlQuery, $parameter)
+	{	
+		$refererQuery = '&'.trim(str_replace(array('%20'), ' ', '&'.$urlQuery));
+		$word = '&'.$parameter.'=';
+				
+		if( $off = strrpos($refererQuery, $word))
+		{
+			$off += strlen($word); // &q=
+			$str = substr($refererQuery, $off);
+			$len = strpos($str, '&');
+			if($len === false)
+			{
+				$len = strlen($str);
+			}
+			$toReturn = substr($refererQuery, $off, $len);
+			return $toReturn;
+		}
+		else
+		{
+			return false;
+		}
+	}
 	
 }
 ?>

modules/LogStats/Plugins.php

+<?php
+
+class Piwik_Plugin_LogStats_Provider extends Piwik_Plugin
+{	
+	public function __construct()
+	{
+	}
+
+	public function getInformation()
+	{
+		$info = array(
+			'name' => 'LogProvider',
+			'description' => 'Log in the DB the hostname looked up from the IP',
+			'author' => 'Piwik',
+			'homepage' => 'http://piwik.org/plugins/LogProvider',
+			'version' => '0.1',
+		);
+		
+		return $info;
+	}
+	
+	function install()
+	{
+		// add column hostname / hostname ext in the visit table
+	}
+	
+	function uninstall()
+	{
+		// add column hostname / hostname ext in the visit table
+	}
+	
+	function getListHooksRegistered()
+	{
+		$hooks = array(
+			'LogsStats.NewVisitor' => 'detectHostname'
+		);
+		return $hooks;
+	}
+	
+	function detectHostname( $notification )
+	{
+		$object = $notification->getNotificationObject();
+		printDebug();
+	}
+}
+/*
+class Piwik_Plugin_LogStats_UserSettings extends Piwik_Plugin
+{
+	
+}*/
+
+?>

modules/Piwik.php

@@ -106,43 +106,43 @@ class Piwik
 			",
 			
 			'log_visit' => "CREATE TABLE {$prefixTables}log_visit (
-								  idvisit INTEGER(10) UNSIGNED NOT NULL AUTO_INCREMENT,
-								  idsite INTEGER(10) UNSIGNED NOT NULL,
-								  visitor_localtime TIME NOT NULL,
-								  visitor_idcookie CHAR(32) NOT NULL,
-								  visitor_returning TINYINT(1) NOT NULL,
-								  visitor_last_visit_time TIME NOT NULL,
-								  visit_server_date DATE NOT NULL,
-								  visit_server_time TIME NOT NULL,
-								  visit_exit_idaction INTEGER(11) NOT NULL,
-								  visit_entry_idaction INTEGER(11) NOT NULL,
-								  visit_total_actions SMALLINT(5) UNSIGNED NOT NULL,
-								  visit_total_time SMALLINT(5) UNSIGNED NOT NULL,
-								  referer_type INTEGER UNSIGNED NULL,
-								  referer_name VARCHAR(70) NULL,
-								  referer_url TEXT NOT NULL,
-								  referer_keyword VARCHAR(255) NULL,
-								  config_md5config CHAR(32) NOT NULL,
-								  config_os CHAR(3) NOT NULL,
-								  config_browser_name VARCHAR(10) NOT NULL,
-								  config_browser_version VARCHAR(20) NOT NULL,
-								  config_resolution VARCHAR(9) NOT NULL,
-								  config_color_depth TINYINT(2) UNSIGNED NOT NULL,
-								  config_pdf TINYINT(1) NOT NULL,
-								  config_flash TINYINT(1) NOT NULL,
-								  config_java TINYINT(1) NOT NULL,
-								  config_javascript TINYINT(1) NOT NULL,
-								  config_director TINYINT(1) NOT NULL,
-								  config_quicktime TINYINT(1) NOT NULL,
-								  config_realplayer TINYINT(1) NOT NULL,
-								  config_windowsmedia TINYINT(1) NOT NULL,
-								  config_cookie TINYINT(1) NOT NULL,
-								  location_ip BIGINT(11) NOT NULL,
-								  location_browser_lang VARCHAR(20) NOT NULL,
-								  location_country CHAR(3) NOT NULL,
-								  location_continent CHAR(3) NOT NULL,
-								  PRIMARY KEY(idvisit)
-								)
+  idvisit INTEGER(10) UNSIGNED NOT NULL AUTO_INCREMENT,
+  idsite INTEGER(10) UNSIGNED NOT NULL,
+  visitor_localtime TIME NOT NULL,
+  visitor_idcookie CHAR(32) NOT NULL,
+  visitor_returning TINYINT(1) NOT NULL,
+  visit_first_action_time DATETIME NOT NULL,
+  visit_last_action_time DATETIME NOT NULL,
+  visit_server_date DATE NOT NULL,
+  visit_exit_idaction INTEGER(11) NOT NULL,
+  visit_entry_idaction INTEGER(11) NOT NULL,
+  visit_total_actions SMALLINT(5) UNSIGNED NOT NULL,
+  visit_total_time SMALLINT(5) UNSIGNED NOT NULL,
+  referer_type INTEGER UNSIGNED NULL,
+  referer_name VARCHAR(70) NULL,
+  referer_url TEXT NOT NULL,
+  referer_keyword VARCHAR(255) NULL,
+  config_md5config CHAR(32) NOT NULL,
+  config_os CHAR(3) NOT NULL,
+  config_browser_name VARCHAR(10) NOT NULL,
+  config_browser_version VARCHAR(20) NOT NULL,
+  config_resolution VARCHAR(9) NOT NULL,
+  config_color_depth TINYINT(2) UNSIGNED NOT NULL,
+  config_pdf TINYINT(1) NOT NULL,
+  config_flash TINYINT(1) NOT NULL,
+  config_java TINYINT(1) NOT NULL,
+  config_javascript TINYINT(1) NOT NULL,
+  config_director TINYINT(1) NOT NULL,
+  config_quicktime TINYINT(1) NOT NULL,
+  config_realplayer TINYINT(1) NOT NULL,
+  config_windowsmedia TINYINT(1) NOT NULL,
+  config_cookie TINYINT(1) NOT NULL,
+  location_ip BIGINT(11) NOT NULL,
+  location_browser_lang VARCHAR(20) NOT NULL,
+  location_country CHAR(3) NOT NULL,
+  location_continent CHAR(3) NOT NULL,
+  PRIMARY KEY(idvisit)
+)
 			",
 			
 			'log_link_visit_action' => "CREATE TABLE {$prefixTables}log_link_visit_action (

modules/PluginManager.php

+<?php
+
+/**
+ * Plugin specification for a statistics logging plugin
+ * 
+ * A plugin that display data in the Piwik Interface is very different from a plugin 
+ * that will save additional data in the database during the statistics logging. 
+ * These two types of plugins don't have the same requirements at all. Therefore a plugin
+ * that saves additional data in the database during the stats logging process will have a different
+ * structure.
+ * 
+ * A plugin for logging data has to focus on performance and therefore has to stay as simple as possible.
+ * For input data, it is strongly advised to use the Piwik methods available in Piwik_Common 
+ *
+ * Things that can be done with such a plugin:
+ * - having a dependency with a list of other plugins
+ * - have an install step that would prepare the plugin environment
+ * 		- install could add columns to the tables
+ * 		- install could create tables 
+ * - register to hooks at several points in the logging process
+ * - register to hooks in other plugins
+ * - generally a plugin method can modify data (filter) and add/remove data 
+ * 
+ * 
+ */ 
+class Piwik_PluginsManager
+{
+	public $dispatcher;
+	private $pluginsPath;
+	
+	static private $instance = null;
+	
+	static public function getInstance()
+	{
+		if (self::$instance == null)
+		{			
+			$c = __CLASS__;
+			self::$instance = new $c();
+		}
+		return self::$instance;
+	}
+	
+	private function __construct()
+	{
+		$this->pluginsPath = 'plugins/';
+		$this->pluginsCategory = 'LogsStats/';
+		
+		$this->dispatcher = Event_Dispatcher::getInstance();
+		$this->loadPlugins();
+	}
+	
+	/**
+	 * Load the plugins classes installed.
+	 * Register the observers for every plugin.
+	 * 
+	 */
+	public function loadPlugins()
+	{
+		$defaultPlugins = array(
+			array( 'fileName' => 'Provider', 'className' => 'Piwik_Plugin_LogStats_Provider' ),
+		//	'Piwik_Plugin_LogStats_UserSettings',
+		);
+		
+		foreach($defaultPlugins as $pluginInfo)
+		{
+			$pluginFileName = $pluginInfo['fileName'];
+			$pluginClassName = $pluginInfo['className'];
+			/*
+			// TODO make sure the plugin name is secure
+			// make sure thepluigin is a child of Piwik_Plugin
+			$path = PIWIK_INCLUDE_PATH 
+					. $this->pluginsPath 
+					. $this->pluginsCategory
+					. $pluginFileName . ".php";
+			
+			if(is_file($path))
+			{
+				throw new Exception("The plugin file $path couldn't be found.");
+			}
+			
+			require_once $path;
+			*/
+			
+			$newPlugin = new $pluginClassName;
+			
+			$this->addPluginObservers( $newPlugin );
+		}
+	}
+	
+	/**
+	 * For the given plugin, add all the observers of this plugin.
+	 */
+	private function addPluginObservers( Piwik_Plugin $plugin )
+	{
+		$hooks = $plugin->getListHooksRegistered();
+		
+		foreach($hooks as $hookName => $methodToCall)
+		{
+			$this->dispatcher->addObserver( array( $plugin, $methodToCall) );
+		}
+	}
+	
+}
+
+/**
+ * Post an event to the dispatcher which will notice the observers
+ */
+function Piwik_PostEvent( $eventName, $object = null, $info = array() )
+{
+	printDebug("Dispatching event $eventName...");
+	Piwik_PluginsManager::getInstance()->dispatcher->post( $object, $eventName, $info, false, false );
+}
+
+/**
+ * Abstract class to define a Piwik_Plugin.
+ * Any plugin has to at least implement the abstract methods of this class.
+ */
+abstract class Piwik_Plugin
+{
+	/**
+	 * Returns the plugin details
+	 */
+	abstract function getInformation();
+	
+	/**
+	 * Returns the list of hooks registered with the methods names
+	 */
+	abstract function getListHooksRegistered();
+	
+	/**
+	 * Returns the names of the required plugins
+	 */
+	public function getListRequiredPlugins()
+	{
+		return array();
+	}
+	 
+	/**
+	 * Install the plugin
+	 * - create tables
+	 * - update existing tables
+	 * - etc.
+	*/
+	public function install()
+	{
+		return;
+	}
+	  
+	/**
+	 * Remove the created resources during the install
+	 */
+	public function uninstall()
+	{
+		return;
+	}
+}
+
+?>

tests/modules/Common.test.php

@@ -58,6 +58,15 @@ class Test_Piwik_Common extends UnitTestCase
 		$this->assertEqual( $a1OK, Piwik_Common::sanitizeInputValues($a1));
 	}
 	
+	// sanitize a string unicode => no change
+	function test_sanitizeInputValues_arrayBadValueutf8()
+	{
+		$a1 =   " Поиск в Интернете  Поgqegиск страниц на рgeqg8978усском";
+		$a1OK = " Поиск в Интернете  Поgqegиск страниц на рgeqg8978усском";
+
+		$this->assertEqual( $a1OK, Piwik_Common::sanitizeInputValues($a1));
+	}
+	
 	// sanitize a bad string
 	function test_sanitizeInputValues_badString()
 	{
@@ -296,5 +305,91 @@ class Test_Piwik_Common extends UnitTestCase
     	$this->assertEqual( Piwik_Common::getRequestVar('test', array(), 'array'), array());
     	
     }
+    
+    
+    
+    /**
+     * no query string => false
+     */
+    function test_getParameterFromQueryString_noQuerystring()
+    {
+    	$urlQuery = "";
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "test''";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = false;
+    	$this->assertEqual($result, $expectedResult);
+    }
+    
+    /**
+     * param not found => false
+     */
+    function test_getParameterFromQueryString_parameternotfound()
+    {
+    	
+    	$urlQuery = "toto=mama&mama=titi";
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "tot";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = false;
+    	$this->assertEqual($result, $expectedResult);
+    }
+    
+    /**
+     * empty parameter value => returns empty string
+     */
+    function test_getParameterFromQueryString_emptyParamValue()
+    {
+    	
+    	$urlQuery = "toto=mama&mama=&tuytyt=teaoi";
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "mama";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = '';
+    	$this->assertEqual($result, $expectedResult);
+    }
+    
+    /**
+     * twice the parameter => returns the last value in the url
+     */
+    function test_getParameterFromQueryString_twiceTheParameterInQuery()
+    {
+    	
+    	$urlQuery = "toto=mama&mama=&tuytyt=teaoi&toto=mama second value";
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "toto";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = 'mama second value';
+    	$this->assertEqual($result, $expectedResult);
+    }
+    
+    /**
+     * normal use case => parameter found
+     */
+    function test_getParameterFromQueryString_normalCase()
+    {
+    	
+    	$urlQuery = "toto=mama&mama=&tuytyt=teaoi&toto=mama second value";
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "tuytyt";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = 'teaoi';
+    	$this->assertEqual($result, $expectedResult);
+    }
+    
+    /**
+     * normal use case with a string with many strange characters
+     */
+    function test_getParameterFromQueryString_strangeChars()
+    {
+    	
+    	$urlQuery = 'toto=mama&mama=&tuytyt=Поиск в Интернете  Поиск страниц на русском _*()!$!£$^!£$%&toto=mama second value';
+    	$urlQuery = htmlentities($urlQuery);
+    	$parameter = "tuytyt";
+    	$result = Piwik_Common::getParameterFromQueryString( $urlQuery, $parameter);
+    	$expectedResult = 'Поиск в Интернете  Поиск страниц на русском _*()!$!£$^!£$%';
+    	$expectedResult = htmlentities($expectedResult);
+    	$this->assertEqual($result, $expectedResult);
+    }
 }
 ?>