- improving language to country detection, fixing existing language code ->...

- improving language to country detection, fixing existing language code -> language detection issues, cleaning code, adding tests, patch by Anthon!

git-svn-id: http://dev.piwik.org/svn/trunk@928 59fd770c-687e-43c8-a1e3-f5a4ff64c105

config/global.ini.php

@@ -197,6 +197,11 @@ use_first_referer_to_determine_goal_referer = false
 ; However when most users have the same IP, and the same configuration, it is advised to set it to false
 enable_detect_unique_visitor_using_settings = true
 
+; if set to true, Piwik attempts a "best guess" at the visitor's country of
+; origin when the preferred language tag omits region information.
+; The mapping is defined in core/DataFiles/LanguageToCountry.php,
+enable_language_to_country_guess = true
+
 [log]
 
 ;possible values for log: screen, database, file

core/Common.php

@@ -463,105 +463,6 @@ class Piwik_Common
 		return md5(uniqid(rand(), true));
 	}
 
-	/**
-	 * Returns a 3 letters ID for the operating system part, given a user agent string.
-	 * @see core/DataFiles/OS.php for the list of OS (also available in $GLOBALS['Piwik_Oslist'])
-	 * If the OS cannot be identified in the user agent, returns 'UNK'
-	 * 
-	 * @param string $userAgent
-	 * 
-	 * @return string
-	 */
-	static public function getOs($userAgent)
-	{
-
-		require_once "DataFiles/OS.php";
-		$osNameToId = $GLOBALS['Piwik_Oslist'];
-
-		foreach($osNameToId as $key => $value)
-		{
-			if ($ok = ereg($key, $userAgent))
-			{
-				return $value;
-			}
-		}
-		return 'UNK';
-	}
-
-	/**
-	 * Returns the browser information from the user agent string.
-	 * 
-	 * @see core/DataFiles/Browsers.php for the list of OS (also available in $GLOBALS['Piwik_BrowserList'])
-	 * 
-	 * @param string $userAgent
-	 * @return array array(		'name' 			=> '', // 2 letters ID or 'UNK' for an unknown browser
-	 * 							'major_number' 	=> '', // 2 in firefox 2.0.12
-	 * 							'minor_number' 	=> '', // 0 in firefox 2.0.12
-	 * 							'version' 		=> ''  // major_number.minor_number
-	 * 				);
-	 */
-	static public function getBrowserInfo($userAgent)
-	{
-		require_once "DataFiles/Browsers.php";
-
-		$browsers = $GLOBALS['Piwik_BrowserList'];
-
-		$info = array(
-			'name' 			=> 'UNK',
-			'major_number' 	=> '',
-			'minor_number' 	=> '',
-			'version' 		=> ''
-			);
-
-		$browser = '';
-		foreach($browsers as $key => $value)
-		{
-			if(!empty($browser)) $browser .= "|";
-			$browser .= $key;
-		}
-
-		$results = array();
-
-		// added fix for Mozilla Suite detection
-		if ((preg_match_all("/(mozilla)[\/\sa-z;.0-9-(]+rv:([0-9]+)([.0-9a-z]+)\) gecko\/[0-9]{8}$/i", $userAgent, $results))
-			||	(preg_match_all("/($browser)[\/\sa-z(]*([0-9]+)([\.0-9a-z]+)?/i", $userAgent, $results))
-			)
-		 {
-		 	$count = count($results[0])-1;
-		 	
-		 	// because google chrome is Mozilla/Chrome/Safari at the same time, we force Chrome
-		 	if(($chrome = array_search('Chrome', $results[1])) !== false)
-		 	{
-		 		$count = $chrome;
-		 	}
-		 	
-		 	// browser code
-		 	$info['name'] = $browsers[strtolower($results[1][$count])];
-		 		
-		 	// major version number (1 in mozilla 1.7)
-		 	$info['major_number'] = $results[2][$count];
-		 		
-		 	// is an minor version number ? If not, 0
-		 	$match = array();
-		 		
-		 	preg_match('/([.\0-9]+)?([\.a-z0-9]+)?/i', $results[3][$count], $match);
-		 		
-		 	if(isset($match[1]))
-		 	{
-		 		// find minor version number (7 in mozilla 1.7, 9 in firefox 0.9.3)
-		 		$info['minor_number'] = substr($match[1], 0, 2);
-		 	}
-		 	else
-		 	{
-		 		$info['minor_number'] = '.0';
-		 	}
-		 		
-		 	$info['version'] = $info['major_number'] . $info['minor_number'];
-		 }
-		 return $info;
-	}
-
-
 	/**
 	 * Returns the best possible IP of the current user, in the format A.B.C.D
 	 *
@@ -650,61 +551,67 @@ class Piwik_Common
 	 *
 	 * @return string
 	 */
-	static public function getBrowserLanguage()
+	static public function getBrowserLanguage($browserLang = NULL)
 	{
-		$browserLang = Piwik_Common::sanitizeInputValues(@$_SERVER['HTTP_ACCEPT_LANGUAGE']);
+		static $replacementPatterns = array(
+				'/(\\\\.)/',     // quoted-pairs (RFC 3282)
+				'/(\s+)/',       // CFWS white space
+				'/(\([^)]*\))/', // CFWS comments
+				'/(;q=[0-9.]+)/' // quality
+			);
+
+		if(is_null($browserLang))
+		{
+			$browserLang = Piwik_Common::sanitizeInputValues(@$_SERVER['HTTP_ACCEPT_LANGUAGE']);
+		}
+
 		if(is_null($browserLang))
 		{
-			$browserLang = '';
+			// a fallback might be to infer the language in HTTP_USER_AGENT (i.e., localized build)
+			$browserLang = "";
 		}
+		else
+		{
+			// language tags are case-insensitive per HTTP/1.1 s3.10 but the region may be capitalized per ISO3166-1;
+			// underscores are not permitted per RFC 4646 or 4647 (which obsolete RFC 1766 and 3066),
+			// but we guard against a bad user agent which naively uses its locale
+			$browserLang = strtolower(str_replace('_', '-', $browserLang));
+
+			// filters
+			$browserLang = preg_replace($replacementPatterns, '', $browserLang);
+
+			$browserLang = preg_replace('/((^|,)chrome:.*)/', '', $browserLang, 1); // Firefox bug
+			$browserLang = preg_replace('/(,)(?:en-securid,)|(?:(^|,)en-securid(,|$))/', '\\1',	$browserLang, 1); // unregistered language tag
+
+			$browserLang = str_replace('sr-sp', 'sr-rs', $browserLang); // unofficial (proposed) code in the wild
+		}
+
 		return $browserLang;
 	}
-	
+
 	/**
 	 * Returns the visitor country based only on the Browser 'accepted language' information
 	 *
 	 * @param string $lang browser lang
-	 *
-	 * @return string 2 letters ISO code 
+	 * @param bool If set to true, some assumption will be made and detection guessed more often, but accuracy could be affected
+	 * @return string 2 letter ISO code 
 	 */
-	static public function getCountry( $lang )
+	static public function getCountry( $lang, $enableLanguageToCountryGuess )
 	{
-		$replaceLangCodeByCountryCode = array(
-		// replace cs language (Serbia Montenegro country code) with czech country code
-			'cs' => 'cz',
-		// replace sr language (Surinam country code) with serbia country code
-			'sr' => 'rs',
-		// replace sv language (El Salvador country code) with sweden country code
-			'sv' => 'se',
-		// replace fa language (Unknown country code) with Iran country code
-			'fa' => 'ir',
-		// replace ja language (Unknown country code) with japan country code
-			'ja' => 'jp',
-		// replace ko language (Unknown country code) with corée country code
-			'ko' => 'kr',
-		// replace he language (Unknown country code) with Israel country code
-			'he' => 'il',
-		// replace da language (Unknown country code) with Danemark country code
-			'da' => 'dk',
-		// replace gb code with UK country code
-			'gb' => 'uk',
-		);
-
 		if(empty($lang) || strlen($lang) < 2)
 		{
 			return 'xx';
 		}
 
-		$browserLanguage = str_replace(
-					array_keys($replaceLangCodeByCountryCode),
-					array_values($replaceLangCodeByCountryCode),
-					$lang
-				);
-		
 		$validCountries = self::getCountriesList();
-		return Piwik_Common::extractLanguageCodeFromBrowserLanguage($browserLanguage, $validCountries);
+		return self::extractCountryCodeFromBrowserLanguage($lang, $validCountries, $enableLanguageToCountryGuess);
 	}
 	
+	/**
+	 * Returns list of valid country codes
+	 *
+	 * @return array of 2 letter ISO codes
+	 */
 	static public function getCountriesList()
 	{
 		static $countriesList = null;
@@ -715,68 +622,79 @@ class Piwik_Common
 		}
 		return $countriesList;
 	}
+	/**
+	 * Returns list of valid country codes
+	 *
+	 * @return array of 2 letter ISO codes
+	 */
+	static public function extractCountryCodeFromBrowserLanguage($browserLanguage, $validCountries, $enableLanguageToCountryGuess)
+ 	{
+		static $langToCountry = null;
+		if(is_null($langToCountry))
+ 		{
+			require_once "DataFiles/LanguageToCountry.php";
+			$langToCountry = array_keys($GLOBALS['Piwik_LanguageToCountry']);
+ 		}
+ 
+ 		if($enableLanguageToCountryGuess)
+ 		{
+			if(preg_match('/^([a-z]{2,3})(?:,|;|$)/', $browserLanguage, $matches))
+ 			{
+				// match language (without region) to infer the country of origin
+				if(in_array($matches[1], $langToCountry))
+				{
+					return $GLOBALS['Piwik_LanguageToCountry'][$matches[1]];
+				}
+ 			}
+ 		}
+ 
+		if(preg_match_all("/[-]([a-z]{2})/", $browserLanguage, $matches, PREG_SET_ORDER))
+ 		{
+			foreach($matches as $parts)
+ 			{
+				// match location; we don't make any inferences from the language
+				if(in_array($parts[1], $validCountries))
+				{
+					return $parts[1];
+				}
+ 			}
+ 		}	
+		return 'xx';
+	}
 	
-	static public function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validCountries)
+	
+ 	/**
+	 * Returns the visitor language based only on the Browser 'accepted language' information
+	 *
+	 * @param string $lang browser lang
+	 *
+	 * @return string 2 letter ISO 639 code
+	 */
+	static public function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages)
 	{
-		// Ex: "fr"
-		if(strlen($browserLanguage) == 2)
-		{
-			if(in_array($browserLanguage, $validCountries))
-			{
-				return $browserLanguage;
-			}
-		}
-
-		// when comma
-		$offcomma = strpos($browserLanguage, ',');
-		if($offcomma == 2)
-		{
-			// in 'fr,en-us', keep first two chars
-			$domain = substr($browserLanguage, 0, 2);
-			if(in_array($domain, $validCountries))
-			{
-				return $domain;
-			}
-
-			// catch the second language Ex: "fr" in "en,fr"
-			$domain = substr($browserLanguage, 3, 2);
-			if(in_array($domain, $validCountries))
-			{
-				return $domain;
-			}
-		}
-
-		// detect second code Ex: "be" in "fr-be"
-		$off = strpos($browserLanguage, '-');
-		if($off !== false)
-		{
-			$domain = substr($browserLanguage, $off + 1, 2);
-			if(in_array($domain, $validCountries))
-			{
-				return $domain;
-			}
-		}
-
-		// catch the second language Ex: "fr" in "en;q=1.0,fr;q=0.9"
-		if(preg_match("/^[a-z]{2};q=[01]\.[0-9],(?P<domain>[a-z]{2});/", $browserLanguage, $parts))
-		{
-			$domain = $parts['domain'];
-			if(in_array($domain, $validCountries))
-			{
-				return $domain;
-			}
-		}
-
-		// finally try with the first ever langage code
-		$domain = substr($browserLanguage, 0, 2);
-		if(in_array($domain, $validCountries))
-		{
-			return $domain;
-		}
-
-		// at this point we really can't guess the country
+		// assumes language preference is sorted;
+		// does not handle language-script-region tags or language range (*)
+		if(preg_match_all("/(?:^|,)([a-z]{2,3})([-][a-z]{2})?/", $browserLanguage, $matches, PREG_SET_ORDER))
+ 		{
+			foreach($matches as $parts)
+ 			{
+				if(count($parts) == 3)
+				{
+					// match locale (langauge and location)
+					if(in_array($parts[1].$parts[2], $validLanguages))
+					{
+						return $parts[1].$parts[2];
+					}
+				}
+				// match language only (where no region provided)
+				if(in_array($parts[1], $validLanguages))
+				{
+					return $parts[1];
+				}
+ 			}
+ 		}
 		return 'xx';
-	}
+ 	}
 	
 	/**
 	 * Generate random string 

core/DataFiles/Countries.php

@@ -16,9 +16,9 @@
  */
 if(!isset($GLOBALS['Piwik_CountryList']))
 {
+	// Reference: ISO 3166-1 alpha-2
 	$GLOBALS['Piwik_CountryList'] = array(
 			'xx' => array('unk'),
-			'ac' => array('afr'),
 			'ad' => array('eur'),
 			'ae' => array('asi'),
 			'af' => array('asi'),
@@ -28,12 +28,13 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'am' => array('asi'),
 			'an' => array('ams'),
 			'ao' => array('afr'),
-			'aq' => array('aut'),
+			'aq' => array('oce'),
 			'ar' => array('ams'),
 			'as' => array('oce'),
 			'at' => array('eur'),
 			'au' => array('oce'),
 			'aw' => array('ams'),
+			'ax' => array('eur'),
 			'az' => array('asi'),
 			'ba' => array('eur'),
 			'bb' => array('ams'),
@@ -44,12 +45,14 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'bh' => array('asi'),
 			'bi' => array('afr'),
 			'bj' => array('afr'),
+			'bl' => array('ams'),
 			'bm' => array('ams'),
 			'bn' => array('asi'),
 			'bo' => array('ams'),
 			'br' => array('ams'),
 			'bs' => array('ams'),
 			'bt' => array('asi'),
+			'bv' => array('oce'),
 			'bw' => array('afr'),
 			'by' => array('eur'),
 			'bz' => array('ams'),
@@ -65,10 +68,10 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'cm' => array('afr'),
 			'cn' => array('asi'),
 			'co' => array('ams'),
-			'cs' => array('eur'),
 			'cr' => array('ams'),
 			'cu' => array('ams'),
 			'cv' => array('afr'),
+			'cx' => array('oce'),
 			'cy' => array('eur'),
 			'cz' => array('eur'),
 			'de' => array('eur'),
@@ -88,6 +91,7 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'fj' => array('oce'),
 			'fk' => array('ams'),
 			'fm' => array('oce'),
+			'fo' => array('eur'),
 			'fr' => array('eur'),
 			'ga' => array('afr'),
 			'gb' => array('eur'),
@@ -105,9 +109,11 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'gr' => array('eur'),
 			'gs' => array('eur'),
 			'gt' => array('ams'),
+			'gu' => array('asi'),
 			'gw' => array('afr'),
 			'gy' => array('ams'),
 			'hk' => array('asi'),
+			'hm' => array('oce'),
 			'hn' => array('ams'),
 			'hr' => array('eur'),
 			'ht' => array('ams'),
@@ -115,11 +121,14 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'id' => array('asi'),
 			'ie' => array('eur'),
 			'il' => array('asi'),
+			'im' => array('eur'),
 			'in' => array('asi'),
+			'io' => array('asi'),
 			'iq' => array('asi'),
 			'ir' => array('asi'),
 			'is' => array('eur'),
 			'it' => array('eur'),
+			'je' => array('eur'),
 			'jm' => array('ams'),
 			'jo' => array('asi'),
 			'jp' => array('asi'),
@@ -128,6 +137,7 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'kh' => array('asi'),
 			'ki' => array('oce'),
 			'km' => array('afr'),
+			'kn' => array('ams'),
 			'kp' => array('asi'),
 			'kr' => array('asi'),
 			'kw' => array('asi'),
@@ -135,6 +145,7 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'kz' => array('asi'),
 			'la' => array('asi'),
 			'lb' => array('asi'),
+			'lc' => array('ams'),
 			'li' => array('eur'),
 			'lk' => array('asi'),
 			'lr' => array('afr'),
@@ -146,6 +157,8 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'ma' => array('afr'),
 			'mc' => array('eur'),
 			'md' => array('eur'),
+			'me' => array('eur'),
+			'mf' => array('ams'),
 			'mg' => array('afr'),
 			'mh' => array('oce'),
 			'mk' => array('eur'),
@@ -153,8 +166,10 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'mm' => array('asi'),
 			'mn' => array('asi'),
 			'mo' => array('asi'),
+			'mp' => array('asi'),
 			'mq' => array('ams'),
 			'mr' => array('afr'),
+			'ms' => array('ams'),
 			'mt' => array('eur'),
 			'mu' => array('afr'),
 			'mv' => array('asi'),
@@ -165,12 +180,14 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'na' => array('afr'),
 			'nc' => array('oce'),
 			'ne' => array('afr'),
+			'nf' => array('oce'),
 			'ng' => array('afr'),
 			'ni' => array('ams'),
 			'nl' => array('eur'),
 			'no' => array('eur'),
 			'np' => array('asi'),
 			'nr' => array('oce'),
+			'nu' => array('oce'),
 			'nz' => array('oce'),
 			'om' => array('asi'),
 			'pa' => array('ams'),
@@ -181,15 +198,17 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'pk' => array('asi'),
 			'pl' => array('eur'),
 			'pm' => array('amn'),
+			'pn' => array('oce'),
 			'pr' => array('ams'),
+			'ps' => array('asi'),
 			'pt' => array('eur'),
 			'pw' => array('oce'),
 			'py' => array('ams'),
 			'qa' => array('asi'),
 			're' => array('afr'),
 			'ro' => array('eur'),
-			'ru' => array('asi'),
 			'rs' => array('eur'),
+			'ru' => array('asi'),
 			'rw' => array('afr'),
 			'sa' => array('asi'),
 			'sb' => array('oce'),
@@ -197,44 +216,76 @@ if(!isset($GLOBALS['Piwik_CountryList']))
 			'sd' => array('afr'),
 			'se' => array('eur'),
 			'sg' => array('asi'),
+			'sh' => array('afr'),
 			'si' => array('eur'),
+			'sj' => array('eur'),
 			'sk' => array('eur'),
 			'sl' => array('afr'),
 			'sm' => array('eur'),
 			'sn' => array('afr'),
 			'so' => array('afr'),
 			'sr' => array('ams'),
+			'st' => array('afr'),
 			'sv' => array('ams'),
 			'sy' => array('asi'),
 			'sz' => array('afr'),
+			'tc' => array('ams'),
 			'td' => array('afr'),
+			'tf' => array('oce'),
 			'tg' => array('afr'),
 			'th' => array('asi'),
 			'tj' => array('asi'),
+			'tk' => array('oce'),
+			'tl' => array('asi'),
 			'tm' => array('asi'),
 			'tn' => array('afr'),
 			'to' => array('oce'),
-			'tp' => array('oce'),
 			'tr' => array('eur'),
 			'tt' => array('ams'),
+			'tv' => array('asi'),
 			'tw' => array('asi'),
 			'tz' => array('afr'),
 			'ua' => array('eur'),
 			'ug' => array('afr'),
-			'uk' => array('eur'),
+			'um' => array('oce'),
 			'us' => array('amn'),
 			'uy' => array('ams'),
 			'uz' => array('asi'),
 			'va' => array('eur'),
+			'vc' => array('ams'),
 			've' => array('ams'),
+			'vg' => array('ams'),
+			'vi' => array('ams'),
 			'vn' => array('asi'),
 			'vu' => array('oce'),
 			'wf' => array('oce'),
+			'ws' => array('asi'),
 			'ye' => array('asi'),
-			'yu' => array('eur'),
+			'yt' => array('afr'),
 			'za' => array('afr'),
 			'zm' => array('afr'),
 			'zw' => array('afr'),
+
+			// exceptionally reserved
+			'ac' => array('afr'), // .ac TLD
+			'cp' => array('ams'),
+			'dg' => array('asi'),
+			'ea' => array('afr'),
+			'eu' => array('eur'), // .eu TLD
+			'fx' => array('eur'),
+			'ic' => array('afr'),
+			'su' => array('eur'), // .su TLD
+			'ta' => array('afr'),
+			'uk' => array('eur'), // .uk TLD
+
+			// transitionally reserved
+			'bu' => array('asi'),
+			'cs' => array('eur'), // former Serbia and Montenegro
+			'nt' => array('asi'),
+			'sf' => array('eur'),
+			'tp' => array('oce'), // .tp TLD
+			'yu' => array('eur'), // .yu TLD
+			'zr' => array('afr')
 		);
 }
 

core/DataFiles/LanguageToCountry.php

+<?php
+/**
+ * Piwik - Open source web analytics
+ * 
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html Gpl v3 or later
+ * @version $Id$
+ * 
+ * @package Piwik_UserCountry
+ */
+
+/**
+ * Language to Country mapping
+ *
+ * This list is used to guess the visitor's country when the region is
+ * not specified in the first language tag.  Inclusion/exclusion is
+ * based on Piwik.org visitor statistics and probability of disambiguation.
+ * (Notably, "en" and "zh" are excluded.)
+ *
+ * If you want to add a new entry, please email us at hello at piwik.org
+ *
+ */
+if(!isset($GLOBALS['Piwik_LanguageToCountry']))
+{
+	$GLOBALS['Piwik_LanguageToCountry'] = array(
+			"bg" => "bg",	// Bulgarian  => Bulgaria
+			"ca" => "es",	// Catalan    => Spain
+			"cs" => "cz",	// Czech      => Czech Republic
+			"da" => "dk",	// Danish     => Denmark
+			"de" => "de",	// German     => Germany
+			"el" => "gr",	// Greek      => Greece
+			"es" => "es",	// Spanish    => Spain
+			"et" => "ee",	// Estonian   => Estonia
+			"fa" => "ir",	// Farsi      => Iran
+			"fi" => "fi",	// Finnish    => Finland
+			"fr" => "fr",	// French     => France
+			"he" => "il",	// Hebrew     => Israel
+			"hr" => "hr",	// Croatian   => Croatia
+			"hu" => "hu",	// Hungarian  => Hungary
+			"id" => "id",	// Indonesian => Indonesia
+			"is" => "is",	// Icelandic  => Iceland
+			"it" => "it",	// Italian    => Italy
+			"ja" => "jp",	// Japanese   => Japan
+			"ko" => "kr",	// Korean     => South Korea
+			"lt" => "lt",	// Lithuanian => Lithuania
+			"lv" => "lv",	// Latvian    => Latvia
+			"mk" => "mk",	// Macedonian => Macedonia
+			"ms" => "my",	// Malay      => Malaysia
+			"nb" => "no",	// Bokmål     => Norway
+			"nl" => "nl",	// Dutch      => Netherlands
+			"nn" => "no",	// Nynorsk    => Norway
+			"no" => "no",	// Norwegian  => Norway
+			"pl" => "pl",	// Polish     => Poland
+			"pt" => "pt",	// Portugese  => Portugal
+			"ro" => "ro",	// Romanian   => Romania
+			"ru" => "ru",	// Russian    => Russia
+			"sk" => "sk",	// Slovak     => Slovakia
+			"sl" => "si",	// Slovene    => Slovenia
+			"sq" => "al",	// Albanian   => Albania
+			"sr" => "rs",	// Serbian    => Serbia
+			"sv" => "se",	// Swedish    => Sweden
+			"th" => "th",	// Thai       => Thailand
+			"tr" => "tr",	// Turkish    => Turkey
+			"uk" => "ua",	// Ukrainian  => Ukraine
+		);
+}

core/Tracker/GoalManager.php

@@ -127,7 +127,7 @@ class Piwik_Tracker_GoalManager
 	
 	function recordGoals($visitorInformation)
 	{
-		$location_country = isset($visitorInformation['location_country']) ? $visitorInformation['location_country'] : Piwik_Common::getCountry(Piwik_Common::getBrowserLanguage());
+		$location_country = isset($visitorInformation['location_country']) ? $visitorInformation['location_country'] : Piwik_Common::getCountry(Piwik_Common::getBrowserLanguage(), $enableLanguageToCountryGuess = Piwik_Tracker_Config::getInstance()->Tracker['enable_language_to_country_guess']);
 		$location_continent = isset($visitorInformation['location_continent']) ? $visitorInformation['location_continent'] : Piwik_Common::getContinent($location_country);
 		
 		$goal = array(

core/Tracker/Visit.php

@@ -185,7 +185,7 @@ class Piwik_Tracker_Visit implements Piwik_Tracker_Visit_Interface
 		$defaultTimeOnePageVisit = Piwik_Tracker_Config::getInstance()->Tracker['default_time_one_page_visit'];
 		
 		$userInfo = $this->getUserSettingsInformation();
-		$country = Piwik_Common::getCountry($userInfo['location_browser_lang']);	
+		$country = Piwik_Common::getCountry($userInfo['location_browser_lang'], $enableLanguageToCountryGuess = Piwik_Tracker_Config::getInstance()->Tracker['enable_language_to_country_guess']);	
 		$refererInfo = $this->getRefererInformation();
 		
 		/**

lang/en.php

@@ -214,6 +214,7 @@ $translations = array(
 	'UserCountry_country_at' => 'Austria',
 	'UserCountry_country_au' => 'Australia',
 	'UserCountry_country_aw' => 'Aruba',
+	'UserCountry_country_ax' => 'Aland Islands',
 	'UserCountry_country_az' => 'Azerbaijan',
 	'UserCountry_country_ba' => 'Bosnia and Herzegovina',
 	'UserCountry_country_bb' => 'Barbados',
@@ -224,12 +225,14 @@ $translations = array(
 	'UserCountry_country_bh' => 'Bahrain',
 	'UserCountry_country_bi' => 'Burundi',
 	'UserCountry_country_bj' => 'Benin',
+	'UserCountry_country_bl' => 'Saint Barthelemy',
 	'UserCountry_country_bm' => 'Bermuda',
 	'UserCountry_country_bn' => 'Bruneo',
 	'UserCountry_country_bo' => 'Bolivia',
 	'UserCountry_country_br' => 'Brazil',
 	'UserCountry_country_bs' => 'Bahamas',
 	'UserCountry_country_bt' => 'Bhutan',
+	'UserCountry_country_bu' => 'Burma',
 	'UserCountry_country_bv' => 'Bouvet Island',
 	'UserCountry_country_bw' => 'Botswana',
 	'UserCountry_country_by' => 'Belarus',
@@ -246,6 +249,7 @@ $translations = array(
 	'UserCountry_country_cm' => 'Cameroon',
 	'UserCountry_country_cn' => 'China',
 	'UserCountry_country_co' => 'Colombia',
+	'UserCountry_country_cp' => 'Clipperton Island',
 	'UserCountry_country_cr' => 'Costa Rica',
 	'UserCountry_country_cs' => 'Serbia Montenegro',
 	'UserCountry_country_cu' => 'Cuba',
@@ -254,11 +258,13 @@ $translations = array(
 	'UserCountry_country_cy' => 'Cyprus',
 	'UserCountry_country_cz' => 'Czech Republic',
 	'UserCountry_country_de' => 'Germany',
+	'UserCountry_country_dg' => 'Diego Garcia',
 	'UserCountry_country_dj' => 'Djibouti',
 	'UserCountry_country_dk' => 'Denmark',
 	'UserCountry_country_dm' => 'Dominica',
 	'UserCountry_country_do' => 'Dominican Republic',
 	'UserCountry_country_dz' => 'Algeria',
+	'UserCountry_country_ea' => 'Ceuta, Melilla',
 	'UserCountry_country_ec' => 'Ecuador',
 	'UserCountry_country_ee' => 'Estonia',
 	'UserCountry_country_eg' => 'Egypt',
@@ -266,13 +272,16 @@ $translations = array(
 	'UserCountry_country_er' => 'Eritrea',
 	'UserCountry_country_es' => 'Spain',
 	'UserCountry_country_et' => 'Ethiopia',
+	'UserCountry_country_eu' => 'European Union',
 	'UserCountry_country_fi' => 'Finland',
 	'UserCountry_country_fj' => 'Fiji',
 	'UserCountry_country_fk' => 'Falkland Islands (Malvinas)',
 	'UserCountry_country_fm' => 'Micronesia, Federated States of',
 	'UserCountry_country_fo' => 'Faroe Islands',
 	'UserCountry_country_fr' => 'France',
+	'UserCountry_country_fx' => 'France, Metropolitan',
 	'UserCountry_country_ga' => 'Gabon',
+	'UserCountry_country_gb' => 'Great Britain',
 	'UserCountry_country_gd' => 'Grenada',
 	'UserCountry_country_ge' => 'Georgia',
 	'UserCountry_country_gf' => 'French Guyana',
@@ -296,6 +305,7 @@ $translations = array(
 	'UserCountry_country_hr' => 'Croatia',
 	'UserCountry_country_ht' => 'Haiti',
 	'UserCountry_country_hu' => 'Hungary',
+	'UserCountry_country_ic' => 'Canary Islands',
 	'UserCountry_country_id' => 'Indonesia',
 	'UserCountry_country_ie' => 'Ireland',
 	'UserCountry_country_il' => 'Israel',
@@ -335,6 +345,8 @@ $translations = array(
 	'UserCountry_country_ma' => 'Morocco',
 	'UserCountry_country_mc' => 'Monaco',
 	'UserCountry_country_md' => 'Moldova, Republic of',
+	'UserCountry_country_me' => 'Montenegro',
+	'UserCountry_country_mf' => 'Saint Martin',
 	'UserCountry_country_mg' => 'Madagascar',
 	'UserCountry_country_mh' => 'Marshall Islands',
 	'UserCountry_country_mk' => 'Macedonia',
@@ -363,6 +375,7 @@ $translations = array(
 	'UserCountry_country_no' => 'Norway',
 	'UserCountry_country_np' => 'Nepal',
 	'UserCountry_country_nr' => 'Nauru',
+	'UserCountry_country_nt' => 'Neutral Zone',
 	'UserCountry_country_nu' => 'Niue',
 	'UserCountry_country_nz' => 'New Zealand',
 	'UserCountry_country_om' => 'Oman',
@@ -391,6 +404,7 @@ $translations = array(
 	'UserCountry_country_sc' => 'Seychelles',
 	'UserCountry_country_sd' => 'Sudan',
 	'UserCountry_country_se' => 'Sweden',
+	'UserCountry_country_sf' => 'Finland',
 	'UserCountry_country_sg' => 'Singapore',
 	'UserCountry_country_sh' => 'Saint Helena',
 	'UserCountry_country_si' => 'Slovenia',
@@ -406,6 +420,7 @@ $translations = array(
 	'UserCountry_country_sv' => 'El Salvador',
 	'UserCountry_country_sy' => 'Syrian Arab Republic',
 	'UserCountry_country_sz' => 'Swaziland',
+	'UserCountry_country_ta' => 'Tristan da Cunha',
 	'UserCountry_country_tc' => 'Turks and Caicos Islands',
 	'UserCountry_country_td' => 'Chad',
 	'UserCountry_country_tf' => 'French Southern Territories',
@@ -413,6 +428,7 @@ $translations = array(
 	'UserCountry_country_th' => 'Thailand',
 	'UserCountry_country_tj' => 'Tajikistan',
 	'UserCountry_country_tk' => 'Tokelau',
+	'UserCountry_country_tl' => 'East Timor',
 	'UserCountry_country_tm' => 'Turkmenistan',
 	'UserCountry_country_tn' => 'Tunisia',
 	'UserCountry_country_to' => 'Tonga',
@@ -425,7 +441,6 @@ $translations = array(
 	'UserCountry_country_ua' => 'Ukraine',
 	'UserCountry_country_ug' => 'Uganda',
 	'UserCountry_country_uk' => 'United Kingdom',
-	'UserCountry_country_gb' => 'Great Britain',
 	'UserCountry_country_um' => 'United States Minor Outlying Islands',
 	'UserCountry_country_us' => 'United States',
 	'UserCountry_country_uy' => 'Uruguay',

tests/core/Common.test.php

@@ -81,7 +81,6 @@ class Test_Piwik_Common extends UnitTestCase
 		$a1OK = array('test1' => 't1', 't45', 'tea1"ta"e', 568, 1 => array('t<e"st'), 1.52);
 		
 		$this->assertEqual( $a1OK, Piwik_Common::sanitizeInputValues($a1));
-		
 	}
 	
 	// sanitize an array with bad value level2
@@ -442,8 +441,7 @@ class Test_Piwik_Common extends UnitTestCase
     	$expectedResult = htmlentities($expectedResult);
     	$this->assertEqual($result, $expectedResult);
     }
-    
-    
+
     public function test_isValidFilenameValidValues()
     {
     
@@ -455,6 +453,7 @@ class Test_Piwik_Common extends UnitTestCase
     		$this->assertTrue(Piwik_Common::isValidFilename($toTest), $toTest." not valid!");
     	}
     }
+
     public function test_isValidFilenameNotValidValues()
     {
     
@@ -466,6 +465,131 @@ class Test_Piwik_Common extends UnitTestCase
     		$this->assertFalse(Piwik_Common::isValidFilename($toTest), $toTest." valid but shouldn't!");
     	}
     }
+
+	/**
+	 * Data driven tests of getBrowserLanguage
+	 */
+	public function test_getBrowserLanguage()
+	{
+		$a1 = array( // user agent, browser language
+				array( "en-gb", "en-gb" ),
+
+				// filter quality attribute
+				array( "en-us,en;q=0.5", "en-us,en" ),
+
+				// bad user agents
+				array( "en-us,chrome://global/locale/intl.properties", "en-us" ),
+
+				// unregistered language tag
+				array( "en,en-securid", "en" ),
+				array( "en-securid,en", "en" ),
+				array( "en-us,en-securid,en", "en-us,en" ),
+
+				// accept private sub tags
+				array( "en-us,x-en-securid", "en-us,x-en-securid" ),
+				array( "en-us,en-x-securid", "en-us,en-x-securid" ),
+
+				// filter arbitrary white space
+				array( "en-us, en", "en-us,en" ),
+				array( "en-ca, en-us ,en", "en-ca,en-us,en" ),
+
+				// handle comments
+				array( " ( comment ) en-us (another comment) ", "en-us" ),
+
+				// handle quoted pairs (embedded in comments)
+				array( " ( \( start ) en-us ( \) end ) ", "en-us" ),
+				array( " ( \) en-ca, \( ) en-us ( \) ,en ) ", "en-us" ),
+			);
+
+		foreach($a1 as $testdata)
+		{
+			$res = Piwik_Common::getBrowserLanguage( $testdata[0] );
+			$this->assertEqual( $testdata[1], $res );
+		}
+	}
     
+	/**
+	 * Data driven tests of extractCountryCodeFromBrowserLanguage
+	 */
+	public function test_extractCountryCodeFromBrowserLanguage()
+	{
+		$a1 = array( // browser language, valid countries, expected result
+				array( "",                        array(),                 "xx" ),
+				array( "",                        array("us"),             "xx" ),
+				array( "en",                      array("us"),             "xx" ),
+				array( "en-us",                   array("us"),             "us" ),
+				array( "en-ca",                   array("us"),             "xx" ),
+				array( "en-ca",                   array("us", "ca"),       "ca" ),
+				array( "fr-fr,fr-ca",             array("us", "ca"),       "ca" ),
+				array( "fr-fr;q=1.0,fr-ca;q=0.9", array("us", "ca"),       "ca" ),
+				array( "fr-ca,fr;q=0.1",          array("us", "ca"),       "ca" ),
+				array( "en-us,en;q=0.5", Piwik_Common::getCountriesList(), "us" ),
+				array( "fr-ca,fr;q=0.1",          array("fr", "us", "ca"), "ca" ),
+				array( "fr-fr,fr-ca",             array("fr", "us", "ca"), "fr" )
+			);
+
+		foreach($a1 as $testdata)
+		{
+			$this->assertEqual( $testdata[2], Piwik_Common::extractCountryCodeFromBrowserLanguage( $testdata[0], $testdata[1], true ));
+			$this->assertEqual( $testdata[2], Piwik_Common::extractCountryCodeFromBrowserLanguage( $testdata[0], $testdata[1], false ));
+		}
+	}
+
+	/**
+	 * Data driven tests of extractCountryCodeFromBrowserLanguage
+	 */
+	public function test_extractCountryCodeFromBrowserLanguage_Infer()
+	{
+		$a1 = array( // browser language, valid countries, expected result (non-guess vs guess)
+				array( "fr,en-us",       array("us", "ca"),       "us", "fr" ),
+				array( "fr,en-us",       array("fr", "us", "ca"), "us", "fr" ),
+				array( "fr,fr-fr,en-us", array("fr", "us", "ca"), "fr", "fr" ),
+				array( "fr-fr,fr,en-us", array("fr", "us", "ca"), "fr", "fr" )
+			);
+
+		if(!class_exists('Piwik_Tracker_Config'))
+		{
+			require_once "Tracker/Config.php";
+		}
+
+		// do not infer country from language
+		foreach($a1 as $testdata)
+		{
+			$this->assertEqual( $testdata[2], Piwik_Common::extractCountryCodeFromBrowserLanguage( $testdata[0], $testdata[1], $enableLanguageToCountryGuess = false ));
+		}
+
+		// infer country from language
+		foreach($a1 as $testdata)
+		{
+			$this->assertEqual( $testdata[3], Piwik_Common::extractCountryCodeFromBrowserLanguage( $testdata[0], $testdata[1], $enableLanguageToCountryGuess = true ));
+		}
+	}
+
+	/**
+	 * Data driven tests of extractLanguageCodeFromBrowserLanguage
+	 */
+	public function test_extractLanguageCodeFromBrowserLanguage()
+	{
+		$a1 = array( // browser language, valid languages, expected result
+				array( "fr-ca",          		  array("fr"),    "fr" ),
+				array( "",                        array(),           "xx"    ),
+				array( "",                        array("en"),       "xx"    ),
+				array( "fr",                      array("en"),       "xx"    ),
+				array( "en",                      array("en"),       "en"    ),
+				array( "en-ca",                   array("en-ca"),    "en-ca" ),
+				array( "en-ca",                   array("en"),       "en"    ),
+				array( "fr,en-us",                array("fr", "en"), "fr"    ),
+				array( "fr,en-us",                array("en", "fr"), "fr"    ),
+				array( "fr-fr,fr-ca",             array("fr"),       "fr"    ),
+				array( "fr-fr,fr-ca",             array("fr-ca"),    "fr-ca" ),
+				array( "fr-fr;q=1.0,fr-ca;q=0.9", array("fr-ca"),    "fr-ca" ),
+				array( "fr-ca,fr;q=0.1",          array("fr-ca"),    "fr-ca" ),
+			);
+
+		foreach($a1 as $testdata)
+		{
+			$this->assertEqual( $testdata[2], Piwik_Common::extractLanguageCodeFromBrowserLanguage( $testdata[0], $testdata[1] ), "test with {$testdata[0]} failed, expected {$testdata[2]}");
+		}
+	}
 }