From 34b954122a534ed7e5d18f7e8ab27f0da6fbd2c0 Mon Sep 17 00:00:00 2001
From: sgiehl <stefan@piwik.org>
Date: Sat, 31 Oct 2015 21:44:11 +0100
Subject: [PATCH] move list of socials to other repo and convert format to yml
---
core/Common.php | 26 ---
plugins/Referrers/API.php | 18 +-
plugins/Referrers/Social.php | 182 ++++++++++++++++++
plugins/Referrers/Tasks.php | 17 +-
plugins/Referrers/functions.php | 84 --------
.../Referrers/tests/Unit/ReferrersTest.php | 76 --------
plugins/Referrers/tests/Unit/SocialTest.php | 100 ++++++++++
7 files changed, 306 insertions(+), 197 deletions(-)
create mode 100644 plugins/Referrers/Social.php
create mode 100644 plugins/Referrers/tests/Unit/SocialTest.php
diff --git a/core/Common.php b/core/Common.php
index 6bb4298e93..95d333db9b 100644
--- a/core/Common.php
+++ b/core/Common.php
@@ -815,32 +815,6 @@ class Common
return $dataProvider->getLanguageToCountryList();
}
- /**
- * Returns list of social networks by URL
- *
- * @see core/DataFiles/Socials.php
- *
- * @return array Array of ( URL => Social Network Name )
- */
- public static function getSocialUrls()
- {
- $cacheId = 'Common.getSocialUrls';
- $cache = Cache::getTransientCache();
- $socialUrls = $cache->fetch($cacheId);
-
- if (empty($socialUrls)) {
- require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
-
- $socialUrls = $GLOBALS['Piwik_socialUrl'];
-
- Piwik::postEvent('Referrer.addSocialUrls', array(&$socialUrls));
-
- $cache->save($cacheId, $socialUrls);
- }
-
- return $socialUrls;
- }
-
/**
* Returns list of provider names
*
diff --git a/plugins/Referrers/API.php b/plugins/Referrers/API.php
index d1c08ac20c..efda586585 100644
--- a/plugins/Referrers/API.php
+++ b/plugins/Referrers/API.php
@@ -309,9 +309,9 @@ class API extends \Piwik\Plugin\API
{
$dataTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, false);
- $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !isSocialUrl($url); }));
- $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSocialMainUrl'));
- $dataTable->filter('GroupBy', array('label', __NAMESPACE__ . '\getSocialNetworkFromDomain'));
+ $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !Social::getInstance()->isSocialUrl($url); }));
+ $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return Social::getInstance()->getMainUrl($url); }));
+ $dataTable->filter('GroupBy', array('label', function ($url) { return Social::getInstance()->getSocialNetworkFromDomain($url); }));
$this->setSocialIdSubtables($dataTable);
$this->removeSubtableMetadata($dataTable);
@@ -320,7 +320,7 @@ class API extends \Piwik\Plugin\API
$this->buildExpandedTableForFlattenGetSocials($idSite, $period, $date, $segment, $expanded, $dataTable);
}
- $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSocialsLogoFromUrl'));
+ $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return Social::getInstance()->getLogoFromUrl($url); }));
return $dataTable;
}
@@ -344,7 +344,7 @@ class API extends \Piwik\Plugin\API
$dataTable = $this->getDataTable(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded = true);
// get the social network domain referred to by $idSubtable
- $socialNetworks = Common::getSocialUrls();
+ $socialNetworks = Social::getInstance()->getSocialDefinitions();
$social = false;
if ($idSubtable !== false) {
@@ -362,7 +362,7 @@ class API extends \Piwik\Plugin\API
$dataTable->filter(
'ColumnCallbackDeleteRow',
array('label',
- function ($url) use ($social) { return !isSocialUrl($url, $social); }
+ function ($url) use ($social) { return !Social::getInstance()->isSocialUrl($url, $social); }
)
);
@@ -443,7 +443,7 @@ class API extends \Piwik\Plugin\API
$socialName = $row->getColumn('label');
$i = 1; // start at one because idSubtable=0 is equivalent to idSubtable=false
- foreach (Common::getSocialUrls() as $name) {
+ foreach (Social::getInstance()->getSocialDefinitions() as $name) {
if ($name == $socialName) {
$row->setNonLoadedSubtableId($i);
break;
@@ -491,7 +491,7 @@ class API extends \Piwik\Plugin\API
{
$urlsTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat = true);
$urlsTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) {
- return !isSocialUrl($url);
+ return !Social::getInstance()->isSocialUrl($url);
}));
$urlsTable = $urlsTable->mergeSubtables();
@@ -504,7 +504,7 @@ class API extends \Piwik\Plugin\API
$rows = $urlsTable->getRows();
foreach ($rows as $id => $urlsTableRow) {
$url = $urlsTableRow->getColumn('label');
- if (isSocialUrl($url, $social)) {
+ if (Social::getInstance()->isSocialUrl($url, $social)) {
$newTable->addRow($urlsTableRow);
$urlsTable->deleteRow($id);
}
diff --git a/plugins/Referrers/Social.php b/plugins/Referrers/Social.php
new file mode 100644
index 0000000000..123f31cd32
--- /dev/null
+++ b/plugins/Referrers/Social.php
@@ -0,0 +1,182 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Referrers;
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Singleton;
+use Piwik\UrlHelper;
+
+/**
+ * Contains methods to access search engine definition data.
+ */
+class Social extends Singleton
+{
+ const OPTION_STORAGE_NAME = 'SocialDefinitions';
+
+ /** @var string location of definition file (relative to PIWIK_INCLUDE_PATH) */
+ const DEFINITION_FILE = '/vendor/piwik/searchengine-and-social-definitions/Socials.yml';
+
+ protected $definitionList = null;
+
+ /**
+ * Returns list of search engines by URL
+ *
+ * @return array Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) )
+ */
+ public function getSocialDefinitions()
+ {
+ $cache = Cache::getEagerCache();
+ $cacheId = 'Social-' . self::OPTION_STORAGE_NAME;
+
+ if ($cache->contains($cacheId)) {
+ $list = $cache->fetch($cacheId);
+ } else {
+ $list = $this->loadSocialDefinitions();
+ $cache->save($cacheId, $list);
+ }
+
+ return $list;
+ }
+
+ private function loadSocialDefinitions()
+ {
+ if ($this->definitionList === null) {
+ // Read first from the auto-updated list in database
+ $list = Option::get(self::OPTION_STORAGE_NAME);
+
+ if ($list) {
+ $this->definitionList = unserialize($list);
+ } else {
+ // Fallback to reading the bundled list
+ $yml = file_get_contents(PIWIK_INCLUDE_PATH . self::DEFINITION_FILE);
+ $this->definitionList = $this->loadYmlData($yml);
+ Option::set(self::OPTION_STORAGE_NAME, serialize($this->definitionList));
+
+ }
+ }
+
+ Piwik::postEvent('Referrer.addSocialUrls', array(&$this->definitionList));
+
+ return $this->definitionList;
+ }
+
+ /**
+ * Parses the given YML string and caches the resulting definitions
+ *
+ * @param string $yml
+ * @return array
+ */
+ public function loadYmlData($yml)
+ {
+ $searchEngines = \Spyc::YAMLLoadString($yml);
+
+ $this->definitionList = $this->transformData($searchEngines);
+
+ return $this->definitionList;
+ }
+
+ protected function transformData($socials)
+ {
+ $urlToName = array();
+ foreach ($socials as $name => $urls) {
+ foreach ($urls as $url) {
+ $urlToName[$url] = $name;
+ }
+ }
+ return $urlToName;
+ }
+
+ /**
+ * Returns true if a URL belongs to a social network, false if otherwise.
+ *
+ * @param string $url The URL to check.
+ * @param string|bool $socialName The social network's name to check for, or false to check
+ * for any.
+ * @return bool
+ */
+ public function isSocialUrl($url, $socialName = false)
+ {
+ foreach ($this->getSocialDefinitions() as $domain => $name) {
+
+ if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) {
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+
+ /**
+ * Get's social network name from URL.
+ *
+ * @param string $url
+ * @return string
+ */
+ public function getSocialNetworkFromDomain($url)
+ {
+ foreach ($this->getSocialDefinitions() as $domain => $name) {
+
+ if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) {
+
+ return $name;
+ }
+ }
+
+ return Piwik::translate('General_Unknown');
+ }
+
+ /**
+ * Returns the main url of the social network the given url matches
+ *
+ * @param string $url
+ *
+ * @return string
+ */
+ public function getMainUrl($url)
+ {
+ $social = $this->getSocialNetworkFromDomain($url);
+ foreach ($this->getSocialDefinitions() as $domain => $name) {
+
+ if ($name == $social) {
+
+ return $domain;
+ }
+ }
+ return $url;
+ }
+
+
+ /**
+ * Return social network logo path by URL
+ *
+ * @param string $domain
+ * @return string path
+ * @see plugins/Referrers/images/socials/
+ */
+ public function getLogoFromUrl($domain)
+ {
+ $social = Social::getInstance()->getSocialNetworkFromDomain($domain);
+ $socialNetworks = Social::getInstance()->getSocialDefinitions();
+
+ $filePattern = 'plugins/Referrers/images/socials/%s.png';
+
+ $socialDomains = array_keys($socialNetworks, $social);
+ foreach ($socialDomains as $domain) {
+ if (file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domain))) {
+ return sprintf($filePattern, $domain);
+ }
+ }
+
+ return sprintf($filePattern, 'xx');
+ }
+}
diff --git a/plugins/Referrers/Tasks.php b/plugins/Referrers/Tasks.php
index 7481dbdca3..f22699632f 100644
--- a/plugins/Referrers/Tasks.php
+++ b/plugins/Referrers/Tasks.php
@@ -17,11 +17,11 @@ class Tasks extends \Piwik\Plugin\Tasks
public function schedule()
{
$this->weekly('updateSearchEngines');
- #$this->weekly('updateSocials');
+ $this->weekly('updateSocials');
}
/**
- * Update the search engine and social definitions
+ * Update the search engine definitions
*
* @see https://github.com/piwik/searchengine-and-social-definitions
*/
@@ -32,4 +32,17 @@ class Tasks extends \Piwik\Plugin\Tasks
$searchEngines = SearchEngine::getInstance()->loadYmlData($list);
Option::set(SearchEngine::OPTION_STORAGE_NAME, serialize($searchEngines));
}
+
+ /**
+ * Update the social definitions
+ *
+ * @see https://github.com/piwik/searchengine-and-social-definitions
+ */
+ public function updateSocials()
+ {
+ $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-definitions/master/Socials.yml';
+ $list = Http::sendHttpRequest($url, 30);
+ $searchEngines = SearchEngine::getInstance()->loadYmlData($list);
+ Option::set(SearchEngine::OPTION_STORAGE_NAME, serialize($searchEngines));
+ }
}
\ No newline at end of file
diff --git a/plugins/Referrers/functions.php b/plugins/Referrers/functions.php
index 2a39f8d1d4..bce573be7e 100644
--- a/plugins/Referrers/functions.php
+++ b/plugins/Referrers/functions.php
@@ -27,89 +27,6 @@ function getPathFromUrl($url)
return $path;
}
-/**
- * Returns the main url of the social network the given url matches
- *
- * @param string $url
- *
- * @return string
- */
-function getSocialMainUrl($url)
-{
- $social = getSocialNetworkFromDomain($url);
- foreach (Common::getSocialUrls() as $domain => $name) {
-
- if ($name == $social) {
-
- return $domain;
- }
- }
- return $url;
-}
-
-/**
- * Get's social network name from URL.
- *
- * @param string $url
- * @return string
- */
-function getSocialNetworkFromDomain($url)
-{
- foreach (Common::getSocialUrls() as $domain => $name) {
-
- if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) {
-
- return $name;
- }
- }
-
- return Piwik::translate('General_Unknown');
-}
-
-/**
- * Returns true if a URL belongs to a social network, false if otherwise.
- *
- * @param string $url The URL to check.
- * @param string|bool $socialName The social network's name to check for, or false to check
- * for any.
- * @return bool
- */
-function isSocialUrl($url, $socialName = false)
-{
- foreach (Common::getSocialUrls() as $domain => $name) {
-
- if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) {
-
- return true;
- }
- }
-
- return false;
-}
-
-/**
- * Return social network logo path by URL
- *
- * @param string $domain
- * @return string path
- * @see plugins/Referrers/images/socials/
- */
-function getSocialsLogoFromUrl($domain)
-{
- $social = getSocialNetworkFromDomain($domain);
- $socialNetworks = Common::getSocialUrls();
-
- $filePattern = 'plugins/Referrers/images/socials/%s.png';
-
- foreach ($socialNetworks as $domainKey => $name) {
- if ($social == $socialNetworks[$domainKey] && file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domainKey))) {
- return sprintf($filePattern, $domainKey);
- }
- }
-
- return sprintf($filePattern, 'xx');
-}
-
/**
* Return search engine URL by name
*
@@ -223,7 +140,6 @@ function getSearchEngineUrlFromKeywordAndUrl($keyword, $url)
*/
function getReferrerTypeLabel($label)
{
- $indexTranslation = '';
switch ($label) {
case Common::REFERRER_TYPE_DIRECT_ENTRY:
$indexTranslation = 'Referrers_DirectEntry';
diff --git a/plugins/Referrers/tests/Unit/ReferrersTest.php b/plugins/Referrers/tests/Unit/ReferrersTest.php
index 22e7dbe216..abd4214895 100644
--- a/plugins/Referrers/tests/Unit/ReferrersTest.php
+++ b/plugins/Referrers/tests/Unit/ReferrersTest.php
@@ -155,82 +155,6 @@ class ReferrersTest extends \PHPUnit_Framework_TestCase
$this->assertEquals($expected, \Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword($url, $keyword));
}
- /**
- * Dataprovider for getSocialNetworkFromDomainTestData
- */
- public function getSocialNetworkFromDomainTestData()
- {
- return array(
- array('http://www.facebook.com', 'Facebook'),
- array('http://www.facebook.com/piwik', 'Facebook'),
- array('http://m.facebook.com', 'Facebook'),
- array('https://m.facebook.com', 'Facebook'),
- array('m.facebook.com', 'Facebook'),
- array('http://lastfm.com.tr', 'Last.fm'),
- array('http://t.co/test', 'Twitter'),
- array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')),
- array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')),
- array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')),
- array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')),
- );
- }
-
- /**
- * @group Plugins
- *
- * @dataProvider getSocialNetworkFromDomainTestData
- */
- public function testGetSocialNetworkFromDomain($url, $expected)
- {
- include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
- $this->assertEquals($expected, \Piwik\Plugins\Referrers\getSocialNetworkFromDomain($url));
- }
-
- public function getSocialsLogoFromUrlTestData()
- {
- return array(
- array('http://www.facebook.com', 'facebook.com.png'),
- array('www.facebook.com', 'facebook.com.png',),
- array('http://lastfm.com.tr', 'last.fm.png'),
- array('http://asdfasdf.org/test', 'xx.png'),
- array('http://www.google.com', 'xx.png'),
- );
- }
-
- /**
- * @group Plugins
- *
- * @dataProvider getSocialsLogoFromUrlTestData
- */
- public function testGetSocialsLogoFromUrl($url, $expected)
- {
- include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
- $this->assertContains($expected, \Piwik\Plugins\Referrers\getSocialsLogoFromUrl($url));
- }
-
- public function isSocialUrlTestData()
- {
- return array(
- array('http://www.facebook.com', 'Facebook', true),
- array('http://www.facebook.com', 'Twitter', false),
- array('http://m.facebook.com', false, true),
- array('http://lastfm.com.tr', 'Last.fm', true),
- array('http://asdfasdf.org/test', false, false),
- array('http://asdfasdf.com/test', 'Facebook', false),
- );
- }
-
- /**
- * @group Plugins
- *
- * @dataProvider isSocialUrlTestData
- */
- public function testIsSocialUrl($url, $assumedSocial, $expected)
- {
- include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
- $this->assertEquals($expected, \Piwik\Plugins\Referrers\isSocialUrl($url, $assumedSocial));
- }
-
public function removeUrlProtocolTestData()
{
return array(
diff --git a/plugins/Referrers/tests/Unit/SocialTest.php b/plugins/Referrers/tests/Unit/SocialTest.php
new file mode 100644
index 0000000000..bc277b314e
--- /dev/null
+++ b/plugins/Referrers/tests/Unit/SocialTest.php
@@ -0,0 +1,100 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests;
+
+use Piwik\Plugins\Referrers\SearchEngine;
+use Piwik\Plugins\Referrers\Social;
+use Spyc;
+
+/**
+ * @group Social
+ * @group Plugins
+ */
+class SocialTest extends \PHPUnit_Framework_TestCase
+{
+ public static function setUpBeforeClass()
+ {
+ // inject definitions to avoid database usage
+ $yml = file_get_contents(PIWIK_INCLUDE_PATH . Social::DEFINITION_FILE);
+ Social::getInstance()->loadYmlData($yml);
+
+ parent::setUpBeforeClass();
+ }
+
+
+ public function isSocialUrlTestData()
+ {
+ return array(
+ array('http://www.facebook.com', 'Facebook', true),
+ array('http://www.facebook.com', 'Twitter', false),
+ array('http://m.facebook.com', false, true),
+ array('http://lastfm.com.tr', 'Last.fm', true),
+ array('http://asdfasdf.org/test', false, false),
+ array('http://asdfasdf.com/test', 'Facebook', false),
+ );
+ }
+
+ /**
+ * @dataProvider isSocialUrlTestData
+ */
+ public function testIsSocialUrl($url, $assumedSocial, $expected)
+ {
+ $this->assertEquals($expected, Social::getInstance()->isSocialUrl($url, $assumedSocial));
+ }
+
+
+ /**
+ * Dataprovider for getSocialNetworkFromDomainTestData
+ */
+ public function getSocialNetworkFromDomainTestData()
+ {
+ return array(
+ array('http://www.facebook.com', 'Facebook'),
+ array('http://www.facebook.com/piwik', 'Facebook'),
+ array('http://m.facebook.com', 'Facebook'),
+ array('https://m.facebook.com', 'Facebook'),
+ array('m.facebook.com', 'Facebook'),
+ array('http://lastfm.com.tr', 'Last.fm'),
+ array('http://t.co/test', 'Twitter'),
+ array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')),
+ array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')),
+ array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')),
+ array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')),
+ );
+ }
+
+ /**
+ * @dataProvider getSocialNetworkFromDomainTestData
+ */
+ public function testGetSocialNetworkFromDomain($url, $expected)
+ {
+ $this->assertEquals($expected, Social::getInstance()->getSocialNetworkFromDomain($url));
+ }
+
+ public function getLogoFromUrlTestData()
+ {
+ return array(
+ array('http://www.facebook.com', 'facebook.com.png'),
+ array('www.facebook.com', 'facebook.com.png',),
+ array('http://lastfm.com.tr', 'last.fm.png'),
+ array('http://asdfasdf.org/test', 'xx.png'),
+ array('http://www.google.com', 'xx.png'),
+ );
+ }
+
+ /**
+ * @group Plugins
+ *
+ * @dataProvider getLogoFromUrlTestData
+ */
+ public function testGetLogoFromUrl($url, $expected)
+ {
+ $this->assertContains($expected, Social::getInstance()->getLogoFromUrl($url));
+ }
+}
\ No newline at end of file
--
GitLab