Merge pull request #8345 from piwik/588_urls_whitelist_2

Fixes #588, add option to ignore actions w/ URLs that are not for the website during tracking.

core/Db/Schema/Mysql.php

@@ -92,6 +92,7 @@ class Mysql implements SchemaInterface
                             sitesearch_category_parameters TEXT NOT NULL,
                             timezone VARCHAR( 50 ) NOT NULL,
                             currency CHAR( 3 ) NOT NULL,
+                            exclude_unknown_urls TINYINT(1) DEFAULT 0,
                             excluded_ips TEXT NOT NULL,
                             excluded_parameters TEXT NOT NULL,
                             excluded_user_agents TEXT NOT NULL,

core/Tracker/VisitExcluded.php

@@ -128,6 +128,14 @@ class VisitExcluded
             }
         }
 
+        // Check if request URL is excluded
+        if (!$excluded) {
+            $excluded = $this->isUrlExcluded();
+            if ($excluded) {
+                Common::printDebug("Unknown URL is not allowed to track.");
+            }
+        }
+
         if (!$excluded) {
             if ($this->isPrefetchDetected()) {
                 $excluded = true;
@@ -274,6 +282,27 @@ class VisitExcluded
         return false;
     }
 
+    /**
+     * Checks if request URL is excluded
+     * @return bool
+     */
+    protected function isUrlExcluded()
+    {
+        $site = Cache::getCacheWebsiteAttributes($this->idSite);
+
+        if (!empty($site['exclude_unknown_urls']) && !empty($site['hosts'])) {
+            $trackingHost = parse_url($this->request->getParam('url'), PHP_URL_HOST);
+            foreach ($site['hosts'] as $siteHost) {
+                if ($trackingHost == $siteHost || (substr($trackingHost, -strlen($siteHost) - 1) === ('.' . $siteHost))) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * Returns true if the specified user agent should be excluded for the current site or not.
      *

core/Updates/2.15.0-b3.php

+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Updates;
+
+use Piwik\Common;
+use Piwik\Updater;
+use Piwik\Updates;
+
+
+class Updates_2_15_0_b3 extends Updates
+{
+    public function getMigrationQueries(Updater $updater)
+    {
+        $updateSql = array(
+            'ALTER TABLE `' . Common::prefixTable('site')
+                . '` ADD COLUMN `exclude_unknown_urls` TINYINT(1) DEFAULT 0 AFTER `currency`' => array(1060)
+        );
+        return $updateSql;
+    }
+
+    public function doUpdate(Updater $updater)
+    {
+        $updater->executeMigrationQueries(__FILE__, $this->getMigrationQueries($updater));
+    }
+}

core/Version.php

@@ -20,7 +20,7 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '2.15.0-b2';
+    const VERSION = '2.15.0-b3';
 
     public function isStableVersion($version)
     {

plugins/SitesManager/API.php

@@ -506,6 +506,7 @@ class API extends \Piwik\Plugin\API
      * @param array|null $settings JSON serialized settings eg {settingName: settingValue, ...}
      * @see getKeepURLFragmentsGlobal.
      * @param string $type The website type, defaults to "website" if not set.
+     * @param bool|null $excludeUnknownUrls Track only URL matching one of website URLs
      *
      * @return int the website ID created
      */
@@ -524,7 +525,8 @@ class API extends \Piwik\Plugin\API
                             $excludedUserAgents = null,
                             $keepURLFragments = null,
                             $type = null,
-                            $settings = null)
+                            $settings = null,
+                            $excludeUnknownUrls = null)
     {
         Piwik::checkUserHasSuperUserAccess();
 
@@ -555,6 +557,7 @@ class API extends \Piwik\Plugin\API
         $bind = array('name'     => $siteName,
                       'main_url' => $url);
 
+        $bind['exclude_unknown_urls'] = (int)$excludeUnknownUrls;
         $bind['excluded_ips'] = $this->checkAndReturnExcludedIps($excludedIps);
         $bind['excluded_parameters']  = $this->checkAndReturnCommaSeparatedStringList($excludedQueryParameters);
         $bind['excluded_user_agents'] = $this->checkAndReturnCommaSeparatedStringList($excludedUserAgents);
@@ -1087,6 +1090,7 @@ class API extends \Piwik\Plugin\API
      *                                   will be removed. If 0, the default global behavior will be used.
      * @param string $type The Website type, default value is "website"
      * @param array|null $settings JSON serialized settings eg {settingName: settingValue, ...}
+     * @param bool|null $excludeUnknownUrls Track only URL matching one of website URLs
      * @throws Exception
      * @see getKeepURLFragmentsGlobal. If null, the existing value will
      *                                   not be modified.
@@ -1109,7 +1113,8 @@ class API extends \Piwik\Plugin\API
                                $excludedUserAgents = null,
                                $keepURLFragments = null,
                                $type = null,
-                               $settings = null)
+                               $settings = null,
+                               $excludeUnknownUrls = null)
     {
         Piwik::checkUserHasAdminAccess($idSite);
 
@@ -1156,6 +1161,7 @@ class API extends \Piwik\Plugin\API
         if (!is_null($startDate)) {
             $bind['ts_created'] = Date::factory($startDate)->getDatetime();
         }
+        $bind['exclude_unknown_urls'] = (int)$excludeUnknownUrls;
         $bind['excluded_ips'] = $this->checkAndReturnExcludedIps($excludedIps);
         $bind['excluded_parameters'] = $this->checkAndReturnCommaSeparatedStringList($excludedQueryParameters);
         $bind['excluded_user_agents'] = $this->checkAndReturnCommaSeparatedStringList($excludedUserAgents);

plugins/SitesManager/SitesManager.php

@@ -117,6 +117,7 @@ class SitesManager extends \Piwik\Plugin
         $array['hosts'] = $this->getTrackerHosts($idSite);
 
         $website = API::getInstance()->getSiteFromId($idSite);
+        $array['exclude_unknown_urls'] = $website['exclude_unknown_urls'];
         $array['excluded_ips'] = $this->getTrackerExcludedIps($website);
         $array['excluded_parameters'] = self::getTrackerExcludedQueryParameters($website);
         $array['excluded_user_agents'] = self::getExcludedUserAgents($website);
@@ -286,6 +287,8 @@ class SitesManager extends \Piwik\Plugin
         $translationKeys[] = "SitesManager_Currency";
         $translationKeys[] = "SitesManager_ShowTrackingTag";
         $translationKeys[] = "SitesManager_AliasUrlHelp";
+        $translationKeys[] = "SitesManager_OnlyMatchedUrlsAllowed";
+        $translationKeys[] = "SitesManager_OnlyMatchedUrlsAllowedHelp";
         $translationKeys[] = "SitesManager_KeepURLFragmentsLong";
         $translationKeys[] = "SitesManager_HelpExcludedIps";
         $translationKeys[] = "SitesManager_ListOfQueryParametersToExclude";

plugins/SitesManager/angularjs/sites-manager/sites-manager-site.controller.js

@@ -111,6 +111,7 @@
                 searchKeywordParameters: sendSiteSearchKeywordParams ? $scope.site.sitesearch_keyword_parameters.join(',') : null,
                 searchCategoryParameters: sendSearchCategoryParameters ? $scope.site.sitesearch_category_parameters.join(',') : null,
                 urls: $scope.site.alias_urls,
+                excludeUnknownUrls: $scope.site.exclude_unknown_urls,
                 settings: flatSettings
             }, 'POST');
 
@@ -133,6 +134,7 @@
                 "http://siteUrl.com/",
                 "http://siteUrl2.com/"
             ];
+            $scope.site.exclude_unknown_urls = 0;
             $scope.site.keep_url_fragment = "0";
             $scope.site.excluded_ips = [];
             $scope.site.excluded_parameters = [];

plugins/SitesManager/lang/en.json

@@ -74,6 +74,8 @@
         "TrackingTags": "Tracking code for %s",
         "Urls": "URLs",
         "UTCTimeIs": "UTC time is %s.",
+        "OnlyMatchedUrlsAllowed": "Only track visits and actions when the action URL starts with one of the above URLs.",
+        "OnlyMatchedUrlsAllowedHelp": "When enabled, Piwik will only track internal actions when the Page URL is one of the known URLs for your website. This prevents people from spamming your analytics with URLs for other websites.",
         "WebsitesManagement": "Websites Management",
         "XManagement": "Manage %s",
         "ChooseMeasurableTypeHeadline": "What would you like to measure?",

plugins/SitesManager/templates/sites-list/site-fields.html

@@ -82,6 +82,13 @@
                 {{ 'SitesManager_AliasUrlHelp' | translate }}
             </div>
             <div sites-manager-multiline-field field="site.alias_urls" cols="25" rows="3"></div>
+
+            <div class="form-help">
+                {{ 'SitesManager_OnlyMatchedUrlsAllowedHelp' | translate }}
+            </div>
+            <label class="checkbox">
+                <input type="checkbox" ng-model="site.exclude_unknown_urls" ng-true-value="1" ng-false-value="0"> {{ 'SitesManager_OnlyMatchedUrlsAllowed' | translate:'':'' }}
+            </label>
         </div>
 
         <div class="form-group">

plugins/SitesManager/tests/System/expected/test_SitesManager__SitesManager.getPatternMatchSites.xml

@@ -11,6 +11,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -29,6 +30,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -47,6 +49,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -65,6 +68,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -83,6 +87,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -101,6 +106,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -119,6 +125,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -137,6 +144,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -155,6 +163,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -173,6 +182,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -191,6 +201,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />

plugins/SitesManager/tests/System/expected/test_SitesManagerwithLimit__SitesManager.getPatternMatchSites.xml

@@ -11,6 +11,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />
@@ -29,6 +30,7 @@
 		<sitesearch_category_parameters />
 		<timezone>UTC</timezone>
 		<currency>USD</currency>
+		<exclude_unknown_urls>0</exclude_unknown_urls>
 		<excluded_ips />
 		<excluded_parameters />
 		<excluded_user_agents />

tests/PHPUnit/Fixtures/InvalidVisits.php

@@ -97,6 +97,21 @@ class InvalidVisits extends Fixture
             self::checkResponse($t->doTrackPageView('visit from IP globally excluded'));
         }
 
+        // test unknown url exclusion works
+        $urls = array("http://piwik.net", "http://my.stuff.com/");
+        API::getInstance()->updateSite($idSite, $siteName = null, $urls, $ecommerce = null, $siteSearch = null,
+            $searchKeywordParameters = null, $searchCategoryParameters = null, $excludedIps = null, $excludedQueryParams = null,
+            $timezone = null, $currency = null, $group = null, $startDate = null, $excludedUserAgents = null,
+            $keepUrlFragments = null, $type = null, $settings = null, $excludeUnknownUrls = 1);
+
+        $t->setIp("125.4.5.6");
+
+        $t->setUrl("http://piwik.com/to/the/moon");
+        $t->doTrackPageView("ignored, not from piwik.net");
+
+        $t->setUrl("http://their.stuff.com/back/to/the/future");
+        $t->doTrackPageView("ignored, not from my.stuff.com");
+
         try {
             @$t->setAttributionInfo(array());
             self::fail();

tests/PHPUnit/Integration/Tracker/VisitTest.php

@@ -99,6 +99,53 @@ class VisitTest extends IntegrationTestCase
         }
     }
 
+    public function getExcludeByUrlData()
+    {
+        return array(
+            array(array('http://test.com'), true, array(
+                'http://test.com' => true,
+                'https://test.com' => true,
+                'http://test.com/uri' => true,
+                'http://test.com/?query' => true,
+                'http://xtest.com' => false,
+            )),
+            array(array('http://test.com', 'http://localhost'), true, array(
+                'http://test.com' => true,
+                'http://localhost' => true,
+                'http://x.com' => false,
+            )),
+            array(array('http://test.com'), false, array(
+                'http://x.com' => true,
+            )),
+            array(array('http://test.com', 'http://sub.test2.com'), true, array(
+                'http://sub.test.com' => true,
+                'http://sub.sub.test.com' => true,
+                'http://subtest.com' => false,
+                'http://test.com.org' => false,
+                'http://sub.test2.com' => true,
+                'http://x.sub.test2.com' => true,
+                'http://xsub.test2.com' => false,
+                'http://sub.test2.com.org' => false,
+            )),
+        );
+    }
+
+    /**
+     * @dataProvider getExcludeByUrlData
+     */
+    public function testExcludeByUrl($siteUrls, $excludeUnknownUrls, array $urlsTracked)
+    {
+        $siteId = API::getInstance()->addSite('name', $siteUrls, $ecommerce = null, $siteSearch = null, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, null, null, null, null, null, null, null, null, $excludeUnknownUrls);
+        foreach ($urlsTracked as $url => $isTracked) {
+            $visitExclude = new VisitExcluded(new Request(array(
+                'idsite' => $siteId,
+                'rec'    => 1,
+                'url'    => $url
+            )));
+            $this->assertEquals($isTracked, !$visitExclude->isExcluded());
+        }
+    }
+
     /**
      * @dataProvider getChromeDataSaverData
      */