Fixes #2268 Implement a default Referrer spam blacklist to block semalt.com...

Fixes #2268 Implement a default Referrer spam blacklist to block semalt.com spammer. (can be extended to other spammers on request!)

; All Visits with a Referrer URL host set to one of these will be excluded.
; If you find new spam entries in Referrers>Websites, please report them here: http://dev.piwik.org/trac/ticket/2268
; List of known Referrer Spammers, ie. bot visits that set a fake Referrer field:
referrer_urls_spam = "semalt.com"

config/global.ini.php

@@ -497,6 +497,11 @@ tracker_cache_file_ttl = 300
 ; Whether Bulk tracking requests to the Tracking API requires the token_auth to be set.
 bulk_requests_require_authentication = 0
 
+; All Visits with a Referrer URL host set to one of these will be excluded.
+; If you find new spam entries in Referrers>Websites, please report them here: http://dev.piwik.org/trac/ticket/2268
+; Comma separated list of known Referrer Spammers, ie. bot visits that set a fake Referrer field:
+referrer_urls_spam = "semalt.com"
+
 ; DO NOT USE THIS SETTING ON PUBLICLY AVAILABLE PIWIK SERVER
 ; !!! Security risk: if set to 0, it would allow anyone to push data to Piwik with custom dates in the past/future and even with fake IPs!
 ; When using the Tracking API, to override either the datetime and/or the visitor IP, 

core/Tracker/VisitExcluded.php

@@ -9,6 +9,7 @@
 namespace Piwik\Tracker;
 
 use Piwik\Common;
+use Piwik\Config;
 use Piwik\IP;
 use Piwik\Piwik;
 
@@ -241,4 +242,23 @@ class VisitExcluded
         }
         return false;
     }
+
+    /**
+     * Returns true if the Referrer is a known spammer.
+     *
+     * @return bool
+     */
+    protected function isReferrerSpamExcluded()
+    {
+        $spamHosts = Config::getInstance()->Tracker['referrer_urls_spam'];
+        $spamHosts = explode(",", $spamHosts);
+
+        $referrerUrl = $this->request->getParam('urlref');
+        foreach($spamHosts as $spamHost) {
+            if( strpos($referrerUrl, $spamHost) !== false) {
+                return true;
+            }
+        }
+        return false;
+    }
 }

tests/PHPUnit/Integration/Core/Tracker/VisitTest.php

@@ -136,7 +136,38 @@ class Core_Tracker_VisitTest extends DatabaseTestCase
         foreach ($tests as $ua => $expected) {
             $excluded = new VisitExcluded_public($request, $ip = false, $ua);
             
-            $this->assertSame($expected, $excluded->public_isUserAgentExcluded($ua), "Result if isUserAgentExcluded('$ua') was not " . ($expected ? 'true' : 'false') . ".");
+            $this->assertSame($expected, $excluded->public_isUserAgentExcluded(), "Result if isUserAgentExcluded('$ua') was not " . ($expected ? 'true' : 'false') . ".");
+        }
+    }
+
+    /**
+     * @group Core
+     * @group referrerIsKnownSpam
+     */
+    public function testIsVisitor_referrerIsKnownSpam()
+    {
+        $knownSpammers = array(
+            'http://semalt.com' => true,
+            'http://semalt.com/random/sub/page' => true,
+            'http://semalt.com/out/of/here?mate' => true,
+            'http://valid.domain/' => false,
+            'http://valid.domain/page' => false,
+        );
+        API::getInstance()->setSiteSpecificUserAgentExcludeEnabled(true);
+
+        $idsite = API::getInstance()->addSite("name", "http://piwik.net/");
+
+
+        // test that user agents that contain excluded user agent strings are excluded
+        foreach ($knownSpammers as $spamUrl => $expectedIsReferrerSpam) {
+            $spamUrl = urlencode($spamUrl);
+            $request = new Request(array(
+                'idsite' => $idsite,
+                'urlref' => $spamUrl
+            ));
+            $excluded = new VisitExcluded_public($request);
+
+            $this->assertSame($expectedIsReferrerSpam, $excluded->public_isReferrerSpamExcluded(), $spamUrl);
         }
     }
 }
@@ -148,8 +179,12 @@ class VisitExcluded_public extends VisitExcluded
         return $this->isVisitorIpExcluded($ip);
     }
 
-    public function public_isUserAgentExcluded($ua)
+    public function public_isUserAgentExcluded()
+    {
+        return $this->isUserAgentExcluded();
+    }
+    public function public_isReferrerSpamExcluded()
     {
-        return $this->isUserAgentExcluded($ua);
+        return $this->isReferrerSpamExcluded();
     }
 }