Merge pull request #6856 from piwik/6672_preprocess_ranges

during core:archive console command, preprocess any custom date ranges that were selected in any User's Preferences

core/CronArchive.php

@@ -12,10 +12,13 @@ use Exception;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\SharedSiteIds;
+use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
 use Piwik\Period\Factory as PeriodFactory;
 use Piwik\DataAccess\InvalidatedReports;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
+use Piwik\Plugins\UsersManager\API as APIUsersManager;
+use Piwik\Plugins\UsersManager\UserPreferences;
 
 /**
  * ./console core:archive runs as a cron and is a useful tool for general maintenance,
@@ -251,7 +254,7 @@ class CronArchive
         $this->allWebsites = APISitesManager::getInstance()->getAllSitesId();
 
         if (!empty($this->shouldArchiveOnlySpecificPeriods)) {
-            $this->log("- Will process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)");
+            $this->log("- Will only process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)");
         }
 
         $websitesIds = $this->initWebsiteIds();
@@ -503,7 +506,14 @@ class CronArchive
             return false;
         }
 
-        $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite);
+        try {
+            $shouldProceed = $this->processArchiveDays($idSite, $lastTimestampWebsiteProcessedDay, $shouldArchivePeriods, $timerWebsite);
+        } catch(UnexpectedWebsiteFoundException $e) {
+            // this website was deleted in the meantime
+            $shouldProceed = false;
+            $this->log("Skipped website id $idSite, got: UnexpectedWebsiteFoundException, " . $timerWebsite->__toString());
+        }
+
         if (!$shouldProceed) {
             return false;
         }
@@ -517,18 +527,8 @@ class CronArchive
             return false;
         }
 
-        $success = true;
-        foreach (array('week', 'month', 'year') as $period) {
-
-            if (!$this->shouldProcessPeriod($period)) {
-                // if any period was skipped, we do not mark the Periods archiving as successful
-                $success = false;
-                continue;
-            }
+        $success = $this->processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods);
 
-            $success = $this->archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessedPeriods)
-                && $success;
-        }
         // Record succesful run of this website's periods archiving
         if ($success) {
             Option::set($this->lastRunKey($idSite, "periods"), time());
@@ -555,6 +555,37 @@ class CronArchive
         return true;
     }
 
+    /**
+     * @param $idSite
+     * @param $lastTimestampWebsiteProcessedPeriods
+     * @return bool
+     */
+    private function processArchiveForPeriods($idSite, $lastTimestampWebsiteProcessedPeriods)
+    {
+        $success = true;
+
+        foreach (array('week', 'month', 'year') as $period) {
+
+            if (!$this->shouldProcessPeriod($period)) {
+                // if any period was skipped, we do not mark the Periods archiving as successful
+                $success = false;
+                continue;
+            }
+
+            $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessedPeriods);
+            $periodArchiveWasSuccessful = $this->archiveVisitsAndSegments($idSite, $period, $date);
+            $success = $periodArchiveWasSuccessful && $success;
+        }
+
+        // period=range
+        $customDateRangesToPreProcessForSite = $this->getCustomDateRangeToPreProcess($idSite);
+        foreach ($customDateRangesToPreProcessForSite as $dateRange) {
+            $periodArchiveWasSuccessful = $this->archiveVisitsAndSegments($idSite, 'range', $dateRange);
+            $success = $periodArchiveWasSuccessful && $success;
+        }
+        return $success;
+    }
+
     /**
      * Checks the config file is found.
      *
@@ -684,7 +715,8 @@ class CronArchive
 
         $this->visitsToday += $visitsToday;
         $this->websitesWithVisitsSinceLastRun++;
-        $this->archiveVisitsAndSegments($idSite, "day", $processDaysSince);
+
+        $this->archiveVisitsAndSegments($idSite, "day", $this->getApiDateParameter($idSite, "day", $processDaysSince));
         $this->logArchivedWebsite($idSite, "day", $date, $visitsLastDays, $visitsToday, $timerWebsite);
 
         return true;
@@ -707,17 +739,16 @@ class CronArchive
      * Requests are triggered using cURL multi handle
      *
      * @param $idSite int
-     * @param $period
-     * @param $lastTimestampWebsiteProcessed
+     * @param $period string
+     * @param $date string
      * @return bool True on success, false if some request failed
      */
-    private function archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessed)
+    private function archiveVisitsAndSegments($idSite, $period, $date)
     {
         $timer = new Timer();
 
         $url  = $this->piwikUrl;
 
-        $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed);
         $url .= $this->getVisitsRequestUrl($idSite, $period, $date);
 
         $url .= self::APPEND_TO_API_REQUEST;
@@ -756,6 +787,12 @@ class CronArchive
                     $this->logError("Error unserializing the following response from $url: " . $content);
                 }
 
+                if($period == 'range') {
+                    // range returns one dataset (the sum of data between the two dates),
+                    // whereas other periods return lastN which is N datasets in an array. Here we make our period=range dataset look like others:
+                    $stats = array($stats);
+                }
+
                 $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats);
                 $visitsLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats);
             }
@@ -1281,7 +1318,7 @@ class CronArchive
      */
     private function logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsToday, Timer $timer)
     {
-        if (substr($date, 0, 4) === 'last') {
+        if (strpos($date, 'last') === 0 || strpos($date, 'previous') === 0) {
             $visitsInLastPeriods = (int)$visitsInLastPeriods . " visits in last " . $date . " " . $period . "s, ";
             $thisPeriod = $period == "day" ? "today" : "this " . $period;
             $visitsInLastPeriod = (int)$visitsToday . " visits " . $thisPeriod . ", ";
@@ -1361,7 +1398,8 @@ class CronArchive
             $dateLastMax = self::DEFAULT_DATE_LAST_WEEKS;
         }
         if (empty($lastTimestampWebsiteProcessed)) {
-            $lastTimestampWebsiteProcessed = strtotime(\Piwik\Site::getCreationDateFor($idSite));
+            $creationDateFor = \Piwik\Site::getCreationDateFor($idSite);
+            $lastTimestampWebsiteProcessed = strtotime($creationDateFor);
         }
 
         // Enforcing last2 at minimum to work around timing issues and ensure we make most archives available
@@ -1423,4 +1461,58 @@ class CronArchive
         $now = time();
         return ($timestamp < $now) ? $timestamp : $now;
     }
+
+    /**
+     * @param $idSite
+     * @return array of date strings
+     */
+    private function getCustomDateRangeToPreProcess($idSite)
+    {
+        static $cache = null;
+        if(is_null($cache)) {
+            $cache = $this->loadCustomDateRangeToPreProcess();
+        }
+        if(empty($cache[$idSite])) {
+            return array();
+        }
+        $dates = array_unique($cache[$idSite]);
+        return $dates;
+    }
+
+    /**
+     * @return array
+     */
+    private function loadCustomDateRangeToPreProcess()
+    {
+        $customDateRangesToProcessForSites = array();
+        // For all users who have selected this website to load by default,
+        // we load the default period/date that will be loaded for this user
+        // and make sure it's pre-archived
+        $userPreferences = APIUsersManager::getInstance()->getAllUsersPreferences(array(APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE, APIUsersManager::PREFERENCE_DEFAULT_REPORT));
+        foreach ($userPreferences as $userLogin => $userPreferences) {
+
+            $defaultDate = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE];
+            $preference = new UserPreferences();
+            $period = $preference->getDefaultPeriod($defaultDate);
+            if ($period != 'range') {
+                continue;
+            }
+
+            $defaultReport = $userPreferences[APIUsersManager::PREFERENCE_DEFAULT_REPORT];
+            if (is_numeric($defaultReport)) {
+                // If user selected one particular website ID
+                $idSites = array($defaultReport);
+            } else {
+                // If user selected "All websites"  or some other random value, we pre-process all websites that he has access to
+                $idSites = APISitesManager::getInstance()->getSitesIdWithAtLeastViewAccess($userLogin);
+            }
+
+            foreach ($idSites as $idSite) {
+                $customDateRangesToProcessForSites[$idSite][] = $defaultDate;
+            }
+        }
+
+        return $customDateRangesToProcessForSites;
+    }
+
 }

core/Plugin/Controller.php

@@ -31,7 +31,6 @@ use Piwik\Piwik;
 use Piwik\Plugins\CoreAdminHome\CustomLogo;
 use Piwik\Plugins\CoreVisualizations\Visualizations\JqplotGraph\Evolution;
 use Piwik\Plugins\LanguagesManager\LanguagesManager;
-use Piwik\Plugins\UsersManager\UserPreferences;
 use Piwik\Registry;
 use Piwik\SettingsPiwik;
 use Piwik\Site;

core/Plugin/Menu.php

@@ -195,7 +195,7 @@ class Menu
             $defaultDate = $userPreferences->getDefaultDate();
         }
         if (empty($defaultPeriod)) {
-            $defaultPeriod = $userPreferences->getDefaultPeriod();
+            $defaultPeriod = $userPreferences->getDefaultPeriod($defaultDate);
         }
         return array(
             'idSite' => $websiteId,

core/Site.php

@@ -231,8 +231,11 @@ class Site
      */
     protected function get($name)
     {
+        if (!isset(self::$infoSites[$this->id])) {
+            throw new UnexpectedWebsiteFoundException('The requested website id = ' . (int)$this->id . ' couldn\'t be found');
+        }
         if (!isset(self::$infoSites[$this->id][$name])) {
-            throw new Exception('The requested website id = ' . (int)$this->id . ' (or its property ' . $name . ') couldn\'t be found');
+            throw new Exception("The property $name could not be found on the website ID " . (int)$this->id);
         }
         return self::$infoSites[$this->id][$name];
     }

plugins/UsersManager/API.php

@@ -32,6 +32,8 @@ use Piwik\Tracker\Cache;
  */
 class API extends \Piwik\Plugin\API
 {
+    const OPTION_NAME_PREFERENCE_SEPARATOR = '_';
+
     /**
      * @var Model
      */
@@ -107,9 +109,34 @@ class API extends \Piwik\Plugin\API
         return $this->getDefaultUserPreference($preferenceName, $userLogin);
     }
 
+    /**
+     * Returns an array of Preferences
+     * @param $preferenceNames array of preference names
+     * @return array
+     * @ignore
+     */
+    public function getAllUsersPreferences(array $preferenceNames)
+    {
+        Piwik::checkUserHasSuperUserAccess();
+
+        $userPreferences = array();
+        foreach($preferenceNames as $preferenceName) {
+            $optionNameMatchAllUsers = $this->getPreferenceId('%', $preferenceName);
+            $preferences = Option::getLike($optionNameMatchAllUsers);
+
+            foreach($preferences as $optionName => $optionValue) {
+                $optionName = explode(self::OPTION_NAME_PREFERENCE_SEPARATOR, $optionName);
+                $userName = $optionName[0];
+                $preference = $optionName[1];
+                $userPreferences[$userName][$preference] = $optionValue;
+            }
+        }
+        return $userPreferences;
+    }
+
     private function getPreferenceId($login, $preference)
     {
-        return $login . '_' . $preference;
+        return $login . self::OPTION_NAME_PREFERENCE_SEPARATOR . $preference;
     }
 
     private function getDefaultUserPreference($preferenceName, $login)

plugins/UsersManager/UserPreferences.php

@@ -39,6 +39,7 @@ class UserPreferences
 
         return false;
     }
+
     /**
      * Returns default site ID that Piwik should load.
      *
@@ -91,27 +92,32 @@ class UserPreferences
     /**
      * Returns default period type for Piwik reports.
      *
+     * @param $defaultDate string the default date string from which the default period will be guessed
      * @return string `'day'`, `'week'`, `'month'`, `'year'` or `'range'`
      * @api
      */
-    public function getDefaultPeriod()
+    public function getDefaultPeriod($defaultDate)
     {
-        $userSettingsDate = APIUsersManager::getInstance()->getUserPreference(Piwik::getCurrentUserLogin(), APIUsersManager::PREFERENCE_DEFAULT_REPORT_DATE);
-
-        if ($userSettingsDate === false) {
-            return Config::getInstance()->General['default_period'];
+        $defaultPeriod = Config::getInstance()->General['default_period'];
+        if ($defaultDate === false) {
+            return $defaultPeriod;
         }
 
-        if (in_array($userSettingsDate, array('today', 'yesterday'))) {
+        if (in_array($defaultDate, array('today', 'yesterday'))) {
             return 'day';
         }
 
-        if (strpos($userSettingsDate, 'last') === 0
-            || strpos($userSettingsDate, 'previous') === 0
+        if (strpos($defaultDate, 'last') === 0
+            || strpos($defaultDate, 'previous') === 0
         ) {
             return 'range';
         }
 
-        return $userSettingsDate;
+        return $defaultPeriod;
+    }
+
+    public function getAllUsersPreferences()
+    {
+
     }
 }
\ No newline at end of file

plugins/UsersManager/tests/Integration/APITest.php

@@ -69,4 +69,43 @@ class APITest extends IntegrationTestCase
         $this->assertFalse($eventTriggered, 'UsersManager.removeSiteAccess event was triggered but should not');
     }
 
+    public function test_getAllUsersPreferences_isEmpty_whenNoPreference()
+    {
+        $preferences = $this->api->getAllUsersPreferences(array('preferenceName'));
+        $this->assertEmpty($preferences);
+    }
+
+    public function test_getAllUsersPreferences_isEmpty_whenNoPreferenceAndMultipleRequested()
+    {
+        $preferences = $this->api->getAllUsersPreferences(array('preferenceName', 'otherOne'));
+        $this->assertEmpty($preferences);
+    }
+
+    public function test_getAllUsersPreferences_shouldGetMultiplePreferences()
+    {
+        $user2 = 'userLogin2';
+        $user3 = 'userLogin3';
+        $this->api->addUser($user2, 'password', 'userlogin2@password.de');
+        $this->api->setUserPreference($user2, 'myPreferenceName', 'valueForUser2');
+        $this->api->setUserPreference($user2, 'Random_NOT_REQUESTED', 'Random_NOT_REQUESTED');
+
+        $this->api->addUser($user3, 'password', 'userlogin3@password.de');
+        $this->api->setUserPreference($user3, 'myPreferenceName', 'valueForUser3');
+        $this->api->setUserPreference($user3, 'otherPreferenceHere', 'otherPreferenceVALUE');
+        $this->api->setUserPreference($user3, 'Random_NOT_REQUESTED', 'Random_NOT_REQUESTED');
+
+        $expected = array(
+            $user2 => array(
+                'myPreferenceName' => 'valueForUser2'
+            ),
+            $user3 => array(
+                'myPreferenceName' => 'valueForUser3',
+                'otherPreferenceHere' => 'otherPreferenceVALUE',
+            ),
+        );
+        $result = $this->api->getAllUsersPreferences(array('myPreferenceName', 'otherPreferenceHere', 'randomDoesNotExist'));
+
+        $this->assertSame($expected, $result);
+    }
+
 }