refs #6508 Now we will purge only the right monthly archive_numeric table and...

refs #6508 Now we will purge only the right monthly archive_numeric table and also only for the websites IDs that were invalidated

refs #6508 Now we will purge only the right monthly archive_numeric table and...
7999b36e · mattab · 07dabe24 · 7999b36e · 7999b36e · 7999b36e
--- a/core/DataAccess/ArchiveInvalidator.php
+++ b/core/DataAccess/ArchiveInvalidator.php
@@ -21,6 +21,8 @@ use Piwik\Period\Week;
 *
 * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
 *
+ * The invalidated archives will be deleted by ArchivePurger
+ *
 * @package Piwik\DataAccess
 */
 class ArchiveInvalidator {
@@ -37,7 +39,7 @@ class ArchiveInvalidator {
     * @return array
     * @throws \Exception
     */
-    public function markArchivesAsInvalidated($idSites, $dates, $period)
+    public function markArchivesAsInvalidated(array $idSites, $dates, $period)
    {
        $this->findOlderDateWithLogs();
        $datesToInvalidate = $this->getDatesToInvalidateFromString($dates);
@@ -45,10 +47,10 @@ class ArchiveInvalidator {
        \Piwik\Plugins\SitesManager\API::getInstance()->updateSiteCreatedTime($idSites, $minDate);
-        $this->markArchivesInvalidatedFor($idSites, $period, $datesToInvalidate);
+        $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate);
+        $this->markArchivesInvalidatedFor($idSites, $period, $datesByMonth);
-        $store = new InvalidatedReports();
+        $this->persistInvalidatedArchives($idSites, $datesByMonth);
-        $store->addInvalidatedSitesToReprocess($idSites);
        return $this->makeOutputLogs();
    }
@@ -79,15 +81,13 @@ class ArchiveInvalidator {
    /**
     * @param $idSites
     * @param $period string
-     * @param $datesToInvalidate Date[]
+     * @param $datesByMonth array
     * @throws \Exception
     */
-    private function markArchivesInvalidatedFor($idSites, $period, $datesToInvalidate)
+    private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth)
    {
        $invalidateForPeriodId = $this->getPeriodId($period);
-        $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate);
        // In each table, invalidate day/week/month/year containing this date
        $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
        foreach ($archiveTables as $table) {
@@ -161,10 +161,11 @@ class ArchiveInvalidator {
                && $date->isEarlier($this->minimumDateWithLogs)
            ) {
                $this->warningDates[] = $date->toString();
-            } else {
+                continue;
-                $this->processedDates[] = $date->toString();
            }
+            $this->processedDates[] = $date->toString();
            $month = $date->toString('Y_m');
            // For a given date, we must invalidate in the monthly archive table
            $datesByMonth[$month][] = $date->toString();
@@ -217,8 +218,23 @@ class ArchiveInvalidator {
        return $invalidateForPeriod;
    }
+    /**
+     * @param array $idSites
+     * @param $datesByMonth
+     */
+    private function persistInvalidatedArchives(array $idSites, $datesByMonth)
+    {
+        $yearMonths = array_keys($datesByMonth);
+        $yearMonths = array_unique($yearMonths);
+        $store = new InvalidatedReports();
+        $store->addInvalidatedSitesToReprocess($idSites);
+        $store->addSitesToPurgeForYearMonths($idSites, $yearMonths);
+    }
    private static function getModel()
    {
        return new Model();
    }
 } 
\ No newline at end of file
--- a/core/DataAccess/ArchivePurger.php
+++ b/core/DataAccess/ArchivePurger.php
@@ -16,7 +16,13 @@ use Piwik\Log;
 use Piwik\Piwik;
 /**
- * Cleans up outdated archives
+ *
+ * This class purges two types of archives:
+ *
+ * (1) Deletes invalidated archives (from ArchiveInvalidator)
+ *
+ * (2) Deletes outdated archives (the temporary or errored archives)
+ *
 *
 * @package Piwik\DataAccess
 */
@@ -24,35 +30,26 @@ class ArchivePurger
 {
    public static function purgeInvalidatedArchives()
    {
-        $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
+        $store = new InvalidatedReports();
+        $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge();
-        foreach ($archiveTables as $archiveTable) {
-            /**
+        foreach ($idSitesByYearMonth as $yearMonth => $idSites) {
-             * Select the archives that have already been invalidated and have been since re-processed.
+            if(empty($idSites)) {
-             * It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
+                continue;
-             */
-            $result = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($archiveTable);
-            if (count($result) > 0) {
-                $archiveIds = array_map(
-                    function ($elm) {
-                        return $elm['idarchive'];
-                    },
-                    $result
-                );
-                $date = ArchiveTableCreator::getDateFromTableName($archiveTable);
-                $date = Date::factory(str_replace('_', '-', $date) . '-01');
-                self::deleteArchiveIds($date, $archiveIds);
            }
-        }
+            $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
-    }
+            $numericTable = ArchiveTableCreator::getNumericTable($date);
-    private static function getModel()
+            $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
-    {
-        return new Model();
+            if (count($archiveIds) == 0) {
+                continue;
+            }
+            self::deleteArchiveIds($date, $archiveIds);
+            $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth);
+        }
    }
    /**
@@ -133,4 +130,9 @@ class ArchivePurger
        }
    }
+    private static function getModel()
+    {
+        return new Model();
+    }
 }
--- a/core/DataAccess/InvalidatedReports.php
+++ b/core/DataAccess/InvalidatedReports.php
@@ -10,7 +10,6 @@
 namespace Piwik\DataAccess;
 use Piwik\Option;
-use Piwik\Piwik;
 /**
 * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API.
@@ -25,19 +24,62 @@ use Piwik\Piwik;
 class InvalidatedReports
 {
    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
+    const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
+    /**
+     * Mark the sites IDs and Dates as being invalidated, so we can purge them later on.
+     *
+     * @param array $idSites
+     * @param array $yearMonths
+     */
+    public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths)
+    {
+        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+        foreach($yearMonths as $yearMonth) {
+            $idSitesByYearMonth[$yearMonth] = $idSites;
+        }
+        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
+    }
+    /**
+     * Returns the list of websites IDs for which invalidated archives can be purged.
+     */
+    public function getSitesByYearMonthArchiveToPurge()
+    {
+        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+        // From this list we remove the websites that are not yet re-processed
+        // so we don't purge them before they were re-processed
+        $idSitesNotYetReprocessed = $this->getSitesToReprocess();
+        foreach($idSitesByYearMonth as $yearMonth => &$idSites) {
+            $idSites = array_diff($idSites, $idSitesNotYetReprocessed);
+        }
+        return $idSitesByYearMonth;
+    }
+    public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth)
+    {
+        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+        if(!isset($idSitesByYearMonth[$yearMonth])) {
+            return;
+        }
+        $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites);
+        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
+    }
    /**
     * Record those website IDs as having been invalidated
     *
     * @param $idSites
     */
-    public function addInvalidatedSitesToReprocess($idSites)
+    public function addInvalidatedSitesToReprocess(array $idSites)
    {
-        $invalidatedIdSites = $this->getSitesToReprocess();
+        $siteIdsToReprocess = $this->getSitesToReprocess();
-        $invalidatedIdSites = array_merge($invalidatedIdSites, $idSites);
+        $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites);
-        $invalidatedIdSites = array_unique($invalidatedIdSites);
+        $this->setSitesToReprocess($siteIdsToReprocess);
-        $invalidatedIdSites = array_values($invalidatedIdSites);
-        $this->setSitesToReprocess($invalidatedIdSites);
    }
@@ -46,13 +88,13 @@ class InvalidatedReports
     */
    public function storeSiteIsReprocessed($idSite)
    {
-        $websiteIdsInvalidated = $this->getSitesToReprocess();
+        $siteIdsToReprocess = $this->getSitesToReprocess();
-        if (count($websiteIdsInvalidated)) {
+        if (count($siteIdsToReprocess)) {
-            $found = array_search($idSite, $websiteIdsInvalidated);
+            $found = array_search($idSite, $siteIdsToReprocess);
            if ($found !== false) {
-                unset($websiteIdsInvalidated[$found]);
+                unset($siteIdsToReprocess[$found]);
-                $this->setSitesToReprocess($websiteIdsInvalidated);
+                $this->setSitesToReprocess($siteIdsToReprocess);
            }
        }
    }
@@ -64,25 +106,56 @@ class InvalidatedReports
     */
    public function getSitesToReprocess()
    {
-        Option::clearCachedOption(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
-        $invalidatedIdSites = Option::get(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+    }
+    /**
+     * @return array|false|mixed|string
+     */
+    private function getSitesByYearMonthToPurge()
+    {
+        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
+    }
+    /**
+     * @param $websiteIdsInvalidated
+     */
+    private function setSitesToReprocess($websiteIdsInvalidated)
+    {
+        $websiteIdsInvalidated = array_unique($websiteIdsInvalidated);
+        $websiteIdsInvalidated = array_values($websiteIdsInvalidated);
+        Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+    }
+    /**
+     * @param $optionName
+     * @return array|false|mixed|string
+     */
+    private function getArrayValueFromOptionName($optionName)
+    {
+        Option::clearCachedOption($optionName);
+        $array = Option::get($optionName);
-        if ($invalidatedIdSites
+        if ($array
-            && ($invalidatedIdSites = unserialize($invalidatedIdSites))
+            && ($array = unserialize($array))
-            && count($invalidatedIdSites)
+            && count($array)
        ) {
-            return $invalidatedIdSites;
+            return $array;
        }
        return array();
    }
    /**
-     * @param $websiteIdsInvalidated
+     * @param $idSitesByYearMonth
     */
-    private function setSitesToReprocess($websiteIdsInvalidated)
+    private function persistSitesByYearMonthToPurge($idSitesByYearMonth)
    {
-        Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+        // remove dates for which there are no sites to purge
+        $idSitesByYearMonth = array_filter($idSitesByYearMonth);
+        Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth));
    }
 }
\ No newline at end of file
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -27,26 +27,39 @@ class Model
     *
     * These archives { archive name (includes segment hash) , idsite, date, period } will be deleted.
     *
-     * @param $archiveTable
+     * @param string $archiveTable
+     * @param array $idSites
     * @return array
     * @throws Exception
     */
-    public function getInvalidatedArchiveIdsSafeToDelete($archiveTable)
+    public function getInvalidatedArchiveIdsSafeToDelete($archiveTable, array $idSites)
    {
        // prevent error 'The SELECT would examine more than MAX_JOIN_SIZE rows'
        Db::get()->query('SET SQL_BIG_SELECTS=1');
        $query = 'SELECT t1.idarchive FROM `' . $archiveTable . '` t1
                  INNER JOIN `' . $archiveTable . '` t2
-                      ON t1.name = t2.name AND t1.idsite=t2.idsite
+                      ON t1.name    = t2.name
-                      AND t1.date1=t2.date1 AND t1.date2=t2.date2 AND t1.period=t2.period
+                      AND t1.idsite = t2.idsite
+                      AND t1.date1  = t2.date1
+                      AND t1.date2  = t2.date2
+                      AND t1.period = t2.period
                  WHERE t1.value = ' . ArchiveWriter::DONE_INVALIDATED . '
+                  AND t1.idsite IN (' . implode(",", $idSites) . ')
                  AND t2.value IN(' . ArchiveWriter::DONE_OK . ', ' . ArchiveWriter::DONE_OK_TEMPORARY . ')
-                  AND t1.ts_archived < t2.ts_archived AND t1.name LIKE \'done%\'';
+                  AND t1.ts_archived < t2.ts_archived
+                  AND t1.name LIKE \'done%\'
+        ';
        $result = Db::fetchAll($query);
-        return $result;
+        $archiveIds = array_map(
+            function ($elm) {
+                return $elm['idarchive'];
+            },
+            $result
+        );
+        return $archiveIds;
    }
    /**

--- a/plugins/Annotations/AnnotationList.php
+++ b/plugins/Annotations/AnnotationList.php
@@ -440,7 +440,7 @@ class AnnotationList
    public static function canUserAddNotesFor($idSite)
    {
        return Piwik::isUserHasViewAccess($idSite)
-        && !Piwik::isUserIsAnonymous($idSite);
+        && !Piwik::isUserIsAnonymous();
    }
    /**