diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php
index bf865e218821eea237172364e3c7a2001cd43fa4..6648c637c8d88aa5f9872defff5c9566dbe21d11 100644
--- a/core/DataAccess/ArchiveInvalidator.php
+++ b/core/DataAccess/ArchiveInvalidator.php
@@ -21,6 +21,8 @@ use Piwik\Period\Week;
*
* Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
*
+ * The invalidated archives will be deleted by ArchivePurger
+ *
* @package Piwik\DataAccess
*/
class ArchiveInvalidator {
@@ -37,7 +39,7 @@ class ArchiveInvalidator {
* @return array
* @throws \Exception
*/
- public function markArchivesAsInvalidated($idSites, $dates, $period)
+ public function markArchivesAsInvalidated(array $idSites, $dates, $period)
{
$this->findOlderDateWithLogs();
$datesToInvalidate = $this->getDatesToInvalidateFromString($dates);
@@ -45,10 +47,10 @@ class ArchiveInvalidator {
\Piwik\Plugins\SitesManager\API::getInstance()->updateSiteCreatedTime($idSites, $minDate);
- $this->markArchivesInvalidatedFor($idSites, $period, $datesToInvalidate);
+ $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate);
+ $this->markArchivesInvalidatedFor($idSites, $period, $datesByMonth);
- $store = new InvalidatedReports();
- $store->addInvalidatedSitesToReprocess($idSites);
+ $this->persistInvalidatedArchives($idSites, $datesByMonth);
return $this->makeOutputLogs();
}
@@ -79,15 +81,13 @@ class ArchiveInvalidator {
/**
* @param $idSites
* @param $period string
- * @param $datesToInvalidate Date[]
+ * @param $datesByMonth array
* @throws \Exception
*/
- private function markArchivesInvalidatedFor($idSites, $period, $datesToInvalidate)
+ private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth)
{
$invalidateForPeriodId = $this->getPeriodId($period);
- $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate);
-
// In each table, invalidate day/week/month/year containing this date
$archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
foreach ($archiveTables as $table) {
@@ -161,10 +161,11 @@ class ArchiveInvalidator {
&& $date->isEarlier($this->minimumDateWithLogs)
) {
$this->warningDates[] = $date->toString();
- } else {
- $this->processedDates[] = $date->toString();
+ continue;
}
+ $this->processedDates[] = $date->toString();
+
$month = $date->toString('Y_m');
// For a given date, we must invalidate in the monthly archive table
$datesByMonth[$month][] = $date->toString();
@@ -217,8 +218,23 @@ class ArchiveInvalidator {
return $invalidateForPeriod;
}
+ /**
+ * @param array $idSites
+ * @param $datesByMonth
+ */
+ private function persistInvalidatedArchives(array $idSites, $datesByMonth)
+ {
+ $yearMonths = array_keys($datesByMonth);
+ $yearMonths = array_unique($yearMonths);
+
+ $store = new InvalidatedReports();
+ $store->addInvalidatedSitesToReprocess($idSites);
+ $store->addSitesToPurgeForYearMonths($idSites, $yearMonths);
+ }
+
private static function getModel()
{
return new Model();
}
+
}
\ No newline at end of file
diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php
index fc9254ac3b7cdbe294fe280e3040c64b64da4228..dbdff36c6b67e482b3fc26c8c10a5a9b7fdb6e6c 100644
--- a/core/DataAccess/ArchivePurger.php
+++ b/core/DataAccess/ArchivePurger.php
@@ -16,7 +16,13 @@ use Piwik\Log;
use Piwik\Piwik;
/**
- * Cleans up outdated archives
+ *
+ * This class purges two types of archives:
+ *
+ * (1) Deletes invalidated archives (from ArchiveInvalidator)
+ *
+ * (2) Deletes outdated archives (the temporary or errored archives)
+ *
*
* @package Piwik\DataAccess
*/
@@ -24,35 +30,26 @@ class ArchivePurger
{
public static function purgeInvalidatedArchives()
{
- $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
-
- foreach ($archiveTables as $archiveTable) {
- /**
- * Select the archives that have already been invalidated and have been since re-processed.
- * It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
- */
- $result = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($archiveTable);
-
- if (count($result) > 0) {
- $archiveIds = array_map(
- function ($elm) {
- return $elm['idarchive'];
- },
- $result
- );
-
- $date = ArchiveTableCreator::getDateFromTableName($archiveTable);
- $date = Date::factory(str_replace('_', '-', $date) . '-01');
-
- self::deleteArchiveIds($date, $archiveIds);
+ $store = new InvalidatedReports();
+ $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge();
+
+ foreach ($idSitesByYearMonth as $yearMonth => $idSites) {
+ if(empty($idSites)) {
+ continue;
}
- }
- }
+ $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
+ $numericTable = ArchiveTableCreator::getNumericTable($date);
- private static function getModel()
- {
- return new Model();
+ $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
+
+ if (count($archiveIds) == 0) {
+ continue;
+ }
+ self::deleteArchiveIds($date, $archiveIds);
+
+ $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth);
+ }
}
/**
@@ -133,4 +130,9 @@ class ArchivePurger
}
}
+ private static function getModel()
+ {
+ return new Model();
+ }
+
}
diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php
index 5f389b394b4bdd2f761fe435f6350f7fca484fc1..ac5a0ebbee0dc9620715b192a63b83192dc57539 100644
--- a/core/DataAccess/InvalidatedReports.php
+++ b/core/DataAccess/InvalidatedReports.php
@@ -10,7 +10,6 @@
namespace Piwik\DataAccess;
use Piwik\Option;
-use Piwik\Piwik;
/**
* Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API.
@@ -25,19 +24,62 @@ use Piwik\Piwik;
class InvalidatedReports
{
const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
+ const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
+
+ /**
+ * Mark the sites IDs and Dates as being invalidated, so we can purge them later on.
+ *
+ * @param array $idSites
+ * @param array $yearMonths
+ */
+ public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths)
+ {
+ $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+ foreach($yearMonths as $yearMonth) {
+ $idSitesByYearMonth[$yearMonth] = $idSites;
+ }
+ $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
+ }
+
+ /**
+ * Returns the list of websites IDs for which invalidated archives can be purged.
+ */
+ public function getSitesByYearMonthArchiveToPurge()
+ {
+ $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+
+ // From this list we remove the websites that are not yet re-processed
+ // so we don't purge them before they were re-processed
+ $idSitesNotYetReprocessed = $this->getSitesToReprocess();
+
+ foreach($idSitesByYearMonth as $yearMonth => &$idSites) {
+ $idSites = array_diff($idSites, $idSitesNotYetReprocessed);
+ }
+ return $idSitesByYearMonth;
+ }
+
+ public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth)
+ {
+ $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
+
+ if(!isset($idSitesByYearMonth[$yearMonth])) {
+ return;
+ }
+
+ $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites);
+ $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
+ }
/**
* Record those website IDs as having been invalidated
*
* @param $idSites
*/
- public function addInvalidatedSitesToReprocess($idSites)
+ public function addInvalidatedSitesToReprocess(array $idSites)
{
- $invalidatedIdSites = $this->getSitesToReprocess();
- $invalidatedIdSites = array_merge($invalidatedIdSites, $idSites);
- $invalidatedIdSites = array_unique($invalidatedIdSites);
- $invalidatedIdSites = array_values($invalidatedIdSites);
- $this->setSitesToReprocess($invalidatedIdSites);
+ $siteIdsToReprocess = $this->getSitesToReprocess();
+ $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites);
+ $this->setSitesToReprocess($siteIdsToReprocess);
}
@@ -46,13 +88,13 @@ class InvalidatedReports
*/
public function storeSiteIsReprocessed($idSite)
{
- $websiteIdsInvalidated = $this->getSitesToReprocess();
+ $siteIdsToReprocess = $this->getSitesToReprocess();
- if (count($websiteIdsInvalidated)) {
- $found = array_search($idSite, $websiteIdsInvalidated);
+ if (count($siteIdsToReprocess)) {
+ $found = array_search($idSite, $siteIdsToReprocess);
if ($found !== false) {
- unset($websiteIdsInvalidated[$found]);
- $this->setSitesToReprocess($websiteIdsInvalidated);
+ unset($siteIdsToReprocess[$found]);
+ $this->setSitesToReprocess($siteIdsToReprocess);
}
}
}
@@ -64,25 +106,56 @@ class InvalidatedReports
*/
public function getSitesToReprocess()
{
- Option::clearCachedOption(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
- $invalidatedIdSites = Option::get(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+ return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+ }
+
+ /**
+ * @return array|false|mixed|string
+ */
+ private function getSitesByYearMonthToPurge()
+ {
+ return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
+ }
+
+ /**
+ * @param $websiteIdsInvalidated
+ */
+ private function setSitesToReprocess($websiteIdsInvalidated)
+ {
+ $websiteIdsInvalidated = array_unique($websiteIdsInvalidated);
+ $websiteIdsInvalidated = array_values($websiteIdsInvalidated);
+ Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+ }
+
+ /**
+ * @param $optionName
+ * @return array|false|mixed|string
+ */
+ private function getArrayValueFromOptionName($optionName)
+ {
+ Option::clearCachedOption($optionName);
+ $array = Option::get($optionName);
- if ($invalidatedIdSites
- && ($invalidatedIdSites = unserialize($invalidatedIdSites))
- && count($invalidatedIdSites)
+ if ($array
+ && ($array = unserialize($array))
+ && count($array)
) {
- return $invalidatedIdSites;
+ return $array;
}
return array();
}
/**
- * @param $websiteIdsInvalidated
+ * @param $idSitesByYearMonth
*/
- private function setSitesToReprocess($websiteIdsInvalidated)
+ private function persistSitesByYearMonthToPurge($idSitesByYearMonth)
{
- Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+ // remove dates for which there are no sites to purge
+ $idSitesByYearMonth = array_filter($idSitesByYearMonth);
+
+ Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth));
}
+
}
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index ee2d8702af2cca2c96ddb18ef1fc4a9ce51f9183..5cc25635af103c37e7be4c2f009ac52b44844ab9 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -27,26 +27,39 @@ class Model
*
* These archives { archive name (includes segment hash) , idsite, date, period } will be deleted.
*
- * @param $archiveTable
+ * @param string $archiveTable
+ * @param array $idSites
* @return array
* @throws Exception
*/
- public function getInvalidatedArchiveIdsSafeToDelete($archiveTable)
+ public function getInvalidatedArchiveIdsSafeToDelete($archiveTable, array $idSites)
{
// prevent error 'The SELECT would examine more than MAX_JOIN_SIZE rows'
Db::get()->query('SET SQL_BIG_SELECTS=1');
$query = 'SELECT t1.idarchive FROM `' . $archiveTable . '` t1
INNER JOIN `' . $archiveTable . '` t2
- ON t1.name = t2.name AND t1.idsite=t2.idsite
- AND t1.date1=t2.date1 AND t1.date2=t2.date2 AND t1.period=t2.period
+ ON t1.name = t2.name
+ AND t1.idsite = t2.idsite
+ AND t1.date1 = t2.date1
+ AND t1.date2 = t2.date2
+ AND t1.period = t2.period
WHERE t1.value = ' . ArchiveWriter::DONE_INVALIDATED . '
+ AND t1.idsite IN (' . implode(",", $idSites) . ')
AND t2.value IN(' . ArchiveWriter::DONE_OK . ', ' . ArchiveWriter::DONE_OK_TEMPORARY . ')
- AND t1.ts_archived < t2.ts_archived AND t1.name LIKE \'done%\'';
+ AND t1.ts_archived < t2.ts_archived
+ AND t1.name LIKE \'done%\'
+ ';
$result = Db::fetchAll($query);
- return $result;
+ $archiveIds = array_map(
+ function ($elm) {
+ return $elm['idarchive'];
+ },
+ $result
+ );
+ return $archiveIds;
}
/**
diff --git a/plugins/Annotations/AnnotationList.php b/plugins/Annotations/AnnotationList.php
index cf068e6f35e889f0d1dcd44ba62b6904a8a4f843..f51a5e86ce600f22160a90155168589608167967 100755
--- a/plugins/Annotations/AnnotationList.php
+++ b/plugins/Annotations/AnnotationList.php
@@ -440,7 +440,7 @@ class AnnotationList
public static function canUserAddNotesFor($idSite)
{
return Piwik::isUserHasViewAccess($idSite)
- && !Piwik::isUserIsAnonymous($idSite);
+ && !Piwik::isUserIsAnonymous();
}
/**