diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index 32699231460ae817ce2b1541baecc7178e1a2569..11142b5d712dd0b20ef0c8c3b103c7c07a70c1f1 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -10,6 +10,7 @@ namespace Piwik\ArchiveProcessor;
use Exception;
use Piwik\Config;
+use Piwik\DataAccess\ArchiveWriter;
use Piwik\Date;
use Piwik\Log;
use Piwik\Option;
@@ -109,6 +110,8 @@ class Rules
}
/**
+ * Return done flags used to tell how the archiving process for a specific archive was completed,
+ *
* @param array $plugins
* @param $segment
* @return array
@@ -309,4 +312,21 @@ class Rules
}
return false;
}
+
+ /**
+ * Returns done flag values allowed to be selected
+ *
+ * @return string
+ */
+ public static function getSelectableDoneFlagValues()
+ {
+ $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
+
+ if (!Rules::isRequestAuthorizedToArchive()) {
+ //If request is not authorized to archive then fetch also invalidated archives
+ $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
+ }
+
+ return $possibleValues;
+ }
}
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 47bd0c98d884efa11a20e03fea822395984278e5..64fee8ba050246927fb3fe5f47275780e2d56113 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -13,7 +13,7 @@ use Piwik\ArchiveProcessor\Rules;
use Piwik\CronArchive\FixedSiteIds;
use Piwik\CronArchive\SharedSiteIds;
use Piwik\Period\Factory as PeriodFactory;
-use Piwik\Plugins\CoreAdminHome\InvalidatedReports;
+use Piwik\DataAccess\InvalidatedReports;
use Piwik\Plugins\SitesManager\API as APISitesManager;
/**
@@ -607,7 +607,7 @@ class CronArchive
if ($this->isOldReportInvalidatedForWebsite($idSite)) {
$store = new InvalidatedReports();
- $store->removeWebsiteFromInvalidatedWebsites($idSite);
+ $store->storeSiteIsReprocessed($idSite);
}
// when some data was purged from this website
@@ -932,7 +932,7 @@ class CronArchive
$websiteIds = array_merge(
$this->addWebsiteIdsWithVisitsSinceLastRun(),
- $this->getWebsiteIdsToInvalidate()
+ $this->getInvalidatedSitesToReprocess()
);
$websiteIds = array_merge($websiteIds, $this->addWebsiteIdsInTimezoneWithNewDay($websiteIds));
return array_unique($websiteIds);
@@ -1001,7 +1001,7 @@ class CronArchive
private function updateIdSitesInvalidatedOldReports()
{
$store = new InvalidatedReports();
- $this->idSitesInvalidatedOldReports = $store->getWebsiteIdsToInvalidate();
+ $this->idSitesInvalidatedOldReports = $store->getSitesToReprocess();
}
/**
@@ -1011,7 +1011,7 @@ class CronArchive
*
* @return array
*/
- private function getWebsiteIdsToInvalidate()
+ private function getInvalidatedSitesToReprocess()
{
$this->updateIdSitesInvalidatedOldReports();
diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php
index db84dc935c070767ab79c97928bef0d98c02b42c..6fc593d661634663876a2f17408f3fa51291c68f 100644
--- a/core/DataAccess/ArchiveInvalidator.php
+++ b/core/DataAccess/ArchiveInvalidator.php
@@ -12,11 +12,17 @@ namespace Piwik\DataAccess;
use Piwik\Date;
use Piwik\Db;
-use Piwik\Plugins\CoreAdminHome\InvalidatedReports;
use Piwik\Plugins\PrivacyManager\PrivacyManager;
use Piwik\Period;
use Piwik\Period\Week;
+/**
+ * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated)
+ *
+ * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
+ *
+ * @package Piwik\DataAccess
+ */
class ArchiveInvalidator {
private $warningDates = array();
@@ -43,7 +49,7 @@ class ArchiveInvalidator {
$store = new InvalidatedReports();
- $store->setSiteIdsToBeInvalidated($idSites);
+ $store->storeInvalidatedSitesAndDates($idSites);
return $this->makeOutputLogs();
}
@@ -129,7 +135,7 @@ class ArchiveInvalidator {
private function findOlderDateWithLogs()
{
-// If using the feature "Delete logs older than N days"...
+ // If using the feature "Delete logs older than N days"...
$purgeDataSettings = PrivacyManager::getPurgeDataSettings();
$logsAreDeletedBeforeThisDate = $purgeDataSettings['delete_logs_schedule_lowest_interval'];
$logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php
index 3ae023fdda973e0b27fd81ea3dea5e0afe2c98c5..fc9254ac3b7cdbe294fe280e3040c64b64da4228 100644
--- a/core/DataAccess/ArchivePurger.php
+++ b/core/DataAccess/ArchivePurger.php
@@ -31,7 +31,7 @@ class ArchivePurger
* Select the archives that have already been invalidated and have been since re-processed.
* It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
*/
- $result = self::getModel()->purgeInvalidatedArchiveTable($archiveTable);
+ $result = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($archiveTable);
if (count($result) > 0) {
$archiveIds = array_map(
diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php
index a342257eca4eb1714643489357890a2e2efc58ad..56dede02b712cc1029e909d81045897cc302ef1b 100644
--- a/core/DataAccess/ArchiveSelector.php
+++ b/core/DataAccess/ArchiveSelector.php
@@ -9,8 +9,8 @@
namespace Piwik\DataAccess;
use Exception;
-use Piwik\ArchiveProcessor\Rules;
use Piwik\ArchiveProcessor;
+use Piwik\ArchiveProcessor\Rules;
use Piwik\Common;
use Piwik\Date;
use Piwik\Db;
@@ -63,10 +63,10 @@ class ArchiveSelector
$isSkipAggregationOfSubTables = $params->isSkipAggregationOfSubTables();
$plugins = array("VisitsSummary", $requestedPlugin);
- $doneFlags = self::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
- $possibleValues = self::getPossibleValues();
+ $doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
+ $doneFlagValues = Rules::getSelectableDoneFlagValues();
- $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $possibleValues);
+ $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues);
if (empty($results)) {
return false;
@@ -289,49 +289,14 @@ class ArchiveSelector
{
// the flags used to tell how the archiving process for a specific archive was completed,
// if it was completed
- $doneFlags = self::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
+ $doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
$allDoneFlags = "'" . implode("','", $doneFlags) . "'";
- $possibleValues = self::getPossibleValues();
+ $possibleValues = Rules::getSelectableDoneFlagValues();
// create the SQL to find archives that are DONE
return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))";
}
- /**
- * Returns the SQL condition used to find successfully completed archives that
- * this instance is querying for.
- *
- * @param array $plugins
- * @param Segment $segment
- * @param bool $isSkipAggregationOfSubTables
- * @return string
- */
- private static function getDoneFlags(array $plugins, Segment $segment, $isSkipAggregationOfSubTables)
- {
- // the flags used to tell how the archiving process for a specific archive was completed,
- // if it was completed
- $doneFlags = Rules::getDoneFlags($plugins, $segment, $isSkipAggregationOfSubTables);
-
- return $doneFlags;
- }
-
- /**
- * Returns the SQL condition used to find successfully completed archives that
- * this instance is querying for.
- *
- * @return string
- */
- private static function getPossibleValues()
- {
- $possibleValues = array(ArchiveWriter::DONE_OK, ArchiveWriter::DONE_OK_TEMPORARY);
-
- if (!Rules::isRequestAuthorizedToArchive()) {
- //If request is not authorized to archive then fetch also invalidated archives
- $possibleValues[] = ArchiveWriter::DONE_INVALIDATED;
- }
-
- return $possibleValues;
- }
}
diff --git a/plugins/CoreAdminHome/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php
similarity index 71%
rename from plugins/CoreAdminHome/InvalidatedReports.php
rename to core/DataAccess/InvalidatedReports.php
index 971a49efd9b0bc8393453fb85668ac46f22243e7..e5d55550a02ef7bd1eae29aca5963d65bd5cc608 100644
--- a/plugins/CoreAdminHome/InvalidatedReports.php
+++ b/core/DataAccess/InvalidatedReports.php
@@ -7,7 +7,7 @@
*
*/
-namespace Piwik\plugins\CoreAdminHome;
+namespace Piwik\DataAccess;
use Piwik\Option;
use Piwik\Piwik;
@@ -27,58 +27,62 @@ class InvalidatedReports
const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
/**
- * Returns array of idSites to force re-process next time core:archive command runs
- *
- * @ignore
- * @return mixed
- */
- public function getWebsiteIdsToInvalidate()
- {
- Piwik::checkUserHasSomeAdminAccess();
-
- Option::clearCachedOption(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
- $invalidatedIdSites = Option::get(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
- if ($invalidatedIdSites
- && ($invalidatedIdSites = unserialize($invalidatedIdSites))
- && count($invalidatedIdSites)
- ) {
- return $invalidatedIdSites;
- }
- return array();
- }
-
-
-
- /**
- * Force to re-process data for these websites in the next cron core:archive command run
+ * Record those website IDs as having been invalidated
*
* @param $idSites
*/
- public function setSiteIdsToBeInvalidated($idSites)
+ public function storeInvalidatedSitesAndDates($idSites)
{
- $store = new InvalidatedReports();
- $invalidatedIdSites = $store->getWebsiteIdsToInvalidate();
+ $invalidatedIdSites = $this->getSitesToReprocess();
$invalidatedIdSites = array_merge($invalidatedIdSites, $idSites);
$invalidatedIdSites = array_unique($invalidatedIdSites);
$invalidatedIdSites = array_values($invalidatedIdSites);
- Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($invalidatedIdSites));
+ $this->setSitesToReprocess($invalidatedIdSites);
}
/**
* @param $idSite
*/
- public function removeWebsiteFromInvalidatedWebsites($idSite)
+ public function storeSiteIsReprocessed($idSite)
{
- $websiteIdsInvalidated = $this->getWebsiteIdsToInvalidate();
+ $websiteIdsInvalidated = $this->getSitesToReprocess();
if (count($websiteIdsInvalidated)) {
$found = array_search($idSite, $websiteIdsInvalidated);
if ($found !== false) {
unset($websiteIdsInvalidated[$found]);
- Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+ $this->setSitesToReprocess($websiteIdsInvalidated);
}
}
}
+ /**
+ * Returns array of idSites to force re-process next time core:archive command runs
+ *
+ * @return array of id sites
+ */
+ public function getSitesToReprocess()
+ {
+ Option::clearCachedOption(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+ $invalidatedIdSites = Option::get(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+
+ if ($invalidatedIdSites
+ && ($invalidatedIdSites = unserialize($invalidatedIdSites))
+ && count($invalidatedIdSites)
+ ) {
+ return $invalidatedIdSites;
+ }
+ return array();
+ }
+
+ /**
+ * @param $websiteIdsInvalidated
+ */
+ private function setSitesToReprocess($websiteIdsInvalidated)
+ {
+ Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
+ }
+
+
}
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 7f5adb7a45b8649a7e856c86146b9b9ebfc0e8a5..ee2d8702af2cca2c96ddb18ef1fc4a9ce51f9183 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -22,15 +22,20 @@ class Model
{
const PREFIX_SQL_LOCK = "locked_";
- public function purgeInvalidatedArchiveTable($archiveTable)
+ /**
+ * Returns the archives IDs that have already been invalidated and have been since re-processed.
+ *
+ * These archives { archive name (includes segment hash) , idsite, date, period } will be deleted.
+ *
+ * @param $archiveTable
+ * @return array
+ * @throws Exception
+ */
+ public function getInvalidatedArchiveIdsSafeToDelete($archiveTable)
{
// prevent error 'The SELECT would examine more than MAX_JOIN_SIZE rows'
Db::get()->query('SET SQL_BIG_SELECTS=1');
- /**
- * Select the archives that have already been invalidated and have been since re-processed.
- * It purges records for each distinct { archive name (includes segment hash) , idsite, date, period } tuple.
- */
$query = 'SELECT t1.idarchive FROM `' . $archiveTable . '` t1
INNER JOIN `' . $archiveTable . '` t2
ON t1.name = t2.name AND t1.idsite=t2.idsite
@@ -115,7 +120,7 @@ class Model
}
}
- public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $possibleValues)
+ public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues)
{
$bindSQL = array($idSite,
$dateStartIso,
@@ -129,7 +134,7 @@ class Model
$bindSQL[] = $minDatetimeIsoArchiveProcessedUTC;
}
- $sqlWhereArchiveName = self::getNameCondition($doneFlags, $possibleValues);
+ $sqlWhereArchiveName = self::getNameCondition($doneFlags, $doneFlagValues);
$sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable
WHERE idsite = ?