From c186252bf9a30a54c15330f0f4c88c1a033e7f25 Mon Sep 17 00:00:00 2001
From: diosmosis <benaka@piwik.pro>
Date: Wed, 7 Oct 2015 17:32:08 -0700
Subject: [PATCH] Fix regression, make sure if period is not supplied to
ArchiveInvalidator then all periods are invalidated efficiently.
---
core/Archive/ArchiveInvalidator.php | 51 ++++++++++++----
core/DataAccess/Model.php | 19 +++---
.../DataAccess/ArchiveInvalidatorTest.php | 60 +++++++++++++++----
3 files changed, 101 insertions(+), 29 deletions(-)
diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index daa847e520..c4fff2bef4 100644
--- a/core/Archive/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -137,11 +137,16 @@ class ArchiveInvalidator
$datesToInvalidate = $this->removeDatesThatHaveBeenPurged($dates, $invalidationInfo);
- $periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
- $periods = $this->getPeriodsByYearMonthAndType($periods);
- $this->markArchivesInvalidated($idSites, $periods, $segment);
+ if (empty($period)) {
+ // if the period is empty, we don't need to cascade in any way, since we'll remove all periods
+ $periodDates = $this->getDatesByYearMonthAndPeriodType($dates);
+ } else {
+ $periods = $this->getPeriodsToInvalidate($datesToInvalidate, $period, $cascadeDown);
+ $periodDates = $this->getPeriodDatesByYearMonthAndPeriodType($periods);
+ }
+ $this->markArchivesInvalidated($idSites, $periodDates, $segment);
- $yearMonths = array_keys($periods);
+ $yearMonths = array_keys($periodDates);
$this->markInvalidatedArchivesForReprocessAndPurge($idSites, $yearMonths);
foreach ($idSites as $idSite) {
@@ -197,35 +202,57 @@ class ArchiveInvalidator
/**
* @param Period[] $periods
- * @return Period[][][]
+ * @return string[][][]
*/
- private function getPeriodsByYearMonthAndType($periods)
+ private function getPeriodDatesByYearMonthAndPeriodType($periods)
{
$result = array();
foreach ($periods as $period) {
- $yearMonth = ArchiveTableCreator::getTableMonthFromDate($period->getDateStart());
+ $date = $period->getDateStart();
$periodType = $period->getId();
- $result[$yearMonth][$periodType][] = $period;
+ $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
+ $result[$yearMonth][$periodType][] = $date->toString();
+ }
+ return $result;
+ }
+
+ /**
+ * Called when deleting all periods.
+ *
+ * @param Date[] $dates
+ * @return string[][][]
+ */
+ private function getDatesByYearMonthAndPeriodType($dates)
+ {
+ $result = array();
+ foreach ($dates as $date) {
+ $yearMonth = ArchiveTableCreator::getTableMonthFromDate($date);
+ $result[$yearMonth][null][] = $date->toString();
+
+ // since we're removing all periods, we must make sure to remove year periods as well.
+ // this means we have to make sure the january table is processed.
+ $janYearMonth = $date->toString('Y') . '_01';
+ $result[$janYearMonth][null][] = $date->toString();
}
return $result;
}
/**
* @param int[] $idSites
- * @param Period[][][] $periods
+ * @param string[][][] $dates
* @throws \Exception
*/
- private function markArchivesInvalidated($idSites, $periods, Segment $segment = null)
+ private function markArchivesInvalidated($idSites, $dates, Segment $segment = null)
{
$archiveNumericTables = ArchiveTableCreator::getTablesArchivesInstalled($type = ArchiveTableCreator::NUMERIC_TABLE);
foreach ($archiveNumericTables as $table) {
$tableDate = ArchiveTableCreator::getDateFromTableName($table);
- if (empty($periods[$tableDate])) {
+ if (empty($dates[$tableDate])) {
continue;
}
- $this->model->updateArchiveAsInvalidated($table, $idSites, $periods[$tableDate], $segment);
+ $this->model->updateArchiveAsInvalidated($table, $idSites, $dates[$tableDate], $segment);
}
}
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index f70a934d13..e8ec27c7de 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -100,28 +100,33 @@ class Model
/**
* @param string $archiveTable Prefixed table name
* @param int[] $idSites
- * @param Period[][] $periodsByType
+ * @param string[][] $datesByPeriodType
* @param Segment $segment
* @return \Zend_Db_Statement
* @throws Exception
*/
- public function updateArchiveAsInvalidated($archiveTable, $idSites, $periodsByType, Segment $segment = null)
+ public function updateArchiveAsInvalidated($archiveTable, $idSites, $datesByPeriodType, Segment $segment = null)
{
$idSites = array_map('intval', $idSites);
$bind = array();
$periodConditions = array();
- foreach ($periodsByType as $periodType => $periods) {
+ foreach ($datesByPeriodType as $periodType => $dates) {
$dateConditions = array();
- foreach ($periods as $period) {
+ foreach ($dates as $date) {
$dateConditions[] = "(date1 <= ? AND ? <= date2)";
- $bind[] = $period->getDateStart()->toString();
- $bind[] = $period->getDateStart()->toString();
+ $bind[] = $date;
+ $bind[] = $date;
}
- $periodConditions[] = "(period = " . (int)$periodType . " AND (" . implode(" OR ", $dateConditions) . "))";
+ $dateConditionsSql = implode(" OR ", $dateConditions);
+ if (empty($periodType)) { // remove all periods
+ $periodConditions[] = "($dateConditionsSql)";
+ } else {
+ $periodConditions[] = "(period = " . (int)$periodType . " AND ($dateConditionsSql))";
+ }
}
if ($segment) {
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index 76433c0eb4..199187d46a 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -243,11 +243,14 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
$expectedWarningDates = array($dateBeforeThreshold->toString());
$this->assertEquals($expectedWarningDates, $result->warningDates);
- $expectedIdArchives = array(
- '2015_03' => array(),
- '2015_04' => array(1),
- );
- $this->assertEquals($expectedIdArchives, $this->getInvalidatedIdArchives());
+ $invalidatedArchives = $this->getInvalidatedIdArchives();
+
+ $countInvalidatedArchives = 0;
+ foreach ($invalidatedArchives as $idarchives) {
+ $countInvalidatedArchives += count($idarchives);
+ }
+
+ $this->assertEquals(1, $countInvalidatedArchives);
}
public function test_markArchivesAsInvalidated_CorrectlyModifiesDistributedLists()
@@ -337,6 +340,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2.2015-01-01.2015-01-01.1.done.VisitsSummary',
'1.2015-01-01.2015-01-31.3.done3736b708e4d20cfc10610e816a1b2341',
'2.2015-01-01.2015-01-31.3.done.VisitsSummary',
+ '1.2015-01-01.2015-12-31.4.done5447835b0a861475918e79e932abdfd8',
+ '2.2015-01-01.2015-12-31.4.done',
),
'2015_02' => array(
'1.2015-02-05.2015-02-05.1.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
@@ -347,9 +352,10 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2.2015-02-01.2015-02-28.3.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
),
'2015_05' => array(
- '1.2015-05-01.2015-05-31.3.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
- '2.2015-05-01.2015-05-31.3.done5447835b0a861475918e79e932abdfd8',
+ '1.2015-05-01.2015-05-31.3.done3736b708e4d20cfc10610e816a1b2341',
+ '2.2015-05-01.2015-05-31.3.done.VisitsSummary',
),
+ '2015_06' => array(),
),
),
@@ -365,11 +371,13 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2014_12' => array(),
'2015_01' => array(
'1.2015-01-01.2015-01-31.3.done3736b708e4d20cfc10610e816a1b2341',
+ '1.2015-01-01.2015-12-31.4.done5447835b0a861475918e79e932abdfd8',
),
'2015_02' => array(),
'2015_03' => array(),
'2015_04' => array(),
- '2015_05' => array()
+ '2015_05' => array(),
+ '2015_06' => array(),
),
),
@@ -422,11 +430,13 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'1.2015-01-19.2015-01-25.2.done',
'1.2015-01-26.2015-02-01.2.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
'1.2015-01-01.2015-01-31.3.done3736b708e4d20cfc10610e816a1b2341',
+ '1.2015-01-01.2015-12-31.4.done5447835b0a861475918e79e932abdfd8',
),
'2015_02' => array(),
'2015_03' => array(),
'2015_04' => array(),
'2015_05' => array(),
+ '2015_06' => array(),
),
),
@@ -461,6 +471,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'1.2015-01-31.2015-01-31.1.done3736b708e4d20cfc10610e816a1b2341',
'1.2015-01-26.2015-02-01.2.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
'1.2015-01-01.2015-01-31.3.done3736b708e4d20cfc10610e816a1b2341',
+ '1.2015-01-01.2015-12-31.4.done5447835b0a861475918e79e932abdfd8',
),
'2015_02' => array(
'1.2015-02-01.2015-02-01.1.done3736b708e4d20cfc10610e816a1b2341',
@@ -469,6 +480,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2015_03' => array(),
'2015_04' => array(),
'2015_05' => array(),
+ '2015_06' => array(),
),
),
@@ -483,7 +495,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2014_01' => array(),
'2014_12' => array(),
'2015_01' => array(
- '1.2015-01-01.2015-01-10.5.done5447835b0a861475918e79e932abdfd8',
+ '1.2015-01-01.2015-01-10.5.done.VisitsSummary',
),
'2015_02' => array(),
'2015_03' => array(
@@ -492,6 +504,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
),
'2015_04' => array(),
'2015_05' => array(),
+ '2015_06' => array(),
),
),
@@ -528,6 +541,32 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
'2015_03' => array(),
'2015_04' => array(),
'2015_05' => array(),
+ '2015_06' => array(),
+ ),
+ ),
+
+ // removing all periods
+ array(
+ array(1),
+ array('2015-05-05'),
+ '',
+ null,
+ false,
+ array(
+ '2014_01' => array(),
+ '2014_12' => array(),
+ '2015_01' => array(
+ '1.2015-01-01.2015-12-31.4.done5447835b0a861475918e79e932abdfd8',
+ ),
+ '2015_02' => array(),
+ '2015_03' => array(),
+ '2015_04' => array(),
+ '2015_05' => array(
+ '1.2015-05-05.2015-05-05.1.done3736b708e4d20cfc10610e816a1b2341.UserCountry',
+ '1.2015-05-04.2015-05-10.2.done5447835b0a861475918e79e932abdfd8',
+ '1.2015-05-01.2015-05-31.3.done3736b708e4d20cfc10610e816a1b2341',
+ ),
+ '2015_06' => array(),
),
),
);
@@ -572,7 +611,8 @@ class ArchiveInvalidatorTest extends IntegrationTestCase
$endDate = Date::factory('2015-05-31');
foreach ($periods as $periodLabel) {
- for ($date = $startDate; $date->isEarlier($endDate); $date = $date->addPeriod(1, $periodLabel)) {
+ $nextEndDate = $endDate->addPeriod(1, $periodLabel);
+ for ($date = $startDate; $date->isEarlier($nextEndDate); $date = $date->addPeriod(1, $periodLabel)) {
foreach ($sites as $idSite) {
$this->insertArchiveRow($idSite, $date->toString(), $periodLabel);
}
--
GitLab