From 6e8ec4575c1b19af63fe4910dd0dd7c5dc8a6003 Mon Sep 17 00:00:00 2001
From: Thomas Steur <thomas.steur@gmail.com>
Date: Wed, 18 Mar 2015 03:40:20 +0000
Subject: [PATCH] faster archive by calculating the recursive count only if
needed
---
core/Archive/DataTableFactory.php | 4 ++--
core/ArchiveProcessor.php | 10 ++++++---
core/DataTable/Row.php | 9 ++------
plugins/Actions/Archiver.php | 3 ++-
plugins/Contents/Archiver.php | 9 +++++++-
plugins/CustomVariables/Archiver.php | 9 ++++++--
plugins/DevicePlugins/Archiver.php | 10 ++++++++-
plugins/DevicesDetection/Archiver.php | 8 ++++++-
plugins/Events/Archiver.php | 9 +++++++-
plugins/Goals/Archiver.php | 32 +++++++++++++++++++++------
plugins/Provider/Archiver.php | 10 ++++++++-
plugins/Referrers/Archiver.php | 10 ++++++++-
plugins/Resolution/Archiver.php | 9 +++++++-
plugins/UserCountry/Archiver.php | 10 ++++++++-
plugins/UserLanguage/Archiver.php | 9 +++++++-
plugins/VisitTime/Archiver.php | 9 +++++++-
plugins/VisitorInterest/Archiver.php | 9 +++++++-
17 files changed, 136 insertions(+), 33 deletions(-)
diff --git a/core/Archive/DataTableFactory.php b/core/Archive/DataTableFactory.php
index 59af8a4e0b..cfc1937813 100644
--- a/core/Archive/DataTableFactory.php
+++ b/core/Archive/DataTableFactory.php
@@ -326,7 +326,7 @@ class DataTableFactory
&& $treeLevel >= $this->maxSubtableDepth
) {
// unset the subtables so DataTableManager doesn't throw
- foreach ($dataTable->getRows() as $row) {
+ foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$row->removeSubtable();
}
@@ -335,7 +335,7 @@ class DataTableFactory
$dataName = reset($this->dataNames);
- foreach ($dataTable->getRows() as $row) {
+ foreach ($dataTable->getRowsWithoutSummaryRow() as $row) {
$sid = $row->getIdSubDataTable();
if ($sid === null) {
continue;
diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php
index 81ac213344..8f7b979915 100644
--- a/core/ArchiveProcessor.php
+++ b/core/ArchiveProcessor.php
@@ -187,6 +187,8 @@ class ArchiveProcessor
* @param array $columnsToRenameAfterAggregation Columns mapped to new names for columns that must change names
* when summed because they cannot be summed, eg,
* `array('nb_uniq_visitors' => 'sum_daily_nb_uniq_visitors')`.
+ * @param bool $countRowsRecursive if set to false, will not calculate the recursive rows count which results in
+ * faster aggregation
* @return array Returns the row counts of each aggregated report before truncation, eg,
*
* array(
@@ -203,7 +205,8 @@ class ArchiveProcessor
$maximumRowsInSubDataTable = null,
$columnToSortByBeforeTruncation = null,
&$columnsAggregationOperation = null,
- $columnsToRenameAfterAggregation = null)
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = true)
{
if (!is_array($recordNames)) {
$recordNames = array($recordNames);
@@ -216,8 +219,9 @@ class ArchiveProcessor
$table = $this->aggregateDataTableRecord($recordName, $columnsAggregationOperation, $columnsToRenameAfterAggregation);
$nameToCount[$recordName]['level0'] = $table->getRowsCount();
-
- $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+ if ($countRowsRecursive) {
+ $nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
+ }
$blob = $table->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
Common::destroy($table);
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index 71dde68031..629f968e9a 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -229,11 +229,6 @@ class Row implements \ArrayAccess, \IteratorAggregate
return $this->c[self::METADATA][$name];
}
- private function getColumnsRaw()
- {
- return $this->c[self::COLUMNS];
- }
-
/**
* Returns true if a column having the given name is already registered. The value will not be evaluated, it will
* just check whether a column exists independent of its value.
@@ -475,12 +470,12 @@ class Row implements \ArrayAccess, \IteratorAggregate
*/
public function sumRow(Row $rowToSum, $enableCopyMetadata = true, $aggregationOperations = false)
{
- foreach ($rowToSum->getColumnsRaw() as $columnToSumName => $columnToSumValue) {
+ foreach ($rowToSum->getColumns() as $columnToSumName => $columnToSumValue) {
if (!$this->isSummableColumn($columnToSumName)) {
continue;
}
- $thisColumnValue = $this->getColumn($columnToSumName);
+ $thisColumnValue = &$this->getColumn($columnToSumName);
$operation = 'sum';
if (is_array($aggregationOperations) && isset($aggregationOperations[$columnToSumName])) {
diff --git a/plugins/Actions/Archiver.php b/plugins/Actions/Archiver.php
index 17fc4f058a..9995b3e2dc 100644
--- a/plugins/Actions/Archiver.php
+++ b/plugins/Actions/Archiver.php
@@ -497,7 +497,8 @@ class Archiver extends \Piwik\Plugin\Archiver
ArchivingHelper::$maximumRowsInSubDataTable,
ArchivingHelper::$columnToSortByBeforeTruncation,
Metrics::$columnsAggregationOperation,
- Metrics::$columnsToRenameAfterAggregation
+ Metrics::$columnsToRenameAfterAggregation,
+ $countRowsRecursive = false
);
$dataTableToSum = array(
diff --git a/plugins/Contents/Archiver.php b/plugins/Contents/Archiver.php
index 6d3a2e05ce..9cd6fbfc49 100644
--- a/plugins/Contents/Archiver.php
+++ b/plugins/Contents/Archiver.php
@@ -48,7 +48,14 @@ class Archiver extends \Piwik\Plugin\Archiver
public function aggregateMultipleReports()
{
$dataTableToSum = $this->getRecordNames();
- $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableToSum,
+ $this->maximumRowsInDataTable,
+ $this->maximumRowsInSubDataTable,
+ $this->columnToSortByBeforeTruncation,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
private function getRecordNames()
diff --git a/plugins/CustomVariables/Archiver.php b/plugins/CustomVariables/Archiver.php
index 704038b0a1..f7b9c9ddff 100644
--- a/plugins/CustomVariables/Archiver.php
+++ b/plugins/CustomVariables/Archiver.php
@@ -51,8 +51,13 @@ class Archiver extends \Piwik\Plugin\Archiver
public function aggregateMultipleReports()
{
$this->getProcessor()->aggregateDataTableRecords(
- self::CUSTOM_VARIABLE_RECORD_NAME, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable,
- $columnToSort = Metrics::INDEX_NB_VISITS);
+ self::CUSTOM_VARIABLE_RECORD_NAME,
+ $this->maximumRowsInDataTableLevelZero,
+ $this->maximumRowsInSubDataTable,
+ $columnToSort = Metrics::INDEX_NB_VISITS,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
public function aggregateDayReport()
diff --git a/plugins/DevicePlugins/Archiver.php b/plugins/DevicePlugins/Archiver.php
index 1b4b92213e..798efec2e0 100644
--- a/plugins/DevicePlugins/Archiver.php
+++ b/plugins/DevicePlugins/Archiver.php
@@ -40,7 +40,15 @@ class Archiver extends \Piwik\Plugin\Archiver
$dataTableRecords = array(
self::PLUGIN_RECORD_NAME,
);
- $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableRecords,
+ $this->maximumRows,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false
+ );
}
protected function aggregateByPlugin()
diff --git a/plugins/DevicesDetection/Archiver.php b/plugins/DevicesDetection/Archiver.php
index 3ecb310028..e19a064172 100644
--- a/plugins/DevicesDetection/Archiver.php
+++ b/plugins/DevicesDetection/Archiver.php
@@ -58,7 +58,13 @@ class Archiver extends \Piwik\Plugin\Archiver
);
foreach ($dataTablesToSum as $dt) {
$this->getProcessor()->aggregateDataTableRecords(
- $dt, $this->maximumRows, $this->maximumRows, $columnToSort = "nb_visits");
+ $dt,
+ $this->maximumRows,
+ $this->maximumRows,
+ $columnToSort = 'nb_visits',
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
}
diff --git a/plugins/Events/Archiver.php b/plugins/Events/Archiver.php
index 5a5160bdb2..256c49491e 100644
--- a/plugins/Events/Archiver.php
+++ b/plugins/Events/Archiver.php
@@ -98,7 +98,14 @@ class Archiver extends \Piwik\Plugin\Archiver
public function aggregateMultipleReports()
{
$dataTableToSum = $this->getRecordNames();
- $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTable, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableToSum,
+ $this->maximumRowsInDataTable,
+ $this->maximumRowsInSubDataTable,
+ $this->columnToSortByBeforeTruncation,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
protected function getRecordNames()
diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php
index 379752a6e2..447ae5a267 100644
--- a/plugins/Goals/Archiver.php
+++ b/plugins/Goals/Archiver.php
@@ -361,7 +361,13 @@ class Archiver extends \Piwik\Plugin\Archiver
foreach ($this->dimensionRecord as $recordName) {
$dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName);
}
- $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+ $this->getProcessor()->aggregateDataTableRecords($dataTableToSum,
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
/*
* Archive General Goal metrics
@@ -385,14 +391,26 @@ class Archiver extends \Piwik\Plugin\Archiver
foreach ($goalIdsToSum as $goalId) {
// sum up the visits to conversion data table & the days to conversion data table
- $this->getProcessor()->aggregateDataTableRecords(array(
- self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
- self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)));
+ $this->getProcessor()->aggregateDataTableRecords(
+ array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId),
+ self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)),
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
// sum up goal overview reports
- $this->getProcessor()->aggregateDataTableRecords(array(
- self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
- self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)));
+ $this->getProcessor()->aggregateDataTableRecords(
+ array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME),
+ self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)),
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
}
diff --git a/plugins/Provider/Archiver.php b/plugins/Provider/Archiver.php
index a8a22c9527..c152807385 100644
--- a/plugins/Provider/Archiver.php
+++ b/plugins/Provider/Archiver.php
@@ -24,6 +24,14 @@ class Archiver extends \Piwik\Plugin\Archiver
public function aggregateMultipleReports()
{
- $this->getProcessor()->aggregateDataTableRecords(array(self::PROVIDER_RECORD_NAME), $this->maximumRows);
+ $this->getProcessor()->aggregateDataTableRecords(
+ array(self::PROVIDER_RECORD_NAME),
+ $this->maximumRows,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false
+ );
}
}
diff --git a/plugins/Referrers/Archiver.php b/plugins/Referrers/Archiver.php
index 3093449a6c..f520aed00e 100644
--- a/plugins/Referrers/Archiver.php
+++ b/plugins/Referrers/Archiver.php
@@ -217,7 +217,15 @@ class Archiver extends \Piwik\Plugin\Archiver
public function aggregateMultipleReports()
{
$dataTableToSum = $this->getRecordNames();
- $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, $this->maximumRowsInDataTableLevelZero, $this->maximumRowsInSubDataTable, $this->columnToSortByBeforeTruncation);
+ $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableToSum,
+ $this->maximumRowsInDataTableLevelZero,
+ $this->maximumRowsInSubDataTable,
+ $this->columnToSortByBeforeTruncation,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false
+ );
$mappingFromArchiveName = array(
self::METRIC_DISTINCT_SEARCH_ENGINE_RECORD_NAME =>
diff --git a/plugins/Resolution/Archiver.php b/plugins/Resolution/Archiver.php
index f44d744c53..cc1b82e056 100644
--- a/plugins/Resolution/Archiver.php
+++ b/plugins/Resolution/Archiver.php
@@ -39,7 +39,14 @@ class Archiver extends \Piwik\Plugin\Archiver
self::RESOLUTION_RECORD_NAME,
self::CONFIGURATION_RECORD_NAME,
);
- $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableRecords,
+ $this->maximumRows,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
protected function aggregateByConfiguration()
diff --git a/plugins/UserCountry/Archiver.php b/plugins/UserCountry/Archiver.php
index d2203e2479..dcb96e2c66 100644
--- a/plugins/UserCountry/Archiver.php
+++ b/plugins/UserCountry/Archiver.php
@@ -62,7 +62,15 @@ class Archiver extends \Piwik\Plugin\Archiver
self::CITY_RECORD_NAME,
);
- $nameToCount = $this->getProcessor()->aggregateDataTableRecords($dataTableToSum);
+ $nameToCount = $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableToSum,
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false
+ );
$this->getProcessor()->insertNumericRecord(self::DISTINCT_COUNTRIES_METRIC,
$nameToCount[self::COUNTRY_RECORD_NAME]['level0']);
}
diff --git a/plugins/UserLanguage/Archiver.php b/plugins/UserLanguage/Archiver.php
index 14fd369eec..edfbfd8608 100644
--- a/plugins/UserLanguage/Archiver.php
+++ b/plugins/UserLanguage/Archiver.php
@@ -46,7 +46,14 @@ class Archiver extends \Piwik\Plugin\Archiver
$dataTableRecords = array(
self::LANGUAGE_RECORD_NAME,
);
- $this->getProcessor()->aggregateDataTableRecords($dataTableRecords, $this->maximumRows);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableRecords,
+ $this->maximumRows,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
protected function aggregateByLanguage()
diff --git a/plugins/VisitTime/Archiver.php b/plugins/VisitTime/Archiver.php
index e7fd9c7a19..63be1dbecb 100644
--- a/plugins/VisitTime/Archiver.php
+++ b/plugins/VisitTime/Archiver.php
@@ -30,7 +30,14 @@ class Archiver extends \Piwik\Plugin\Archiver
self::LOCAL_TIME_RECORD_NAME,
self::SERVER_TIME_RECORD_NAME,
);
- $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableRecords,
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
protected function aggregateByServerTime()
diff --git a/plugins/VisitorInterest/Archiver.php b/plugins/VisitorInterest/Archiver.php
index cba45c6934..67a963ffcf 100644
--- a/plugins/VisitorInterest/Archiver.php
+++ b/plugins/VisitorInterest/Archiver.php
@@ -128,7 +128,14 @@ class Archiver extends \Piwik\Plugin\Archiver
self::VISITS_COUNT_RECORD_NAME,
self::DAYS_SINCE_LAST_RECORD_NAME
);
- $this->getProcessor()->aggregateDataTableRecords($dataTableRecords);
+ $this->getProcessor()->aggregateDataTableRecords(
+ $dataTableRecords,
+ $maximumRowsInDataTableLevelZero = null,
+ $maximumRowsInSubDataTable = null,
+ $columnToSortByBeforeTruncation = null,
+ $columnsAggregationOperation = null,
+ $columnsToRenameAfterAggregation = null,
+ $countRowsRecursive = false);
}
/**
--
GitLab