From db1991fdd02f225b13cd725d7b8a20b2b1e2b32f Mon Sep 17 00:00:00 2001 From: benakamoorthi <benaka.moorthi@gmail.com> Date: Thu, 13 Sep 2012 02:22:04 +0000 Subject: [PATCH] Refs #3330, #766, #3227 use RankingQuery and truncate tables as they are created in Actions plugin. Also modified phpunit integration testing mechanism so all API calls are tested and outputted before a test case throws. git-svn-id: http://dev.piwik.org/svn/trunk@6980 59fd770c-687e-43c8-a1e3-f5a4ff64c105 --- config/global.ini.php | 5 + core/ArchiveProcessing/Day.php | 10 +- core/RankingQuery.php | 19 +- core/Segment.php | 10 +- plugins/Actions/Actions.php | 371 +++++++++++++++--- tests/PHPUnit/BenchmarkTestCase.php | 13 - .../Integration/BlobReportLimitingTest.php | 26 ++ tests/PHPUnit/IntegrationTestCase.php | 60 ++- ...ImportLogs__Actions.getDownloads_month.xml | 2 +- ...portLimiting__Actions.getDownloads_day.xml | 6 - ...eportLimiting__Actions.getOutlinks_day.xml | 6 - ...ortLimiting__Actions.getPageTitles_day.xml | 71 +--- ...eportLimiting__Actions.getPageUrls_day.xml | 14 +- 13 files changed, 443 insertions(+), 170 deletions(-) diff --git a/config/global.ini.php b/config/global.ini.php index 61fa2bdc9e..4f1ac9a0d9 100644 --- a/config/global.ini.php +++ b/config/global.ini.php @@ -234,6 +234,11 @@ datatable_archiving_maximum_rows_subtable_actions = 100 ; maximum number of rows for other tables (Providers, User settings configurations) datatable_archiving_maximum_rows_standard = 500 +; maximum number of rows to fetch from the database when archiving. if set to 0, no limit is used. +; this can be used to speed up the archiving process, but is only useful if you're site has a large +; amount of actions, referrers or custom variable name/value pairs. +archiving_ranking_query_row_limit = 50000 + ; by default, the real time Live! widget will update every 5 seconds and refresh with new visits/actions/etc. ; you can change the timeout so the widget refreshes more often, or not as frequently live_widget_refresh_after_seconds = 5 diff --git a/core/ArchiveProcessing/Day.php b/core/ArchiveProcessing/Day.php index 372615dfd1..32072084d6 100644 --- a/core/ArchiveProcessing/Day.php +++ b/core/ArchiveProcessing/Day.php @@ -729,7 +729,15 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing $row = $maybeDatatableRow; } - $dataTableToReturn->addRow($row); + if ($row->getMetadata('issummaryrow') == true) + { + $row->deleteMetadata('issummaryrow'); + $dataTableToReturn->addSummaryRow($row); + } + else + { + $dataTableToReturn->addRow($row); + } } return $dataTableToReturn; } diff --git a/core/RankingQuery.php b/core/RankingQuery.php index 8de988b7df..a743f898d3 100644 --- a/core/RankingQuery.php +++ b/core/RankingQuery.php @@ -80,6 +80,11 @@ class Piwik_RankingQuery */ private $partitionColumnValues = array(); + /** + * The value to use in the label of the 'Others' row. + * @var string + */ + private $othersLabelValue = 'Others'; /** * The constructor. @@ -103,6 +108,16 @@ class Piwik_RankingQuery $this->limit = $limit; } + /** + * Set the value to use for the label in the 'Others' row. + * + * @param $value string + */ + public function setOthersLabel($value) + { + $this->othersLabelValue = $value; + } + /** * Add a label column. * Labels are the columns that are replaced with "Others" after the limit. @@ -273,7 +288,7 @@ class Piwik_RankingQuery { $labelColumnsOthersSwitch[] = " CASE - WHEN counter = $limit THEN \"Others\" + WHEN counter = $limit THEN \"{$this->othersLabelValue}\" ELSE `$column` END AS `$column` "; @@ -380,4 +395,4 @@ class Piwik_RankingQuery "; } -} \ No newline at end of file +} diff --git a/core/Segment.php b/core/Segment.php index 5b5c2d40ee..6824206fed 100644 --- a/core/Segment.php +++ b/core/Segment.php @@ -398,7 +398,7 @@ class Piwik_Segment */ private function buildWrappedSelectQuery($select, $from, $where, $orderBy, $groupBy) { - preg_match_all("/(log_visit|log_conversion).[a-z0-9_\*]+/", $select, $matches); + preg_match_all("/(log_visit|log_conversion|log_action).[a-z0-9_\*]+/", $select, $matches); $neededFields = array_unique($matches[0]); if (count($neededFields) == 0) @@ -407,9 +407,9 @@ class Piwik_Segment ."Please use a table prefix."); } - $select = preg_replace('/(log_visit|log_conversion)\./', 'log_inner.', $select); - $orderBy = preg_replace('/(log_visit|log_conversion)\./', 'log_inner.', $orderBy); - $groupBy = preg_replace('/(log_visit|log_conversion)\./', 'log_inner.', $groupBy); + $select = preg_replace('/(log_visit|log_conversion|log_action)\./', 'log_inner.', $select); + $orderBy = preg_replace('/(log_visit|log_conversion|log_action)\./', 'log_inner.', $orderBy); + $groupBy = preg_replace('/(log_visit|log_conversion|log_action)\./', 'log_inner.', $groupBy); $from = "( SELECT @@ -426,4 +426,4 @@ class Piwik_Segment return $this->buildSelectQuery($select, $from, $where, $orderBy, $groupBy); } -} \ No newline at end of file +} diff --git a/plugins/Actions/Actions.php b/plugins/Actions/Actions.php index f7e8722867..ff1c653b1a 100644 --- a/plugins/Actions/Actions.php +++ b/plugins/Actions/Actions.php @@ -28,6 +28,13 @@ class Piwik_Actions extends Piwik_Plugin protected $maximumRowsInSubDataTable; protected $columnToSortByBeforeTruncation; + const OTHERS_ROW_KEY = ''; + + /** + * The maximum number of rows to get from the database per action type. + */ + protected $rankingQueryRowLimit; + public function getInformation() { $info = array( @@ -439,6 +446,11 @@ class Piwik_Actions extends Piwik_Plugin ); public function __construct() + { + $this->reloadConfig(); + } + + public function reloadConfig() { // for BC, we read the old style delimiter first (see #1067) $actionDelimiter = @Piwik_Config::getInstance()->General['action_category_delimiter']; @@ -456,6 +468,7 @@ class Piwik_Actions extends Piwik_Plugin $this->columnToSortByBeforeTruncation = Piwik_Archive::INDEX_NB_VISITS; $this->maximumRowsInDataTableLevelZero = Piwik_Config::getInstance()->General['datatable_archiving_maximum_rows_actions']; $this->maximumRowsInSubDataTable = Piwik_Config::getInstance()->General['datatable_archiving_maximum_rows_subtable_actions']; + $this->rankingQueryRowLimit = Piwik_Config::getInstance()->General['archiving_ranking_query_row_limit']; // Piwik_DataTable::MAXIMUM_DEPTH_LEVEL_ALLOWED must be greater than the category level limit Piwik_DataTable::setMaximumDepthLevelAllowedAtLeast(self::getSubCategoryLevelLimit() + 1); @@ -521,6 +534,8 @@ class Piwik_Actions extends Piwik_Plugin Piwik_Archive::INDEX_PAGE_NB_HITS => 1, ))); + $rankingQuery = false; + /* * Page URLs and Page names, general stats */ @@ -547,26 +562,63 @@ class Piwik_Actions extends Piwik_Plugin AND log_link_visit_action.%s IS NOT NULL"; $groupBy = "log_action.idaction"; - $orderBy = "`". Piwik_Archive::INDEX_PAGE_NB_HITS ."` DESC"; + $orderBy = "`". Piwik_Archive::INDEX_PAGE_NB_HITS ."` DESC, name ASC"; + + if ($this->rankingQueryRowLimit > 0) + { + $rankingQuery = new Piwik_RankingQuery($this->rankingQueryRowLimit); + $rankingQuery->setOthersLabel('-1'); + $rankingQuery->addLabelColumn(array('idaction', 'name')); + $rankingQuery->addColumn(array('url_prefix', Piwik_Archive::INDEX_NB_UNIQ_VISITORS)); + $rankingQuery->addColumn(array(Piwik_Archive::INDEX_PAGE_NB_HITS, Piwik_Archive::INDEX_NB_VISITS), 'sum'); + $rankingQuery->partitionResultIntoMultipleGroups('type', array_keys($this->actionsTablesByType)); + } $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, - "idaction_url", $archiveProcessing); + "idaction_url", $archiveProcessing, $rankingQuery); $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, - "idaction_name", $archiveProcessing); + "idaction_name", $archiveProcessing, $rankingQuery); /* * Entry actions for Page URLs and Page names */ - $select = "log_visit.%s as idaction, + if ($this->rankingQueryRowLimit > 0) + { + $rankingQuery = new Piwik_RankingQuery($this->rankingQueryRowLimit); + $rankingQuery->setOthersLabel('-1'); + $rankingQuery->addLabelColumn('idaction'); + $rankingQuery->addColumn(Piwik_Archive::INDEX_PAGE_ENTRY_NB_UNIQ_VISITORS); + $rankingQuery->addColumn(array(Piwik_Archive::INDEX_PAGE_ENTRY_NB_VISITS, + Piwik_Archive::INDEX_PAGE_ENTRY_NB_ACTIONS, + Piwik_Archive::INDEX_PAGE_ENTRY_SUM_VISIT_LENGTH, + Piwik_Archive::INDEX_PAGE_ENTRY_BOUNCE_COUNT), 'sum'); + $rankingQuery->partitionResultIntoMultipleGroups('type', array_keys($this->actionsTablesByType)); + + $extraSelects = 'log_action.type, log_action.name,'; + $from = array( + "log_visit", + array( + "table" => "log_action", + "joinOn" => "log_visit.%s = log_action.idaction" + ) + ); + $orderBy = "`".Piwik_Archive::INDEX_PAGE_ENTRY_NB_ACTIONS."` DESC, log_action.name ASC"; + } + else + { + $extraSelects = false; + $from = "log_visit"; + $orderBy = false; + } + + $select = "log_visit.%s as idaction, $extraSelects count(distinct log_visit.idvisitor) as `". Piwik_Archive::INDEX_PAGE_ENTRY_NB_UNIQ_VISITORS ."`, count(*) as `". Piwik_Archive::INDEX_PAGE_ENTRY_NB_VISITS ."`, sum(log_visit.visit_total_actions) as `". Piwik_Archive::INDEX_PAGE_ENTRY_NB_ACTIONS ."`, sum(log_visit.visit_total_time) as `". Piwik_Archive::INDEX_PAGE_ENTRY_SUM_VISIT_LENGTH ."`, sum(case log_visit.visit_total_actions when 1 then 1 when 0 then 1 else 0 end) as `". Piwik_Archive::INDEX_PAGE_ENTRY_BOUNCE_COUNT ."`"; - $from = "log_visit"; - $where = "log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? AND log_visit.idsite = ? @@ -574,21 +626,45 @@ class Piwik_Actions extends Piwik_Plugin $groupBy = "log_visit.%s, idaction"; - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "visit_entry_idaction_url", $archiveProcessing); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "visit_entry_idaction_url", $archiveProcessing, $rankingQuery); - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "visit_entry_idaction_name", $archiveProcessing); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "visit_entry_idaction_name", $archiveProcessing, $rankingQuery); /* * Exit actions */ - $select = "log_visit.%s as idaction, + if ($this->rankingQueryRowLimit > 0) + { + $rankingQuery = new Piwik_RankingQuery($this->rankingQueryRowLimit); + $rankingQuery->setOthersLabel('-1'); + $rankingQuery->addLabelColumn('idaction'); + $rankingQuery->addColumn(Piwik_Archive::INDEX_PAGE_EXIT_NB_UNIQ_VISITORS); + $rankingQuery->addColumn(Piwik_Archive::INDEX_PAGE_EXIT_NB_VISITS, 'sum'); + $rankingQuery->partitionResultIntoMultipleGroups('type', array_keys($this->actionsTablesByType)); + + $extraSelects = 'log_action.type, log_action.name,'; + $from = array( + "log_visit", + array( + "table" => "log_action", + "joinOn" => "log_visit.%s = log_action.idaction" + ) + ); + $orderBy = "`".Piwik_Archive::INDEX_PAGE_EXIT_NB_VISITS."` DESC, log_action.name ASC"; + } + else + { + $extraSelects = false; + $from = "log_visit"; + $orderBy = false; + } + + $select = "log_visit.%s as idaction, $extraSelects count(distinct log_visit.idvisitor) as `". Piwik_Archive::INDEX_PAGE_EXIT_NB_UNIQ_VISITORS ."`, count(*) as `". Piwik_Archive::INDEX_PAGE_EXIT_NB_VISITS ."`"; - $from = "log_visit"; - $where = "log_visit.visit_last_action_time >= ? AND log_visit.visit_last_action_time <= ? AND log_visit.idsite = ? @@ -596,20 +672,42 @@ class Piwik_Actions extends Piwik_Plugin $groupBy = "log_visit.%s, idaction"; - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "visit_exit_idaction_url", $archiveProcessing); - - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "visit_exit_idaction_name", $archiveProcessing); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "visit_exit_idaction_url", $archiveProcessing, $rankingQuery); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "visit_exit_idaction_name", $archiveProcessing, $rankingQuery); /* * Time per action */ - $select = "log_link_visit_action.%s as idaction, - sum(log_link_visit_action.time_spent_ref_action) as `".Piwik_Archive::INDEX_PAGE_SUM_TIME_SPENT."`"; + if ($this->rankingQueryRowLimit > 0) + { + $rankingQuery = new Piwik_RankingQuery($this->rankingQueryRowLimit); + $rankingQuery->setOthersLabel('-1'); + $rankingQuery->addLabelColumn('idaction'); + $rankingQuery->addColumn(Piwik_Archive::INDEX_PAGE_SUM_TIME_SPENT, 'sum'); + $rankingQuery->partitionResultIntoMultipleGroups('type', array_keys($this->actionsTablesByType)); + + $extraSelects = "log_action.type, log_action.name, count(*) as `".Piwik_Archive::INDEX_PAGE_NB_HITS."`,"; + $from = array( + "log_link_visit_action", + array( + "table" => "log_action", + "joinOn" => "log_link_visit_action.%s = log_action.idaction" + ) + ); + $orderBy = "`".Piwik_Archive::INDEX_PAGE_NB_HITS."` DESC, log_action.name ASC"; + } + else + { + $extraSelects = false; + $from = "log_link_visit_action"; + $orderBy = false; + } - $from = "log_link_visit_action"; + $select = "log_link_visit_action.%s as idaction, $extraSelects + sum(log_link_visit_action.time_spent_ref_action) as `".Piwik_Archive::INDEX_PAGE_SUM_TIME_SPENT."`"; $where = "log_link_visit_action.server_time >= ? AND log_link_visit_action.server_time <= ? @@ -619,11 +717,11 @@ class Piwik_Actions extends Piwik_Plugin $groupBy = "log_link_visit_action.%s, idaction"; - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "idaction_url_ref", $archiveProcessing); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "idaction_url_ref", $archiveProcessing, $rankingQuery); - $this->archiveDayQueryProcess($select, $from, $where, $orderBy=false, $groupBy, - "idaction_name_ref", $archiveProcessing); + $this->archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, + "idaction_name_ref", $archiveProcessing, $rankingQuery); // Empty static cache self::$cacheParsedAction = array(); @@ -640,10 +738,11 @@ class Piwik_Actions extends Piwik_Plugin * @param $groupBy * @param $sprintfField * @param Piwik_ArchiveProcessing $archiveProcessing + * @param Piwik_RankingQuery|false $rankingQuery * @return int */ protected function archiveDayQueryProcess($select, $from, $where, $orderBy, $groupBy, - $sprintfField, $archiveProcessing) + $sprintfField, $archiveProcessing, $rankingQuery = false) { // idaction field needs to be set in select clause before calling getSelectQuery(). // if a complex segmentation join is needed, the field needs to be propagated @@ -652,13 +751,18 @@ class Piwik_Actions extends Piwik_Plugin // get query with segmentation $bind = array(); - $orderBy = false; $query = $archiveProcessing->getSegment()->getSelectQuery( $select, $from, $where, $bind, $orderBy, $groupBy); // replace the rest of the %s $querySql = str_replace("%s", $sprintfField, $query['sql']); + // apply ranking query + if ($rankingQuery) + { + $querySql = $rankingQuery->generateQuery($querySql); + } + // extend bindings $bind = array_merge(array($archiveProcessing->getStartDatetimeUTC(), $archiveProcessing->getEndDatetimeUTC(), $archiveProcessing->idsite), $query['bind']); @@ -706,15 +810,20 @@ class Piwik_Actions extends Piwik_Plugin protected function deleteInvalidSummedColumnsFromDataTable($dataTable) { - foreach($dataTable->getRows() as $row) + foreach($dataTable->getRows() as $id => $row) { - if(($idSubtable = $row->getIdSubDataTable()) !== null) + if(($idSubtable = $row->getIdSubDataTable()) !== null || $id === Piwik_DataTable::ID_SUMMARY_ROW) { foreach(self::$invalidSummedColumnNameToDeleteFromDayArchive as $name) { $row->deleteColumn($name); } - $this->deleteInvalidSummedColumnsFromDataTable(Piwik_DataTable_Manager::getInstance()->getTable($idSubtable)); + + if ($idSubtable !== null) + { + $subtable = Piwik_DataTable_Manager::getInstance()->getTable($idSubtable); + $this->deleteInvalidSummedColumnsFromDataTable($subtable); + } } } } @@ -847,7 +956,55 @@ class Piwik_Actions extends Piwik_Plugin const CACHE_PARSED_INDEX_NAME = 0; const CACHE_PARSED_INDEX_TYPE = 1; static $cacheParsedAction = array(); - + + /** + * Get cached action row by id & type. If $idAction is set to -1, the 'Others' row + * for the specific action type will be returned. + * + * @param int $idAction + * @param int $actionType + * @return Piwik_DataTable_Row|false + */ + private static function getCachedActionRow( $idAction, $actionType ) + { + $cacheLabel = self::getCachedActionRowKey($idAction, $actionType); + + if (!isset(self::$cacheParsedAction[$cacheLabel])) + { + // This can happen when + // - We select an entry page ID that was only seen yesterday, so wasn't selected in the first query + // - We count time spent on a page, when this page was only seen yesterday + return false; + } + + return self::$cacheParsedAction[$cacheLabel]; + } + + /** + * Set cached action row for an id & type. + * + * @param int $idAction + * @param int $actionType + * @param Piwik_DataTable_Row + */ + private static function setCachedActionRow( $idAction, $actionType, $actionRow ) + { + $cacheLabel = self::getCachedActionRowKey($idAction, $actionType); + self::$cacheParsedAction[$cacheLabel] = $actionRow; + } + + /** + * Gets the key for the cache of action rows from an action ID and type. + * + * @param int $idAction + * @param int $actionType + * @return string|int + */ + private static function getCachedActionRowKey( $idAction, $actionType ) + { + return $idAction == -1 ? $actionType.'_others' : $idAction; + } + /** * @param Zend_Db_Statement|PDOStatement $query * @param string|bool $fieldQueried @@ -873,28 +1030,27 @@ class Piwik_Actions extends Piwik_Plugin $actionName = $row['name']; $actionType = $row['type']; $urlPrefix = $row['url_prefix']; + $idaction = $row['idaction']; // in some unknown case, the type field is NULL, as reported in #1082 - we ignore this page view if(empty($actionType)) { - self::$cacheParsedAction[$row['idaction']] = false; + if ($idaction != -1) + { + self::setCachedActionRow($idaction, $actionType, false); + } + continue; } $currentTable = $this->parseActionNameCategoriesInDataTable($actionName, $actionType, $urlPrefix); - self::$cacheParsedAction[$row['idaction']] = $currentTable; + self::setCachedActionRow($idaction, $actionType, $currentTable); } else { - if(!isset(self::$cacheParsedAction[$row['idaction']])) - { - // This can happen when - // - We select an entry page ID that was only seen yesterday, so wasn't selected in the first query - // - We count time spent on a page, when this page was only seen yesterday - continue; - } - $currentTable = self::$cacheParsedAction[$row['idaction']]; + $currentTable = self::getCachedActionRow($row['idaction'], $row['type']); + // Action processed as "to skip" for some reasons if($currentTable === false) { @@ -906,6 +1062,12 @@ class Piwik_Actions extends Piwik_Plugin unset($row['type']); unset($row['idaction']); unset($row['url_prefix']); + + if (is_null($currentTable)) + { + continue; + } + foreach($row as $name => $value) { // in some edge cases, we have twice the same action name with 2 different idaction @@ -955,17 +1117,38 @@ class Piwik_Actions extends Piwik_Plugin { // we work on the root table of the given TYPE (either ACTION_URL or DOWNLOAD or OUTLINK etc.) $currentTable =& $this->actionsTablesByType[$actionType]; - + + // check for ranking query cut-off + if ($actionName == -1) + { + return $this->createOthersRow($currentTable, $actionType); + } + // go to the level of the subcategory $actionExplodedNames = $this->getActionExplodedNames($actionName, $actionType, $urlPrefix); $end = count($actionExplodedNames)-1; + $maxRows = $this->maximumRowsInDataTableLevelZero; for($level = 0 ; $level < $end; $level++) { $actionCategory = $actionExplodedNames[$level]; + + $othersRow = $this->getOthersRowIfTableFull($currentTable, $actionCategory, $actionType, $maxRows); + if ($othersRow) + { + return $othersRow; + } + $currentTable =& $currentTable[$actionCategory]; + $maxRows = $this->maximumRowsInSubDataTable; } $actionShortName = $actionExplodedNames[$end]; - + + $othersRow = $this->getOthersRowIfTableFull($currentTable, $actionShortName, $actionType, $maxRows); + if ($othersRow) + { + return $othersRow; + } + // currentTable is now the array element corresponding the the action // at this point we may be for example at the 4th level of depth in the hierarchy $currentTable =& $currentTable[$actionShortName]; @@ -973,29 +1156,95 @@ class Piwik_Actions extends Piwik_Plugin // add the row to the matching sub category subtable if(!($currentTable instanceof Piwik_DataTable_Row)) { - $defaultColumnsNewRow = array( - 'label' => (string)$actionShortName, - Piwik_Archive::INDEX_NB_VISITS => 0, - Piwik_Archive::INDEX_NB_UNIQ_VISITORS => 0, - Piwik_Archive::INDEX_PAGE_NB_HITS => 0, - Piwik_Archive::INDEX_PAGE_SUM_TIME_SPENT => 0, - ); - if( $actionType == Piwik_Tracker_Action::TYPE_ACTION_NAME ) + $currentTable = $this->createActionsTableRow( + (string)$actionShortName, $actionType, $actionName, $urlPrefix); + } + return $currentTable; + } + + /** + * Checks if the given table is full (has the maximum number of rows allowed in config) + * and if so, returns an 'Others' summary row. Returns false if the table is not full. + * + * @param array $currentTable Array of Piwik_DataTable_Rows. + * @param string $actionCategory The current table key. + * @param int $actionType The action type. + * @param int $maxRows The maximum number of rows allowed in $currentTable. + * @return Piwik_DataTable_Row|false + */ + private function getOthersRowIfTableFull( &$currentTable, $actionCategory, $actionType, $maxRows ) + { + if (!isset($currentTable[$actionCategory])) + { + if (count($currentTable) == $maxRows - 1) { - $currentTable = new Piwik_DataTable_Row(array( - Piwik_DataTable_Row::COLUMNS => $defaultColumnsNewRow, - )); + return $this->createOthersRow($currentTable, $actionType); } - else + else if (count($currentTable) >= $maxRows) { - $currentTable = new Piwik_DataTable_Row(array( - Piwik_DataTable_Row::COLUMNS => $defaultColumnsNewRow, - Piwik_DataTable_Row::METADATA => array('url' => - Piwik_Tracker_Action::reconstructNormalizedUrl((string)$actionName, $urlPrefix)), - )); + return $currentTable[self::OTHERS_ROW_KEY]; // return existing 'others' row } } - return $currentTable; + + return false; + } + + /** + * Create an 'Others' action row. The row created by this function is used + * as the summary row in a truncated DataTable. + * + * @param array $currentTable The array of rows to add the 'Others' row to. + * @param int $actionType The type of actions the row will hold stats for. + * @return Piwik_DataTable_Row + */ + private function createOthersRow( &$currentTable, $actionType ) + { + // create other row and return it + $othersRow = $this->createActionsTableRow(Piwik_DataTable::LABEL_SUMMARY_ROW, $actionType); + $othersRow->setMetadata('issummaryrow', true); + + $currentTable[self::OTHERS_ROW_KEY] = $othersRow; + return $othersRow; + } + + /** + * Creates a new empty row for DataTables created by this plugin. + * + * @param string $label The row label. + * @param int $actionType The action type of the action the row will describe. + * @param string $actionName + * @param string $urlPrefix + * @return Piwik_DataTable_Row + */ + private function createActionsTableRow( $label, $actionType, $actionName = null, $urlPrefix = null ) + { + $defaultColumnsNewRow = array( + 'label' => $label, + Piwik_Archive::INDEX_NB_VISITS => 0, + Piwik_Archive::INDEX_NB_UNIQ_VISITORS => 0, + Piwik_Archive::INDEX_PAGE_NB_HITS => 0, + Piwik_Archive::INDEX_PAGE_SUM_TIME_SPENT => 0, + ); + if( $actionType == Piwik_Tracker_Action::TYPE_ACTION_NAME ) + { + return new Piwik_DataTable_Row(array( + Piwik_DataTable_Row::COLUMNS => $defaultColumnsNewRow, + )); + } + else + { + $metadata = array(); + if (!is_null($actionName)) + { + $url = Piwik_Tracker_Action::reconstructNormalizedUrl((string)$actionName, $urlPrefix); + $metadata['url'] = $url; + } + + return new Piwik_DataTable_Row(array( + Piwik_DataTable_Row::COLUMNS => $defaultColumnsNewRow, + Piwik_DataTable_Row::METADATA => $metadata, + )); + } } /** diff --git a/tests/PHPUnit/BenchmarkTestCase.php b/tests/PHPUnit/BenchmarkTestCase.php index e750f91492..aee39f8a83 100755 --- a/tests/PHPUnit/BenchmarkTestCase.php +++ b/tests/PHPUnit/BenchmarkTestCase.php @@ -81,19 +81,6 @@ abstract class BenchmarkTestCase extends IntegrationTestCase parent::tearDownAfterClass($dropDatabase); } - /** - * Drops all archive tables. - */ - public static function deleteArchiveTables() - { - foreach (Piwik::getTablesArchivesInstalled() as $table) - { - Piwik_Query("DROP TABLE IF EXISTS $table"); - } - - Piwik_TablePartitioning::$tablesAlreadyInstalled = Piwik::getTablesInstalled($forceReload = true); - } - /** * Creates a tracking object that invokes the tracker directly (w/o going through HTTP). */ diff --git a/tests/PHPUnit/Integration/BlobReportLimitingTest.php b/tests/PHPUnit/Integration/BlobReportLimitingTest.php index ae9db250d1..3ba5916613 100755 --- a/tests/PHPUnit/Integration/BlobReportLimitingTest.php +++ b/tests/PHPUnit/Integration/BlobReportLimitingTest.php @@ -58,6 +58,31 @@ class Test_Piwik_Integration_BlobReportLimitingTest extends IntegrationTestCase { $this->runApiTests($api, $params); } + + /** + * @group Integration + * @group BlobReportLimiting + */ + public function testApiWithRankingQuery() + { + // custom setup + self::deleteArchiveTables(); + $generalConfig['datatable_archiving_maximum_rows_referers'] = 4; + $generalConfig['datatable_archiving_maximum_rows_subtable_referers'] = 4; + $generalConfig['datatable_archiving_maximum_rows_actions'] = 4; + $generalConfig['datatable_archiving_maximum_rows_subtable_actions'] = 4; + $generalConfig['datatable_archiving_maximum_rows_standard'] = 4; + Piwik_Config::getInstance()->General['archiving_ranking_query_row_limit'] = 3; + Piwik_PluginsManager::getInstance()->getLoadedPlugin('Actions')->reloadConfig(); + + foreach ($this->getApiForTesting() as $pair) + { + list($apiToCall, $params) = $pair; + $params['testSuffix'] = '_rankingQuery'; + + $this->runApiTests($apiToCall, $params); + } + } public function getOutputPrefix() { @@ -73,6 +98,7 @@ class Test_Piwik_Integration_BlobReportLimitingTest extends IntegrationTestCase $generalConfig['datatable_archiving_maximum_rows_actions'] = 3; $generalConfig['datatable_archiving_maximum_rows_subtable_actions'] = 2; $generalConfig['datatable_archiving_maximum_rows_standard'] = 3; + $generalConfig['archiving_ranking_query_row_limit'] = 50000; } protected static function setUpWebsitesAndGoals() diff --git a/tests/PHPUnit/IntegrationTestCase.php b/tests/PHPUnit/IntegrationTestCase.php index 6ca0775a39..f393e6326b 100755 --- a/tests/PHPUnit/IntegrationTestCase.php +++ b/tests/PHPUnit/IntegrationTestCase.php @@ -172,6 +172,9 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase ); const DEFAULT_USER_PASSWORD = 'nopass'; + + protected $missingExpectedFiles = array(); + protected $comparisonFailures = array(); /** * Forces the test to only call and fetch XML for the specified plugins, @@ -828,15 +831,23 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase $response = str_replace('.1</revenue>', '</revenue>', $response); } - if (strpos($requestUrl, 'format=xml') !== false) { - $this->assertXmlStringEqualsXmlString($expected, $response, "Differences with expected in: $processedFilePath"); - } else { - $this->assertEquals(strlen($expected), strlen($response), "Differences with expected in: $processedFilePath"); - $this->assertEquals($expected, $response, "Differences with expected in: $processedFilePath"); - } - if (trim($response) == trim($expected)) { - file_put_contents($processedFilePath, $response); - } + try + { + if (strpos($requestUrl, 'format=xml') !== false) { + $this->assertXmlStringEqualsXmlString($expected, $response, "Differences with expected in: $processedFilePath"); + } else { + $this->assertEquals(strlen($expected), strlen($response), "Differences with expected in: $processedFilePath"); + $this->assertEquals($expected, $response, "Differences with expected in: $processedFilePath"); + } + + if (trim($response) == trim($expected)) { + file_put_contents($processedFilePath, $response); + } + } + catch (Exception $ex) + { + $this->comparisonFailures[] = $ex; + } } protected function removeAllLiveDatesFromXml($input) @@ -905,7 +916,7 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase if(empty($result)) { $expectedDir = dirname($filePath); - $this->markTestIncomplete(" ERROR: Could not find expected API output '$filePath'. For new tests, to pass the test, you can copy files from the processed/ directory into $expectedDir after checking that the output is valid. %s "); + $this->missingExpectedFiles[] = $filePath; return null; } return $result; @@ -986,6 +997,8 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase protected function runApiTests($api, $params) { $testName = 'test_' . $this->getOutputPrefix(); + $this->missingExpectedFiles = array(); + $this->comparisonFailures = array(); $this->_setCallableApi($api); @@ -1032,6 +1045,20 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase { $this->changeLanguage('en'); } + + if (!empty($this->missingExpectedFiles)) + { + $expectedDir = dirname(reset($this->missingExpectedFiles)); + $this->markTestIncomplete(" ERROR: Could not find expected API output '" + . implode("', '", $this->missingExpectedFiles) + . "'. For new tests, to pass the test, you can copy files from the processed/ directory into" + . " $expectedDir after checking that the output is valid. %s "); + } + + if (!empty($this->comparisonFailures)) + { + throw reset($this->comparisonFailures); + } } /** @@ -1141,4 +1168,17 @@ abstract class IntegrationTestCase extends PHPUnit_Framework_TestCase Piwik_Query($sql); } } + + /** + * Drops all archive tables. + */ + public static function deleteArchiveTables() + { + foreach (Piwik::getTablesArchivesInstalled() as $table) + { + Piwik_Query("DROP TABLE IF EXISTS $table"); + } + + Piwik_TablePartitioning::$tablesAlreadyInstalled = Piwik::getTablesInstalled($forceReload = true); + } } diff --git a/tests/integration/expected/test_ImportLogs__Actions.getDownloads_month.xml b/tests/integration/expected/test_ImportLogs__Actions.getDownloads_month.xml index 5be39d8391..13b1f1c594 100755 --- a/tests/integration/expected/test_ImportLogs__Actions.getDownloads_month.xml +++ b/tests/integration/expected/test_ImportLogs__Actions.getDownloads_month.xml @@ -5,11 +5,11 @@ <nb_visits>3</nb_visits> <nb_hits>3</nb_hits> <sum_time_spent>122</sum_time_spent> + <exit_nb_visits>1</exit_nb_visits> <entry_nb_visits>1</entry_nb_visits> <entry_nb_actions>3</entry_nb_actions> <entry_sum_visit_length>123</entry_sum_visit_length> <entry_bounce_count>0</entry_bounce_count> - <exit_nb_visits>1</exit_nb_visits> <subtable> <row> <label>/other.png</label> diff --git a/tests/integration/expected/test_reportLimiting__Actions.getDownloads_day.xml b/tests/integration/expected/test_reportLimiting__Actions.getDownloads_day.xml index 417cacfb5f..6ecb433305 100755 --- a/tests/integration/expected/test_reportLimiting__Actions.getDownloads_day.xml +++ b/tests/integration/expected/test_reportLimiting__Actions.getDownloads_day.xml @@ -29,15 +29,12 @@ <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> </row> </subtable> @@ -71,15 +68,12 @@ <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> </row> </subtable> diff --git a/tests/integration/expected/test_reportLimiting__Actions.getOutlinks_day.xml b/tests/integration/expected/test_reportLimiting__Actions.getOutlinks_day.xml index b00793d913..c94e65c1da 100755 --- a/tests/integration/expected/test_reportLimiting__Actions.getOutlinks_day.xml +++ b/tests/integration/expected/test_reportLimiting__Actions.getOutlinks_day.xml @@ -29,15 +29,12 @@ <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> </row> </subtable> @@ -71,15 +68,12 @@ <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> </row> </subtable> diff --git a/tests/integration/expected/test_reportLimiting__Actions.getPageTitles_day.xml b/tests/integration/expected/test_reportLimiting__Actions.getPageTitles_day.xml index 82050a50a7..09fe1a1637 100755 --- a/tests/integration/expected/test_reportLimiting__Actions.getPageTitles_day.xml +++ b/tests/integration/expected/test_reportLimiting__Actions.getPageTitles_day.xml @@ -34,15 +34,12 @@ <row> <label>Others</label> <nb_visits>3</nb_visits> - <nb_uniq_visitors>3</nb_uniq_visitors> <nb_hits>3</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>3</entry_nb_uniq_visitors> <entry_nb_visits>3</entry_nb_visits> <entry_nb_actions>3</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>3</entry_bounce_count> - <exit_nb_uniq_visitors>3</exit_nb_uniq_visitors> <exit_nb_visits>3</exit_nb_visits> <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> @@ -51,68 +48,32 @@ </subtable> </row> <row> - <label>title_1</label> - <nb_visits>4</nb_visits> - <nb_hits>4</nb_hits> + <label> title_0</label> + <nb_visits>1</nb_visits> + <nb_uniq_visitors>1</nb_uniq_visitors> + <nb_hits>1</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_visits>4</entry_nb_visits> - <entry_nb_actions>4</entry_nb_actions> + <entry_nb_uniq_visitors>1</entry_nb_uniq_visitors> + <entry_nb_visits>1</entry_nb_visits> + <entry_nb_actions>1</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> - <entry_bounce_count>4</entry_bounce_count> - <exit_nb_visits>4</exit_nb_visits> + <entry_bounce_count>1</entry_bounce_count> + <exit_nb_uniq_visitors>1</exit_nb_uniq_visitors> + <exit_nb_visits>1</exit_nb_visits> <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> <exit_rate>100%</exit_rate> - <subtable> - <row> - <label> title_4</label> - <nb_visits>1</nb_visits> - <nb_uniq_visitors>1</nb_uniq_visitors> - <nb_hits>1</nb_hits> - <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>1</entry_nb_uniq_visitors> - <entry_nb_visits>1</entry_nb_visits> - <entry_nb_actions>1</entry_nb_actions> - <entry_sum_visit_length>0</entry_sum_visit_length> - <entry_bounce_count>1</entry_bounce_count> - <exit_nb_uniq_visitors>1</exit_nb_uniq_visitors> - <exit_nb_visits>1</exit_nb_visits> - <avg_time_on_page>0</avg_time_on_page> - <bounce_rate>100%</bounce_rate> - <exit_rate>100%</exit_rate> - </row> - <row> - <label>Others</label> - <nb_visits>3</nb_visits> - <nb_uniq_visitors>3</nb_uniq_visitors> - <nb_hits>3</nb_hits> - <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>3</entry_nb_uniq_visitors> - <entry_nb_visits>3</entry_nb_visits> - <entry_nb_actions>3</entry_nb_actions> - <entry_sum_visit_length>0</entry_sum_visit_length> - <entry_bounce_count>3</entry_bounce_count> - <exit_nb_uniq_visitors>3</exit_nb_uniq_visitors> - <exit_nb_visits>3</exit_nb_visits> - <avg_time_on_page>0</avg_time_on_page> - <bounce_rate>100%</bounce_rate> - <exit_rate>100%</exit_rate> - </row> - </subtable> </row> <row> <label>Others</label> - <nb_visits>17</nb_visits> - <nb_hits>17</nb_hits> + <nb_visits>20</nb_visits> + <nb_hits>20</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_visits>17</entry_nb_visits> - <entry_nb_actions>17</entry_nb_actions> + <entry_nb_visits>20</entry_nb_visits> + <entry_nb_actions>20</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> - <entry_bounce_count>17</entry_bounce_count> - <exit_nb_visits>17</exit_nb_visits> - <nb_uniq_visitors>5</nb_uniq_visitors> - <entry_nb_uniq_visitors>5</entry_nb_uniq_visitors> - <exit_nb_uniq_visitors>5</exit_nb_uniq_visitors> + <entry_bounce_count>20</entry_bounce_count> + <exit_nb_visits>20</exit_nb_visits> <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> <exit_rate>100%</exit_rate> diff --git a/tests/integration/expected/test_reportLimiting__Actions.getPageUrls_day.xml b/tests/integration/expected/test_reportLimiting__Actions.getPageUrls_day.xml index d245c8cbf4..35bca66a4a 100755 --- a/tests/integration/expected/test_reportLimiting__Actions.getPageUrls_day.xml +++ b/tests/integration/expected/test_reportLimiting__Actions.getPageUrls_day.xml @@ -15,7 +15,7 @@ <exit_rate>100%</exit_rate> <subtable> <row> - <label>/0</label> + <label>/index</label> <nb_visits>1</nb_visits> <nb_uniq_visitors>1</nb_uniq_visitors> <nb_hits>1</nb_hits> @@ -30,20 +30,17 @@ <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> <exit_rate>100%</exit_rate> - <url>http://piwik.net/0/0</url> + <url>http://piwik.net/0/</url> </row> <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> @@ -66,7 +63,7 @@ <exit_rate>100%</exit_rate> <subtable> <row> - <label>/4</label> + <label>/index</label> <nb_visits>1</nb_visits> <nb_uniq_visitors>1</nb_uniq_visitors> <nb_hits>1</nb_hits> @@ -81,20 +78,17 @@ <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> <exit_rate>100%</exit_rate> - <url>http://piwik.net/1/4</url> + <url>http://piwik.net/1/</url> </row> <row> <label>Others</label> <nb_visits>4</nb_visits> - <nb_uniq_visitors>4</nb_uniq_visitors> <nb_hits>4</nb_hits> <sum_time_spent>0</sum_time_spent> - <entry_nb_uniq_visitors>4</entry_nb_uniq_visitors> <entry_nb_visits>4</entry_nb_visits> <entry_nb_actions>4</entry_nb_actions> <entry_sum_visit_length>0</entry_sum_visit_length> <entry_bounce_count>4</entry_bounce_count> - <exit_nb_uniq_visitors>4</exit_nb_uniq_visitors> <exit_nb_visits>4</exit_nb_visits> <avg_time_on_page>0</avg_time_on_page> <bounce_rate>100%</bounce_rate> -- GitLab