Skip to content
Extraits de code Groupes Projets
Valider dc681e6c rédigé par Thomas Steur's avatar Thomas Steur
Parcourir les fichiers

Merge pull request #8625 from piwik/diagnostic_archive_cmd

Adding diagnostic command that analyzes archive tables
parents 13eb55ec 6e22cc2e
Aucune branche associée trouvée
Aucune étiquette associée trouvée
Aucune requête de fusion associée trouvée
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\DataAccess;
use Piwik\Common;
use Piwik\Config;
use Piwik\Db;
/**
* Data Access class for querying numeric & blob archive tables.
*/
class ArchiveTableDao
{
/**
* Analyzes numeric & blob tables for a single table date (ie, `'2015_01'`) and returns
* statistics including:
*
* - number of archives present
* - number of invalidated archives
* - number of temporary archives
* - number of error archives
* - number of segment archives
* - number of numeric rows
* - number of blob rows
*
* @param string $tableDate ie `'2015_01'`
* @return array
*/
public function getArchiveTableAnalysis($tableDate)
{
$numericQueryEmptyRow = array(
'count_archives' => '-',
'count_invalidated_archives' => '-',
'count_temporary_archives' => '-',
'count_error_archives' => '-',
'count_segment_archives' => '-',
'count_numeric_rows' => '-',
);
$tableDate = str_replace("`", "", $tableDate); // for sanity
$numericTable = Common::prefixTable("archive_numeric_$tableDate");
$blobTable = Common::prefixTable("archive_blob_$tableDate");
// query numeric table
$sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
SUM(CASE WHEN name LIKE 'done%' THEN 1 ELSE 0 END) AS count_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_invalidated_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_temporary_archives,
SUM(CASE WHEN name LIKE 'done%' AND value = ? THEN 1 ELSE 0 END) AS count_error_archives,
SUM(CASE WHEN name LIKE 'done%' AND CHAR_LENGTH(name) > 32 THEN 1 ELSE 0 END) AS count_segment_archives,
SUM(CASE WHEN name NOT LIKE 'done%' THEN 1 ELSE 0 END) AS count_numeric_rows,
0 AS count_blob_rows
FROM `$numericTable`
GROUP BY idsite, date1, date2, period";
$rows = Db::fetchAll($sql, array(ArchiveWriter::DONE_INVALIDATED, ArchiveWriter::DONE_OK_TEMPORARY,
ArchiveWriter::DONE_ERROR));
// index result
$result = array();
foreach ($rows as $row) {
$result[$row['label']] = $row;
}
// query blob table & manually merge results (no FULL OUTER JOIN in mysql)
$sql = "SELECT CONCAT_WS('.', idsite, date1, date2, period) AS label,
COUNT(*) AS count_blob_rows
FROM `$blobTable`
GROUP BY idsite, date1, date1, period";
foreach (Db::fetchAll($sql) as $blobStatsRow) {
$label = $blobStatsRow['label'];
if (isset($result[$label])) {
$result[$label] = array_merge($result[$label], $blobStatsRow);
} else {
$result[$label] = $blobStatsRow + $numericQueryEmptyRow;
}
}
return $result;
}
}
\ No newline at end of file
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\Diagnostics\Commands;
use Piwik\Container\StaticContainer;
use Piwik\Piwik;
use Piwik\Plugin\ConsoleCommand;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Helper\Table;
/**
* Diagnostic command that analyzes a single archive table. Displays information like # of segment archives,
* # invalidated archives, # temporary archives, etc.
*/
class AnalyzeArchiveTable extends ConsoleCommand
{
protected function configure()
{
$this->setName('diagnostics:analyze-archive-table');
$this->setDescription('Analyze an archive table and display human readable information about what is stored. '
. 'This command can be used to diagnose issues like bloated archive tables.');
$this->addArgument('table-date', InputArgument::REQUIRED, "The table's associated date, eg, 2015_01 or 2015_02");
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$tableDate = $input->getArgument('table-date');
$output->writeln("<comment>Statistics for the archive_numeric_$tableDate and archive_blob_$tableDate tables:</comment>");
$output->writeln("");
$archiveTableDao = StaticContainer::get('Piwik\DataAccess\ArchiveTableDao');
$rows = $archiveTableDao->getArchiveTableAnalysis($tableDate);
// process labels
$periodIdsToLabels = array_flip(Piwik::$idPeriods);
foreach ($rows as $key => &$row) {
list($idSite, $date1, $date2, $period) = explode('.', $key);
$periodLabel = isset($periodIdsToLabels[$period]) ? $periodIdsToLabels[$period] : "Unknown Period ($period)";
$row['label'] = $periodLabel . "[" . $date1 . " - " . $date2 . "] idSite = " . $idSite;
}
$headers = array('Group', '# Archives', '# Invalidated', '# Temporary', '# Error', '# Segment',
'# Numeric Rows', '# Blob Rows');
// display all rows
$table = new Table($output);
$table->setHeaders($headers)->setRows($rows);
$table->render();
// display summary
$totalArchives = 0;
$totalInvalidated = 0;
$totalTemporary = 0;
$totalError = 0;
$totalSegment = 0;
foreach ($rows as $row) {
$totalArchives += $row['count_archives'];
$totalInvalidated += $row['count_invalidated_archives'];
$totalTemporary += $row['count_temporary_archives'];
$totalError += $row['count_error_archives'];
$totalSegment += $row['count_segment_archives'];
}
$output->writeln("");
$output->writeln("Total # Archives: <comment>$totalArchives</comment>");
$output->writeln("Total # Invalidated Archives: <comment>$totalInvalidated</comment>");
$output->writeln("Total # Temporary Archives: <comment>$totalTemporary</comment>");
$output->writeln("Total # Error Archives: <comment>$totalError</comment>");
$output->writeln("Total # Segment Archives: <comment>$totalSegment</comment>");
$output->writeln("");
}
}
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Plugins\Diagnostics\Test\Integration\Commands;
use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
use Piwik\Tests\Framework\TestCase\ConsoleCommandTestCase;
use Piwik\Plugins\VisitsSummary\API as VisitsSummaryAPI;
/**
* TODO: This could be a unit test if we could inject the ArchiveTableDao in the command
*/
class AnalyzeArchiveTableTest extends ConsoleCommandTestCase
{
/**
* @var OneVisitorTwoVisits
*/
public static $fixture = null;
public static function setUpBeforeClass()
{
parent::setUpBeforeClass();
// make sure archiving is initiated so there is data in the archive tables
VisitsSummaryAPI::getInstance()->get(self::$fixture->idSite, 'month', '2010-03-01');
VisitsSummaryAPI::getInstance()->get(self::$fixture->idSite, 'month', '2010-03-01', 'browserCode==FF');
VisitsSummaryAPI::getInstance()->get(self::$fixture->idSite, 'month', '2010-03-01', 'daysSinceFirstVisit==2');
}
public function test_CommandOutput_IsAsExpected()
{
$expected = <<<OUTPUT
Statistics for the archive_numeric_2010_03 and archive_blob_2010_03 tables:
+-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+
| Group | # Archives | # Invalidated | # Temporary | # Error | # Segment | # Numeric Rows | # Blob Rows |
+-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+
| week[2010-03-01 - 2010-03-07] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 36 | 64 |
| month[2010-03-01 - 2010-03-31] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 36 | 64 |
| day[2010-03-03 - 2010-03-03] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-04 - 2010-03-04] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-05 - 2010-03-05] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-06 - 2010-03-06] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 36 | 52 |
| day[2010-03-07 - 2010-03-07] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-08 - 2010-03-08] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| week[2010-03-08 - 2010-03-14] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-09 - 2010-03-09] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-10 - 2010-03-10] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-11 - 2010-03-11] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-12 - 2010-03-12] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-13 - 2010-03-13] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-14 - 2010-03-14] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-15 - 2010-03-15] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| week[2010-03-15 - 2010-03-21] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-16 - 2010-03-16] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-17 - 2010-03-17] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-18 - 2010-03-18] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-19 - 2010-03-19] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-20 - 2010-03-20] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-21 - 2010-03-21] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-22 - 2010-03-22] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| week[2010-03-22 - 2010-03-28] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-23 - 2010-03-23] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-24 - 2010-03-24] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-25 - 2010-03-25] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-26 - 2010-03-26] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-27 - 2010-03-27] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-28 - 2010-03-28] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-29 - 2010-03-29] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-30 - 2010-03-30] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
| day[2010-03-31 - 2010-03-31] idSite = 1 | 3 | 0 | 0 | 0 | 2 | 0 | 0 |
+-------------------------------------------+------------+---------------+-------------+---------+-----------+----------------+-------------+
Total # Archives: 102
Total # Invalidated Archives: 0
Total # Temporary Archives: 0
Total # Error Archives: 0
Total # Segment Archives: 68
OUTPUT;
$this->applicationTester->run(array(
'command' => 'diagnostics:analyze-archive-table',
'table-date' => '2010_03',
));
$actual = $this->applicationTester->getDisplay();
$this->assertEquals($expected, $actual);
}
}
AnalyzeArchiveTableTest::$fixture = new OneVisitorTwoVisits();
\ No newline at end of file
<?php
/**
* Piwik - free/libre analytics platform
*
* @link http://piwik.org
* @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
*/
namespace Piwik\Tests\Integration\DataAccess;
use Piwik\Common;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\ArchiveTableDao;
use Piwik\DataAccess\ArchiveWriter;
use Piwik\Date;
use Piwik\Db;
use Piwik\Piwik;
use Piwik\Segment;
use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
/**
* @group Core
*/
class ArchiveTableDaoTest extends IntegrationTestCase
{
/**
* @var ArchiveTableDao
*/
private $archiveTableDao;
public function setUp()
{
parent::setUp();
$this->archiveTableDao = self::$fixture->piwikEnvironment->getContainer()->get(
'Piwik\DataAccess\ArchiveTableDao');
ArchiveTableCreator::getBlobTable(Date::factory('2015-01-01'));
ArchiveTableCreator::getNumericTable(Date::factory('2015-01-01'));
}
/**
*
*/
public function test_getArchiveTableAnalysis_QueriesNumericAndBlobTable_IncludingArchivesInBlobThatAreNotInNumeric()
{
$tableMonth = '2015_01';
$this->insertArchive($tableMonth, $idSite = 1, $period = 'day', $date1 = '2015-01-01', $date2 = '2015-01-01');
$this->insertArchive($tableMonth, $idSite = 2, $period = 'day', $date1 = '2015-01-03', $date2 = '2015-01-03');
$this->insertArchive($tableMonth, $idSite = 1, $period = 'week', $date1 = '2015-01-04', $date2 = '2015-01-11');
$this->insertArchive($tableMonth, $idSite = 3, $period = 'month', $date1 = '2015-01-01', $date2 = '2015-01-31');
$this->insertArchive($tableMonth, $idSite = 4, $period = 'year', $date1 = '2015-01-01', $date2 = '2015-12-31',
$segment = 'browserCode==FF');
$this->insertArchive($tableMonth, $idSite = 1, $period = 'range', $date1 = '2015-01-15', $date2 = '2015-01-20');
// invalid
$this->insertArchive($tableMonth, $idSite = 1, $period = 'day', $date1 = '2015-01-01', $date2 = '2015-01-01',
$segment = false, $doneValue = ArchiveWriter::DONE_INVALIDATED);
$this->insertArchive($tableMonth, $idSite = 1, $period = 'day', $date1 = '2015-01-01', $date2 = '2015-01-01',
$segment = false, $doneValue = ArchiveWriter::DONE_INVALIDATED);
$this->insertArchive($tableMonth, $idSite = 4, $period = 'year', $date1 = '2015-01-01', $date2 = '2015-12-31',
$segment = 'browserCode==FF', $doneValue = ArchiveWriter::DONE_INVALIDATED);
// temporary
$this->insertArchive($tableMonth, $idSite = 1, $period = 'week', $date1 = '2015-01-04', $date2 = '2015-01-11',
$segment = false, $doneValue = ArchiveWriter::DONE_OK_TEMPORARY);
$this->insertArchive($tableMonth, $idSite = 3, $period = 'month', $date1 = '2015-01-01', $date2 = '2015-01-31',
$segment = 'daysSinceFirstVisit==1', $doneValue = ArchiveWriter::DONE_OK_TEMPORARY);
// error
$this->insertArchive($tableMonth, $idSite = 1, $period = 'week', $date1 = '2015-01-04', $date2 = '2015-01-11',
$segment = false, $doneValue = ArchiveWriter::DONE_ERROR);
$this->insertArchive($tableMonth, $idSite = 3, $period = 'month', $date1 = '2015-01-01', $date2 = '2015-01-31',
$segment = 'daysSinceFirstVisit==1', $doneValue = ArchiveWriter::DONE_ERROR);
// blob only
$this->insertBlobArchive($tableMonth, $idSite = 1, $period = 'day', $date1 = '2015-01-20',
$date2 = '2015-01-20');
$this->insertBlobArchive($tableMonth, $idSite = 2, $period = 'day', $date1 = '2015-01-21',
$date2 = '2015-01-21', $segment = 'browserCode==SF');
$expectedStats = array(
'1.2015-01-01.2015-01-01.1' => array(
'label' => '1.2015-01-01.2015-01-01.1',
'count_archives' => '3',
'count_invalidated_archives' => '2',
'count_temporary_archives' => '0',
'count_error_archives' => '0',
'count_segment_archives' => '0',
'count_numeric_rows' => '9',
'count_blob_rows' => '9',
),
'1.2015-01-04.2015-01-11.2' => array(
'label' => '1.2015-01-04.2015-01-11.2',
'count_archives' => '3',
'count_invalidated_archives' => '0',
'count_temporary_archives' => '1',
'count_error_archives' => '1',
'count_segment_archives' => '0',
'count_numeric_rows' => '9',
'count_blob_rows' => '9',
),
'1.2015-01-15.2015-01-20.5' => array(
'label' => '1.2015-01-15.2015-01-20.5',
'count_archives' => '1',
'count_invalidated_archives' => '0',
'count_temporary_archives' => '0',
'count_error_archives' => '0',
'count_segment_archives' => '0',
'count_numeric_rows' => '3',
'count_blob_rows' => '3',
),
'2.2015-01-03.2015-01-03.1' => array(
'label' => '2.2015-01-03.2015-01-03.1',
'count_archives' => '1',
'count_invalidated_archives' => '0',
'count_temporary_archives' => '0',
'count_error_archives' => '0',
'count_segment_archives' => '0',
'count_numeric_rows' => '3',
'count_blob_rows' => '3',
),
'3.2015-01-01.2015-01-31.3' => array(
'label' => '3.2015-01-01.2015-01-31.3',
'count_archives' => '3',
'count_invalidated_archives' => '0',
'count_temporary_archives' => '1',
'count_error_archives' => '1',
'count_segment_archives' => '2',
'count_numeric_rows' => '9',
'count_blob_rows' => '9',
),
'4.2015-01-01.2015-12-31.4' => array(
'label' => '4.2015-01-01.2015-12-31.4',
'count_archives' => '2',
'count_invalidated_archives' => '1',
'count_temporary_archives' => '0',
'count_error_archives' => '0',
'count_segment_archives' => '2',
'count_numeric_rows' => '6',
'count_blob_rows' => '6',
),
'1.2015-01-20.2015-01-20.1' => array(
'label' => '1.2015-01-20.2015-01-20.1',
'count_blob_rows' => '3',
'count_archives' => '-',
'count_invalidated_archives' => '-',
'count_temporary_archives' => '-',
'count_error_archives' => '-',
'count_segment_archives' => '-',
'count_numeric_rows' => '-',
),
'2.2015-01-21.2015-01-21.1' => array(
'label' => '2.2015-01-21.2015-01-21.1',
'count_blob_rows' => '3',
'count_archives' => '-',
'count_invalidated_archives' => '-',
'count_temporary_archives' => '-',
'count_error_archives' => '-',
'count_segment_archives' => '-',
'count_numeric_rows' => '-',
),
);
$actualStats = $this->archiveTableDao->getArchiveTableAnalysis($tableMonth);
$this->assertEquals($expectedStats, $actualStats);
}
private function insertArchive($tableMonth, $idSite, $period, $date1, $date2, $segment = false,
$doneValue = ArchiveWriter::DONE_OK)
{
$this->insertNumericArchive($tableMonth, $idSite, $period, $date1, $date2, $segment, $doneValue);
$this->insertBlobArchive($tableMonth, $idSite, $period, $date1, $date2, $segment);
}
private function insertNumericArchive($tableMonth, $idSite, $period, $date1, $date2, $segment, $doneValue)
{
$this->insertRow('archive_numeric', $tableMonth, $idSite, $period, $date1, $date2, 'nb_schweetz', 2);
$this->insertRow('archive_numeric', $tableMonth, $idSite, $period, $date1, $date2, 'nb_fixes', 3);
$this->insertRow('archive_numeric', $tableMonth, $idSite, $period, $date1, $date2, 'nb_wrecks', 4);
$doneFlag = 'done';
if (!empty($segment)) {
$segmentObj = new Segment($segment, array());
$doneFlag .= $segmentObj->getHash();
}
$this->insertRow('archive_numeric', $tableMonth, $idSite, $period, $date1, $date2, $doneFlag, $doneValue);
}
private function insertBlobArchive($tableMonth, $idSite, $period, $date1, $date2, $segment = false)
{
$this->insertRow('archive_blob', $tableMonth, $idSite, $period, $date1, $date2, 'nb_cybugz', 'blob value 1');
$this->insertRow('archive_blob', $tableMonth, $idSite, $period, $date1, $date2, 'max_turbo', 'blob value 2');
$this->insertRow('archive_blob', $tableMonth, $idSite, $period, $date1, $date2, 'nb_fps', 'blob value 3');
}
private function insertRow($type, $tableMonth, $idSite, $period, $date1, $date2, $name, $value)
{
$table = Common::prefixTable($type.'_'.$tableMonth);
$idArchive = (int)Db::fetchOne("SELECT MAX(idarchive) FROM $table") + 1;
$sql = "INSERT INTO $table (idarchive, name, idsite, date1, date2, period, ts_archived, value)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
$bind = array($idArchive, $name, $idSite, $date1, $date2, Piwik::$idPeriods[$period], date('Y-m-d'), $value);
Db::query($sql, $bind);
}
}
0% Chargement en cours ou .
You are about to add 0 people to the discussion. Proceed with caution.
Veuillez vous inscrire ou vous pour commenter