Merge pull request #8633 from piwik/force_optimize_cmd

Add command to optimize archive tables (even if on InnoDB) since archive tables can get bloated even when purging functions normally.

core/DataAccess/ArchiveTableCreator.php

@@ -33,7 +33,7 @@ class ArchiveTableCreator
     protected static function getTable(Date $date, $type)
     {
         $tableNamePrefix = "archive_" . $type;
-        $tableName = $tableNamePrefix . "_" . $date->toString('Y_m');
+        $tableName = $tableNamePrefix . "_" . self::getTableMonthFromDate($date);
         $tableName = Common::prefixTable($tableName);
 
         self::createArchiveTablesIfAbsent($tableName, $tableNamePrefix);
@@ -100,6 +100,11 @@ class ArchiveTableCreator
         return $date;
     }
 
+    public static function getTableMonthFromDate(Date $date)
+    {
+        return $date->toString('Y_m');
+    }
+
     public static function getTypeFromTableName($tableName)
     {
         if (strpos($tableName, 'archive_numeric_') !== false) {

plugins/CoreAdminHome/Commands/OptimizeArchiveTables.php

+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\CoreAdminHome\Commands;
+
+use Piwik\Common;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Plugin\ConsoleCommand;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Administration command that optimizes archive tables (even if they use InnoDB).
+ */
+class OptimizeArchiveTables extends ConsoleCommand
+{
+    const ALL_TABLES_STRING = 'all';
+    const CURRENT_MONTH_STRING = 'now';
+
+    protected function configure()
+    {
+        $this->setName('database:optimize-archive-tables');
+        $this->setDescription("Runs an OPTIMIZE TABLE query on the specified archive tables.");
+        $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::REQUIRED,
+            "The months of the archive tables to optimize. Use '" . self::ALL_TABLES_STRING. "' for all dates or '" .
+            self::CURRENT_MONTH_STRING . "' to optimize the current month only.");
+        $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'For testing purposes.');
+        $this->setHelp("This command can be used to ease or automate maintenance. Instead of manually running "
+            . "OPTIMIZE TABLE queries, the command can be used.\n\nYou should run the command if you find your "
+            . "archive tables grow and do not shrink after purging. Optimizing them will reclaim some space.");
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $dryRun = $input->getOption('dry-run');
+
+        $tableMonths = $this->getTableMonthsToOptimize($input);
+
+        foreach ($tableMonths as $month) {
+            $this->optimizeTable($output, $dryRun, 'archive_numeric_' . $month);
+            $this->optimizeTable($output, $dryRun, 'archive_blob_' . $month);
+        }
+    }
+
+    private function optimizeTable(OutputInterface $output, $dryRun, $table)
+    {
+        $output->write("Optimizing table '$table'...");
+
+        if ($dryRun) {
+            $output->write("[dry-run, not optimising table]");
+        } else {
+            Db::optimizeTables(Common::prefixTable($table), $force = true);
+        }
+
+        $output->writeln("Done.");
+    }
+
+    private function getTableMonthsToOptimize(InputInterface $input)
+    {
+        $dateSpecifier = $input->getArgument('dates');
+        if (count($dateSpecifier) === 1) {
+            $dateSpecifier = reset($dateSpecifier);
+
+            if ($dateSpecifier == self::ALL_TABLES_STRING) {
+                return $this->getAllArchiveTableMonths();
+            } else if ($dateSpecifier == self::CURRENT_MONTH_STRING) {
+                $now = Date::factory('now');
+                return array(ArchiveTableCreator::getTableMonthFromDate($now));
+            } else if (strpos($dateSpecifier, 'last') === 0) {
+                $lastN = substr($dateSpecifier, 4);
+                if (!ctype_digit($lastN)) {
+                    throw new \Exception("Invalid lastN specifier '$lastN'. The end must be an integer, eg, last1 or last2.");
+                }
+
+                if ($lastN <= 0) {
+                    throw new \Exception("Invalid lastN value '$lastN'.");
+                }
+
+                return $this->getLastNTableMonths((int)$lastN);
+            }
+        }
+
+        $tableMonths = array();
+        foreach ($dateSpecifier as $date) {
+            $date = Date::factory($date);
+            $tableMonths[] = ArchiveTableCreator::getTableMonthFromDate($date);
+        }
+        return $tableMonths;
+    }
+
+    private function getAllArchiveTableMonths()
+    {
+        $tableMonths = array();
+        foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
+            $tableMonths[] = ArchiveTableCreator::getDateFromTableName($table);
+        }
+        return $tableMonths;
+    }
+
+    /**
+     * @param int $lastN
+     * @return string[]
+     */
+    private function getLastNTableMonths($lastN)
+    {
+        $now = Date::factory('now');
+
+        $result = array();
+        for ($i = 0; $i < $lastN; ++$i) {
+            $date = $now->subMonth($i + 1);
+            $result[] = ArchiveTableCreator::getTableMonthFromDate($date);
+        }
+        return $result;
+    }
+}

plugins/CoreAdminHome/tests/Integration/Commands/OptimizeArchiveTablesTest.php

+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\CoreAdminHome\tests\Integration\Commands;
+
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\Date;
+use Piwik\Tests\Framework\TestCase\ConsoleCommandTestCase;
+
+
+/**
+ * @group Core
+ */
+class OptimizeArchiveTablesTest extends ConsoleCommandTestCase
+{
+    public static function setUpBeforeClass()
+    {
+        parent::setUpBeforeClass();
+
+        ArchiveTableCreator::getNumericTable(Date::factory('2015-01-01'));
+        ArchiveTableCreator::getNumericTable(Date::factory('2015-02-02'));
+        ArchiveTableCreator::getNumericTable(Date::factory('2015-03-03'));
+    }
+
+    /**
+     * @dataProvider getDatesToTest
+     */
+    public function test_Command_OptimizesCorrectTables($dates, $expectedOptimizedTableDates)
+    {
+        $code = $this->applicationTester->run(array(
+            'command' => 'database:optimize-archive-tables',
+            'dates' => $dates,
+            '--dry-run' => true,
+        ));
+
+        $this->assertEquals(0, $code, $this->getCommandDisplayOutputErrorMessage());
+
+        $output = $this->applicationTester->getDisplay();
+
+        preg_match_all('/Optimizing table \'([^\']+)\'/', $output, $matches);
+        $tablesOptimized = $matches[1];
+
+        $expectedOptimizedTables = array();
+        foreach ($expectedOptimizedTableDates as $date) {
+            $expectedOptimizedTables[] = 'archive_numeric_' . $date;
+            $expectedOptimizedTables[] = 'archive_blob_' . $date;
+        }
+
+        $this->assertEquals($expectedOptimizedTables, $tablesOptimized);
+    }
+
+    public function getDatesToTest()
+    {
+        return array(
+            array(
+                array('all'),
+                array('2015_01', '2015_02', '2015_03'),
+            ),
+
+            array(
+                array('now'),
+                array(date('Y_m')),
+            ),
+
+            array(
+                array('2015-01-01', '2015-02-03', '2014-01-01', '2013-05-12', '2015-05-05'),
+                array('2015_01', '2015_02', '2014_01', '2013_05', '2015_05'),
+            ),
+
+            array(
+                array('last1'),
+                array(Date::factory('now')->subMonth(1)->toString('Y_m')),
+            ),
+
+            array(
+                array('last5'),
+                array(
+                    Date::factory('now')->subMonth(1)->toString('Y_m'),
+                    Date::factory('now')->subMonth(2)->toString('Y_m'),
+                    Date::factory('now')->subMonth(3)->toString('Y_m'),
+                    Date::factory('now')->subMonth(4)->toString('Y_m'),
+                    Date::factory('now')->subMonth(5)->toString('Y_m'),
+                ),
+            ),
+        );
+    }
+}
\ No newline at end of file