refs #6750 - only rename archives in update script. do not rebuild archives...

refs #6750 - only rename archives in update script. do not rebuild archives without versions where they do not exist. Instead added a fallback in the api methods to build the reports on the fly if requested

core/Updates/2.10.0-b1.php

@@ -31,8 +31,7 @@ use Piwik\Plugins\Dashboard\Model AS DashboardModel;
  * For big archives (month/year) that ment that some of the data was truncated, due to the datatable entry limit.
  * To avoid that data loss / inaccuracy in the future, DevicesDetection plugin will also store archives without the version.
  * For data archived after DevicesDetection plugin was enabled, those archive already exist. As we are removing the
- * UserSettings reports, we need to move the existing old data to the new archives, which means we need to build up
- * those archives, where they do not exist.
+ * UserSettings reports, there is a fallback in DevicesDetection API to build the report out of the datatable with versions.
  *
  * NOTE: Some archives might not contain "all" data.
  * That might have happened directly after the day DevicesDetection plugin was enabled. For the days before, there were
@@ -106,12 +105,12 @@ class Updates_2_10_0_b1 extends Updates
         Updater::updateDatabase(__FILE__, self::getSql());
 
         // DeviceDetection upgrade in beta1 timed out on demo #6750
-//        $archiveBlobTables = self::getAllArchiveBlobTables();
-//
-//        foreach ($archiveBlobTables as $table) {
-//            self::updateBrowserArchives($table);
-//            self::updateOsArchives($table);
-//        }
+        $archiveBlobTables = self::getAllArchiveBlobTables();
+
+        foreach ($archiveBlobTables as $table) {
+            self::updateBrowserArchives($table);
+            self::updateOsArchives($table);
+        }
     }
 
     /**
@@ -193,12 +192,6 @@ class Updates_2_10_0_b1 extends Updates
                 Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_browserVersions', $blob['idarchive'], 'UserSettings_browser'));
             }
         }
-
-        // rebuild archives without versions
-        $browserBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_browserVersions'", $table));
-        foreach ($browserBlobs as $blob) {
-            self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_browsers', $table);
-        }
     }
 
     public static function updateOsArchives($table) {
@@ -218,40 +211,5 @@ class Updates_2_10_0_b1 extends Updates
                 Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_osVersions', $blob['idarchive'], 'UserSettings_os'));
             }
         }
-
-        // rebuild archives without versions
-        $osBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_osVersions'", $table));
-        foreach ($osBlobs as $blob) {
-            self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_os', $table);
-        }
-    }
-
-    protected static function createArchiveBlobWithoutVersions($blob, $newName, $table)
-    {
-        $blob['value'] = @gzuncompress($blob['value']);
-
-        try {
-            $datatable = DataTable::fromSerializedArray($blob['value']);
-            $datatable->filter('GroupBy', array('label', function ($label) {
-                if (preg_match("/(.+) [0-9]+(?:\.[0-9]+)?$/", $label, $matches)) {
-                    return $matches[1]; // should match for browsers
-                }
-
-                if (strpos($label, ';')) {
-                    return substr($label, 0, 3); // should match for os
-                }
-
-                return $label;
-            }));
-
-            $newData = $datatable->getSerialized();
-
-            $blob['value'] = @gzcompress($newData[0]);
-            $blob['name'] = $newName;
-
-            Db::get()->query(sprintf('REPLACE INTO %s (`idarchive`, `name`, `idsite`, `date1`, `date2`, `period`, `ts_archived`, `value`) VALUES (?, ? , ?, ?, ?, ?, ?, ?)', $table), array_values($blob));
-        } catch (\Exception $e) {
-            // fail silently and simply skip the current record
-        }
     }
 }

plugins/DevicesDetection/API.php

@@ -123,11 +123,60 @@ class API extends \Piwik\Plugin\API
     public function getOsFamilies($idSite, $period, $date, $segment = false)
     {
         $dataTable = $this->getDataTable('DevicesDetection_os', $idSite, $period, $date, $segment);
+
+        // handle legacy archives
+        $this->checkForFallbackToOsVersions($dataTable, $idSite, $period, $date, $segment);
+
         $dataTable->filter('GroupBy', array('label', __NAMESPACE__ . '\getOSFamilyFullName'));
         $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'logo', __NAMESPACE__ . '\getOsFamilyLogo'));
         return $dataTable;
     }
 
+
+    /**
+     * That methods handles teh fallback to os version datatables to calculate those without versions.
+     *
+     * Unlike DevicesDetection plugin now, the UserSettings plugin did not store archives holding the os and browser data without
+     * their version number. The "version-less" reports were always generated out of the "version-containing" archives .
+     * For big archives (month/year) that ment that some of the data was truncated, due to the datatable entry limit.
+     * To avoid that data loss / inaccuracy in the future, DevicesDetection plugin will also store archives without the version.
+     * For data archived before DevicesDetection plugin was enabled, those archives do not exist, so we try to calculate
+     * them here from the "version-containing" reports if possible.
+     *
+     * @param DataTable\DataTableInterface $dataTable
+     * @param $idSite
+     * @param $period
+     * @param $date
+     * @param $segment
+     */
+    protected function checkForFallbackToOsVersions(DataTable\DataTableInterface &$dataTable, $idSite, $period, $date, $segment)
+    {
+        if ($dataTable instanceof DataTable\Map) {
+            $dataTables = $dataTable->getDataTables();
+            foreach ($dataTables as $date => &$table) {
+                if (!$table->getRowsCount()) {
+                    $subTable = $this->getDataTable('DevicesDetection_osVersions', $idSite, $period, $date, $segment);
+                    $subTable->filter('GroupBy', array('label', function ($osWithVersion) {
+                        if (strpos($osWithVersion, ';')) {
+                            return substr($osWithVersion, 0, 3);
+                        }
+                        return $osWithVersion;
+                    }));
+                    $dataTable->addTable($subTable, $date);
+                }
+            }
+
+        } else if (!$dataTable->getRowsCount()) {
+            $dataTable = $this->getDataTable('DevicesDetection_osVersions', $idSite, $period, $date, $segment);
+            $dataTable->filter('GroupBy', array('label', function ($osWithVersion) {
+                if (strpos($osWithVersion, ';')) {
+                    return substr($osWithVersion, 0, 3);
+                }
+                return $osWithVersion;
+            }));
+        }
+    }
+
     /**
      * Gets datatable displaying number of visits by OS version (eg. Android 4.0, Windows 7)
      * @param int $idSite
@@ -171,11 +220,59 @@ class API extends \Piwik\Plugin\API
     public function getBrowsers($idSite, $period, $date, $segment = false)
     {
         $dataTable = $this->getDataTable('DevicesDetection_browsers', $idSite, $period, $date, $segment);
+
+        // handle legacy archives
+        $this->checkForFallbackToBrowserVersions($dataTable, $idSite, $period, $date, $segment);
+
         $dataTable->filter('GroupBy', array('label', __NAMESPACE__ . '\getBrowserName'));
         $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'logo', __NAMESPACE__ . '\getBrowserFamilyLogo'));
         return $dataTable;
     }
 
+    /**
+     * That methods handles teh fallback to browser version datatables to calculate those without versions.
+     *
+     * Unlike DevicesDetection plugin now, the UserSettings plugin did not store archives holding the os and browser data without
+     * their version number. The "version-less" reports were always generated out of the "version-containing" archives .
+     * For big archives (month/year) that ment that some of the data was truncated, due to the datatable entry limit.
+     * To avoid that data loss / inaccuracy in the future, DevicesDetection plugin will also store archives without the version.
+     * For data archived before DevicesDetection plugin was enabled, those archives do not exist, so we try to calculate
+     * them here from the "version-containing" reports if possible.
+     *
+     * @param DataTable\DataTableInterface $dataTable
+     * @param $idSite
+     * @param $period
+     * @param $date
+     * @param $segment
+     */
+    protected function checkForFallbackToBrowserVersions(DataTable\DataTableInterface &$dataTable, $idSite, $period, $date, $segment)
+    {
+        if ($dataTable instanceof DataTable\Map) {
+            $dataTables = $dataTable->getDataTables();
+            foreach ($dataTables as $date => &$table) {
+                if (!$table->getRowsCount()) {
+                    $subTable = $this->getDataTable('DevicesDetection_browserVersions', $idSite, $period, $date, $segment);
+                    $subTable->filter('GroupBy', array('label', function ($browserWithVersion) {
+                        if (preg_match("/(.+) [0-9]+(?:\.[0-9]+)?$/", $browserWithVersion, $matches) === 0) {
+                            return $browserWithVersion;
+                        }
+                        return $matches[1];
+                    }));
+                    $dataTable->addTable($subTable, $date);
+                }
+            }
+
+        } else if (!$dataTable->getRowsCount()) {
+            $dataTable = $this->getDataTable('DevicesDetection_browserVersions', $idSite, $period, $date, $segment);
+            $dataTable->filter('GroupBy', array('label', function ($browserWithVersion) {
+                if (preg_match("/(.+) [0-9]+(?:\.[0-9]+)?$/", $browserWithVersion, $matches) === 0) {
+                    return $browserWithVersion;
+                }
+                return $matches[1];
+            }));
+        }
+    }
+
     /**
      * Gets datatable displaying number of visits by Browser version (eg. Firefox 20, Safari 6.0)
      * @param int $idSite