Adding force-idsegments parameter to core:archive param and handle the...

Adding force-idsegments parameter to core:archive param and handle the parameter in CronArchive.php class. New parameter limits list of segments that should be archived during archiving, all other segments are skipped.

core/CronArchive.php

@@ -21,6 +21,7 @@ use Piwik\Period\Factory as PeriodFactory;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\CronArchive\SegmentArchivingRequestUrlProvider;
 use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
+use Piwik\Plugins\SegmentEditor\Model as SegmentEditorModel;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
 use Piwik\Plugins\UsersManager\UserPreferences;
@@ -189,6 +190,14 @@ class CronArchive
      */
     public $concurrentRequestsPerWebsite = false;
 
+    /**
+     * List of segment strings to force archiving for. If a stored segment is not in this list, it will not
+     * be archived.
+     *
+     * @var string[]
+     */
+    public $segmentsToForce = array();
+
     private $websitesWithVisitsSinceLastRun = 0;
     private $skippedPeriodsArchivesWebsite = 0;
     private $skippedDayArchivesWebsites = 0;
@@ -278,6 +287,8 @@ class CronArchive
             $this->logger->info('Will ignore websites and help finish a previous started queue instead. IDs: ' . implode(', ', $this->websites->getInitialSiteIds()));
         }
 
+        $this->logForcedSegmentInfo();
+
         /**
          * This event is triggered after a CronArchive instance is initialized.
          *
@@ -394,6 +405,26 @@ class CronArchive
         throw new Exception($m);
     }
 
+    /**
+     * @param int[] $idSegments
+     */
+    public function setSegmentsToForceFromSegmentIds($idSegments)
+    {
+        /** @var SegmentEditorModel $segmentEditorModel */
+        $segmentEditorModel = StaticContainer::get('Piwik\Plugins\SegmentEditor\Model');
+        $segments = $segmentEditorModel->getAllSegmentsAndIgnoreVisibility();
+
+        $segments = array_filter($segments, function ($segment) use ($idSegments) {
+            return in_array($segment['idsegment'], $idSegments);
+        });
+
+        $segments = array_map(function ($segment) {
+            return $segment['definition'];
+        }, $segments);
+
+        $this->segmentsToForce = $segments;
+    }
+
     public function runScheduledTasks()
     {
         $this->logSection("SCHEDULED TASKS");
@@ -1413,10 +1444,21 @@ class CronArchive
     private function getUrlsWithSegment($idSite, $period, $date)
     {
         $urlsWithSegment = array();
-        $segments = $this->getSegmentsForSite($idSite, $period);
+        $segmentsForSite = $this->getSegmentsForSite($idSite, $period);
+
+        $segments = array();
+        foreach ($segmentsForSite as $segment) {
+            if ($this->shouldSkipSegmentArchiving($segment)) {
+                $this->logger->info("- skipping segment archiving for '{segment}'.", array('segment' => $segment));
+
+                continue;
+            }
+
+            $segments[] = $segment;
+        }
+
         $segmentCount = count($segments);
         $processedSegmentCount = 0;
-
         foreach ($segments as $segment) {
             $dateParamForSegment = $this->segmentArchivingRequestUrlProvider->getUrlParameterDateString($idSite, $period, $date, $segment);
 
@@ -1469,4 +1511,21 @@ class CronArchive
         ));
         $this->logger->info('- pre-processing all visits');
     }
+
+    private function shouldSkipSegmentArchiving($segment)
+    {
+        return !empty($this->segmentsToForce) && !in_array($segment, $this->segmentsToForce);
+    }
+
+    private function logForcedSegmentInfo()
+    {
+        if (empty($this->segmentsToForce)) {
+            return;
+        }
+
+        $this->logger->info("- Limiting segment archiving to following segments:");
+        foreach ($this->segmentsToForce as $segmentDefinition) {
+            $this->logger->info("  * " . $segmentDefinition);
+        }
+    }
 }

core/SettingsPiwik.php

@@ -99,7 +99,7 @@ class SettingsPiwik
      * cron archiving.
      *
      * @param int $idSite The ID of the site to get stored segments for.
-     * @return string The list of stored segments that apply to the requested site.
+     * @return string[] The list of stored segments that apply to the requested site.
      */
     public static function getKnownSegmentsToArchiveForSite($idSite)
     {

plugins/CoreConsole/Commands/CoreArchiver.php

@@ -51,6 +51,11 @@ class CoreArchiver extends ConsoleCommand
         $archiver->dateLastForced = $input->getOption('force-date-last-n');
         $archiver->concurrentRequestsPerWebsite = $input->getOption('concurrent-requests-per-website');
 
+        $segmentIds = $input->getOption('force-idsegments');
+        $segmentIds = explode(',', $segmentIds);
+        $segmentIds = array_map('trim', $segmentIds);
+        $archiver->setSegmentsToForceFromSegmentIds($segmentIds);
+
         return $archiver;
     }
 
@@ -80,6 +85,7 @@ class CoreArchiver extends ConsoleCommand
         $command->addOption('force-periods', null, InputOption::VALUE_OPTIONAL, "If specified, archiving will be processed only for these Periods (comma separated eg. day,week,month)");
         $command->addOption('force-date-last-n', null, InputOption::VALUE_REQUIRED, "This script calls the API with period=lastN. You can force the N in lastN by specifying this value.");
         $command->addOption('force-date-range', null, InputOption::VALUE_OPTIONAL, "If specified, archiving will be processed only for periods included in this date range. Format: YYYY-MM-DD,YYYY-MM-DD");
+        $command->addOption('force-idsegments', null, InputOption::VALUE_REQUIRED, 'If specified, only these segments will be processed (if the segment should be applied to a site in the first place). Specify stored segment IDs, not the segments themselves, eg, 1,2,3');
         $command->addOption('concurrent-requests-per-website', null, InputOption::VALUE_OPTIONAL, "When processing a website and its segments, number of requests to process in parallel", CronArchive::MAX_CONCURRENT_API_REQUESTS);
         $command->addOption('disable-scheduled-tasks', null, InputOption::VALUE_NONE, "Skips executing Scheduled tasks (sending scheduled reports, db optimization, etc.).");
         $command->addOption('accept-invalid-ssl-certificate', null, InputOption::VALUE_NONE, "It is _NOT_ recommended to use this argument. Instead, you should use a valid SSL certificate!\nIt can be useful if you specified --url=https://... or if you are using Piwik with force_ssl=1");

tests/PHPUnit/Framework/Mock/FakeLogger.php

@@ -2,6 +2,7 @@
 
 namespace Piwik\Tests\Framework\Mock;
 
+use Monolog\Processor\PsrLogMessageProcessor;
 use Psr\Log\AbstractLogger;
 use Psr\Log\LoggerInterface;
 
@@ -12,8 +13,20 @@ class FakeLogger extends AbstractLogger implements LoggerInterface
      */
     public $output = '';
 
+    /**
+     * @var PsrLogMessageProcessor
+     */
+    private $processor;
+
+    public function __construct()
+    {
+        $this->processor = new PsrLogMessageProcessor();
+    }
+
     public function log($level, $message, array $context = array())
     {
-        $this->output .= $message . PHP_EOL;
+        $record = $this->processor->__invoke(array('message' => $message, 'context' => $context));
+
+        $this->output .= $record['message'] . PHP_EOL;
     }
 }

tests/PHPUnit/Integration/CronArchiveTest.php

@@ -48,16 +48,33 @@ class CronArchiveTest extends IntegrationTestCase
         $this->assertEquals($expectedInvalidations, $api->getInvalidatedReports());
     }
 
+    public function test_setSegmentsToForceFromSegmentIds_CorrectlyGetsSegmentDefinitions_FromSegmentIds()
+    {
+        Fixture::createWebsite('2014-12-12 00:01:02');
+        SegmentAPI::getInstance()->add('foo', 'actions>=1', 1, true, true);
+        SegmentAPI::getInstance()->add('barb', 'actions>=2', 1, true, true);
+        SegmentAPI::getInstance()->add('burb', 'actions>=3', 1, true, true);
+        SegmentAPI::getInstance()->add('sub', 'actions>=4', 1, true, true);
+
+        $cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php');
+        $cronarchive->setSegmentsToForceFromSegmentIds(array(2, 4));
+
+        $expectedSegments = array('actions>=2', 'actions>=4');
+        $this->assertEquals($expectedSegments, array_values($cronarchive->segmentsToForce));
+    }
+
     public function test_output()
     {
         Fixture::createWebsite('2014-12-12 00:01:02');
         SegmentAPI::getInstance()->add('foo', 'actions>=2', 1, true, true);
+        SegmentAPI::getInstance()->add('burr', 'actions>=4', 1, true, true);
 
         $logger = new FakeLogger();
 
         $archiver = new CronArchive(null, $logger);
         $archiver->shouldArchiveAllSites = true;
         $archiver->shouldArchiveAllPeriodsSince = true;
+        $archiver->segmentsToForce = array('actions>=2;browserCode=FF', 'actions>=2');
         $archiver->init();
         $archiver->run();
 
@@ -72,23 +89,30 @@ NOTES
 - Reports for today will be processed at most every %s seconds. You can change this value in Piwik UI > Settings > General Settings.
 - Reports for the current week/month/year will be refreshed at most every %s seconds.
 - Will process all 1 websites
+- Limiting segment archiving to following segments:
+  * actions>=2;browserCode=FF
+  * actions>=2
 ---------------------------
 START
 Starting Piwik reports archiving...
 Will pre-process for website id = 1, day period
 - pre-processing all visits
+- skipping segment archiving for 'actions>=4'.
 - pre-processing segment 1/1 actions>=2
 Archived website id = 1, period = day, 1 segments, 0 visits in last %s days, 0 visits today, Time elapsed: %s
 Will pre-process for website id = 1, week period
 - pre-processing all visits
+- skipping segment archiving for 'actions>=4'.
 - pre-processing segment 1/1 actions>=2
 Archived website id = 1, period = week, 1 segments, 0 visits in last %s weeks, 0 visits this week, Time elapsed: %s
 Will pre-process for website id = 1, month period
 - pre-processing all visits
+- skipping segment archiving for 'actions>=4'.
 - pre-processing segment 1/1 actions>=2
 Archived website id = 1, period = month, 1 segments, 0 visits in last %s months, 0 visits this month, Time elapsed: %s
 Will pre-process for website id = 1, year period
 - pre-processing all visits
+- skipping segment archiving for 'actions>=4'.
 - pre-processing segment 1/1 actions>=2
 Archived website id = 1, period = year, 1 segments, 0 visits in last %s years, 0 visits this year, Time elapsed: %s
 Archived website id = 1, %s API requests, Time elapsed: %s [1/1 done]