diff --git a/plugins/TestRunner/Aws/CloudWatch.php b/plugins/TestRunner/Aws/CloudWatch.php index 0748ac129f34ba984712f1f05f1fadb0cecc692b..0718157d693a73f54a5507fc4c2d5a34697cef24 100644 --- a/plugins/TestRunner/Aws/CloudWatch.php +++ b/plugins/TestRunner/Aws/CloudWatch.php @@ -21,16 +21,33 @@ class CloudWatch */ private $config; + /** + * @var CloudWatchClient + */ + private $client; + public function __construct(Config $awsConfig) { $this->config = $awsConfig; + $this->client = $this->getCloudWatchClient(); } - public function terminateInstanceIfIdleForTooLong($instanceIds) + public function hasAssignedAlarms($instanceIds) { - $client = $this->getCloudWatchClient(); + $result = $this->client->describeAlarmsForMetric(array( + 'MetricName' => 'CPUUtilization', + 'Namespace' => $this->getNamespace(), + 'Dimensions' => $this->getDimensions($instanceIds) + )); - $client->putMetricAlarm(array( + $metricAlarms = $result->getPath('MetricAlarms'); + + return !empty($metricAlarms); + } + + public function terminateInstanceIfIdleForTooLong($instanceIds) + { + $this->client->putMetricAlarm(array( 'AlarmName' => 'TerminateInstanceBecauseIdle', 'AlarmDescription' => 'Terminate instances if CPU is on average < 10% for 5 minutes in a row 8 times consecutively', 'ActionsEnabled' => true, @@ -48,7 +65,7 @@ class CloudWatch 'ComparisonOperator' => ComparisonOperator::LESS_THAN_THRESHOLD, )); - $client->putMetricAlarm(array( + $this->client->putMetricAlarm(array( 'AlarmName' => 'TerminateInstanceIfStatusCheckFails', 'AlarmDescription' => 'Terminate instances in case two status check fail within one minute', 'ActionsEnabled' => true, diff --git a/plugins/TestRunner/Aws/Instance.php b/plugins/TestRunner/Aws/Instance.php index 622f7cf9bfdf5e1c941a59d987411e06d41af51a..769b35b3379da16d611cbad01b12786263a3b4bc 100644 --- a/plugins/TestRunner/Aws/Instance.php +++ b/plugins/TestRunner/Aws/Instance.php @@ -63,6 +63,9 @@ class Instance if (!empty($reservations)) { $host = $this->getHostFromDescribedInstances($instances); + $instanceIds = $instances->getPath('Reservations/*/Instances/*/InstanceId'); + $this->verifySetup($instanceIds); + return $host; } } @@ -96,15 +99,15 @@ class Instance $instanceIds = $result->getPath('Instances/*/InstanceId'); + $this->client->waitUntilInstanceRunning(array( + 'InstanceIds' => $instanceIds, + )); + return $instanceIds; } public function setup($instanceIds) { - $this->client->waitUntilInstanceRunning(array( - 'InstanceIds' => $instanceIds, - )); - $awsCloudWatch = new CloudWatch($this->config); $awsCloudWatch->terminateInstanceIfIdleForTooLong($instanceIds); @@ -120,6 +123,17 @@ class Instance return $host; } + public function verifySetup($instanceIds) + { + $awsCloudWatch = new CloudWatch($this->config); + $hasAlarms = $awsCloudWatch->hasAssignedAlarms($instanceIds); + + if (!$hasAlarms) { + return $this->setup($instanceIds); + } + + } + /** * @param \Guzzle\Service\Resource\Model $resources * @return mixed