Skip to content

Commit 164473b

Browse files
BradyShoberBradyShober
authored
Call TerminateInstanceInAutoScalingGroup to terminate if using an ASG (#462)
* Call TerminateInstanceInAutoScalingGroup to terminate if using an ASG * Fixed the tests * Add permission to README * Remove deprecated code
1 parent e14119f commit 164473b

9 files changed

Lines changed: 119 additions & 15 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ Add an inline policy to the IAM user or EC2 instance role to allow it to use EC2
129129
"Effect": "Allow",
130130
"Action": [
131131
"autoscaling:DescribeAutoScalingGroups",
132+
"autoscaling:TerminateInstanceInAutoScalingGroup",
132133
"autoscaling:UpdateAutoScalingGroup"
133134
],
134135
"Resource": "*"

‎src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.amazon.jenkins.ec2fleet.aws.AwsPermissionChecker;
44
import com.amazon.jenkins.ec2fleet.aws.RegionHelper;
5+
import com.amazon.jenkins.ec2fleet.fleet.AutoScalingGroupFleet;
56
import com.amazon.jenkins.ec2fleet.fleet.EC2Fleet;
67
import com.amazon.jenkins.ec2fleet.fleet.EC2Fleets;
78
import com.amazonaws.services.ec2.AmazonEC2;
@@ -29,6 +30,7 @@
2930

3031
import javax.annotation.Nonnull;
3132
import java.io.IOException;
33+
import java.lang.reflect.Field;
3234
import java.util.ArrayList;
3335
import java.util.Collection;
3436
import java.util.Collections;
@@ -612,7 +614,14 @@ public void run() {
612614
}
613615
}
614616
});
615-
Registry.getEc2Api().terminateInstances(ec2, currentInstanceIdsToTerminate.keySet());
617+
if(EC2Fleets.get(fleet).isAutoScalingGroup()){
618+
fine("Terminating instances in AutoScalingGroup: %s", currentInstanceIdsToTerminate.keySet());
619+
((AutoScalingGroupFleet) EC2Fleets.get(fleet)).terminateInstances(awsCredentialsId, region, endpoint, currentInstanceIdsToTerminate.keySet());
620+
}
621+
else {
622+
fine("Terminating instances: %s", currentInstanceIdsToTerminate.keySet());
623+
Registry.getEc2Api().terminateInstances(ec2, currentInstanceIdsToTerminate.keySet());
624+
}
616625
info("Terminated instances: %s", currentInstanceIdsToTerminate);
617626
}
618627

‎src/main/java/com/amazon/jenkins/ec2fleet/fleet/AutoScalingGroupFleet.java‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import com.amazonaws.services.autoscaling.model.LaunchTemplateOverrides;
1515
import com.amazonaws.services.autoscaling.model.MixedInstancesPolicy;
1616
import com.amazonaws.services.autoscaling.model.UpdateAutoScalingGroupRequest;
17+
import com.amazonaws.services.autoscaling.model.TerminateInstanceInAutoScalingGroupRequest;
1718
import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsHelper;
1819
import com.cloudbees.jenkins.plugins.awscredentials.AmazonWebServicesCredentials;
1920
import hudson.util.ListBoxModel;
@@ -23,17 +24,15 @@
2324

2425
import javax.annotation.Nullable;
2526
import javax.annotation.concurrent.ThreadSafe;
26-
import java.util.Collection;
27-
import java.util.Collections;
28-
import java.util.HashSet;
29-
import java.util.Map;
30-
import java.util.Optional;
31-
import java.util.Set;
27+
import java.util.*;
28+
import java.util.logging.Logger;
3229
import java.util.stream.Collectors;
3330

3431
@ThreadSafe
3532
public class AutoScalingGroupFleet implements EC2Fleet {
3633

34+
private static final Logger LOGGER = Logger.getLogger(AutoScalingGroupFleet.class.getName());
35+
3736
@Override
3837
public void describe(
3938
final String awsCredentialsId, final String regionName, final String endpoint,
@@ -112,6 +111,11 @@ public Map<String, FleetStateStats> getStateBatch(String awsCredentialsId, Strin
112111
throw new UnsupportedOperationException();
113112
}
114113

114+
@Override
115+
public Boolean isAutoScalingGroup() {
116+
return true;
117+
}
118+
115119
// TODO: move to Registry
116120
public AmazonAutoScalingClient createClient(
117121
final String awsCredentialsId, final String regionName, final String endpoint) {
@@ -126,6 +130,24 @@ public AmazonAutoScalingClient createClient(
126130
return client;
127131
}
128132

133+
public void terminateInstances(final String awsCredentialsId, final String regionName, final String endpoint, final Collection<String> instanceIds) {
134+
final AmazonAutoScalingClient client = createClient(awsCredentialsId, regionName, endpoint);
135+
136+
for(String instanceId : instanceIds) {
137+
if (StringUtils.isBlank(instanceId)) {
138+
throw new IllegalArgumentException("Instance ID cannot be null or empty");
139+
}
140+
try{
141+
// Attempt to terminate the instance in the Auto Scaling group first
142+
client.terminateInstanceInAutoScalingGroup(new TerminateInstanceInAutoScalingGroupRequest()
143+
.withInstanceId(instanceId)
144+
.withShouldDecrementDesiredCapacity(false));
145+
} catch (Exception e) {
146+
LOGGER.warning(String.format("Failed to terminate instance %s in Auto Scaling group: %s", instanceId, e.getMessage()));
147+
}
148+
}
149+
}
150+
129151
// TODO: merge with EC2Api#getEndpoint
130152
@Nullable
131153
private String getEndpoint(@Nullable final String regionName, @Nullable final String endpoint) {

‎src/main/java/com/amazon/jenkins/ec2fleet/fleet/EC2EC2Fleet.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,4 +160,9 @@ public Map<String, FleetStateStats> getStateBatch(String awsCredentialsId, Strin
160160
}
161161
return r;
162162
}
163+
164+
@Override
165+
public Boolean isAutoScalingGroup() {
166+
return false;
167+
}
163168
}

‎src/main/java/com/amazon/jenkins/ec2fleet/fleet/EC2Fleet.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,6 @@ Map<String, FleetStateStats> getStateBatch(
3232
final String awsCredentialsId, final String regionName, final String endpoint,
3333
final Collection<String> ids);
3434

35+
Boolean isAutoScalingGroup();
36+
3537
}

‎src/main/java/com/amazon/jenkins/ec2fleet/fleet/EC2SpotFleet.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,4 +204,9 @@ public Map<String, FleetStateStats> getStateBatch(
204204
return r;
205205
}
206206

207+
@Override
208+
public Boolean isAutoScalingGroup() {
209+
return false;
210+
}
211+
207212
}

‎src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudTest.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.mockito.stubbing.Answer;
4343

4444
import java.io.IOException;
45+
import java.lang.reflect.Field;
4546
import java.util.ArrayList;
4647
import java.util.Arrays;
4748
import java.util.Collection;
@@ -2080,6 +2081,69 @@ public void update_whenScalingByNodeHardwareWithNoVCPUsAndNoMemory_shouldSetExec
20802081
assertEquals(fleetCloud.getNumExecutors(), actualFleetNode.getNumExecutors());
20812082
}
20822083

2084+
@Test
2085+
public void update_shouldTerminateInstancesInAutoScalingGroup() throws IOException, IllegalAccessException, NoSuchFieldException {
2086+
// Arrange
2087+
final AutoScalingGroupFleet autoScalingGroupFleet = mock(AutoScalingGroupFleet.class);
2088+
when(EC2Fleets.get(anyString())).thenReturn(autoScalingGroupFleet);
2089+
when(autoScalingGroupFleet.isAutoScalingGroup()).thenReturn(true);
2090+
2091+
final FleetStateStats stats = new FleetStateStats("fleetId", 1, FleetStateStats.State.active(),
2092+
Collections.singleton("i-0"), Collections.<String, Double>emptyMap());
2093+
when(autoScalingGroupFleet.getState(anyString(), any(), any(), anyString())).thenReturn(stats);
2094+
2095+
EC2FleetCloud fleetCloud = new EC2FleetCloud("TestCloud", "credId", null, "region",
2096+
null, "fleetId", null, null, mock(ComputerConnector.class), false, false,
2097+
0, 0, 10, 0, 1, false, false, null, false, null, null, null, false, false, null);
2098+
2099+
// Set up instanceIdsToTerminate
2100+
HashMap<String, EC2AgentTerminationReason> toTerminate = new HashMap<>();
2101+
toTerminate.put("i-0", EC2AgentTerminationReason.MAX_TOTAL_USES_EXHAUSTED);
2102+
fleetCloud.setStats(stats);
2103+
Field field = EC2FleetCloud.class.getDeclaredField("instanceIdsToTerminate");
2104+
field.setAccessible(true);
2105+
field.set(fleetCloud, toTerminate);
2106+
2107+
// Act
2108+
fleetCloud.update();
2109+
2110+
// Assert
2111+
verify(autoScalingGroupFleet).terminateInstances(anyString(), any(), any(), eq(Collections.singleton("i-0")));
2112+
}
2113+
2114+
@Test
2115+
public void update_shouldTerminateInstancesInEC2Api() throws IOException, NoSuchFieldException, IllegalAccessException {
2116+
// Arrange
2117+
final EC2Fleet ec2Fleet = mock(EC2Fleet.class);
2118+
when(EC2Fleets.get(anyString())).thenReturn(ec2Fleet);
2119+
when(ec2Fleet.isAutoScalingGroup()).thenReturn(false);
2120+
2121+
final AmazonEC2 amazonEC2 = mock(AmazonEC2.class);
2122+
when(Registry.getEc2Api().connect(anyString(), any(), any())).thenReturn(amazonEC2);
2123+
2124+
final FleetStateStats stats = new FleetStateStats("fleetId", 1, FleetStateStats.State.active(),
2125+
Collections.singleton("i-0"), Collections.<String, Double>emptyMap());
2126+
when(ec2Fleet.getState(anyString(), any(), any(), anyString())).thenReturn(stats);
2127+
2128+
EC2FleetCloud fleetCloud = new EC2FleetCloud("TestCloud", "credId", null, "region",
2129+
null, "fleetId", null, null, mock(ComputerConnector.class), false, false,
2130+
0, 0, 10, 0, 1, false, false, null, false, null, null, null, false, false, null);
2131+
2132+
// Set up instanceIdsToTerminate
2133+
HashMap<String, EC2AgentTerminationReason> toTerminate = new HashMap<>();
2134+
toTerminate.put("i-0", EC2AgentTerminationReason.MAX_TOTAL_USES_EXHAUSTED);
2135+
fleetCloud.setStats(stats);
2136+
Field field = EC2FleetCloud.class.getDeclaredField("instanceIdsToTerminate");
2137+
field.setAccessible(true);
2138+
field.set(fleetCloud, toTerminate);
2139+
2140+
// Act
2141+
fleetCloud.update();
2142+
2143+
// Assert
2144+
verify(Registry.getEc2Api()).terminateInstances(eq(amazonEC2), eq(Collections.singleton("i-0")));
2145+
}
2146+
20832147
@Test
20842148
public void removeScheduledFutures_success() {
20852149
// given

‎src/test/java/com/amazon/jenkins/ec2fleet/EC2RetentionStrategyIntegrationTest.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,7 @@
3131
import static org.mockito.ArgumentMatchers.any;
3232
import static org.mockito.ArgumentMatchers.anyString;
3333
import static org.mockito.ArgumentMatchers.nullable;
34-
import static org.mockito.Mockito.mock;
35-
import static org.mockito.Mockito.spy;
36-
import static org.mockito.Mockito.times;
37-
import static org.mockito.Mockito.verify;
38-
import static org.mockito.Mockito.when;
34+
import static org.mockito.Mockito.*;
3935

4036
public class EC2RetentionStrategyIntegrationTest extends IntegrationTest {
4137

@@ -269,7 +265,7 @@ public void shouldTerminateWhenMaxTotalUsesIsExhausted() throws Exception {
269265
cloud.update();
270266

271267
final ArgumentCaptor<TerminateInstancesRequest> argument = ArgumentCaptor.forClass(TerminateInstancesRequest.class);
272-
verify((amazonEC2)).terminateInstances(argument.capture());
268+
verify((amazonEC2), atLeastOnce()).terminateInstances(argument.capture());
273269
assertTrue(argument.getAllValues().get(0).getInstanceIds().containsAll(Arrays.asList("i-1", "i-2")));
274270
}
275271
}

‎src/test/java/com/amazon/jenkins/ec2fleet/RealTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public void run() {
152152
final DescribeAutoScalingGroupsResult r = autoScalingClient.describeAutoScalingGroups(
153153
new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(autoScalingGroupName));
154154
Assert.assertEquals(1, r.getAutoScalingGroups().size());
155-
Assert.assertEquals(new Integer(0), r.getAutoScalingGroups().get(0).getDesiredCapacity());
155+
Assert.assertEquals(Integer.valueOf(0), r.getAutoScalingGroups().get(0).getDesiredCapacity());
156156
}
157157
}, TimeUnit.MINUTES.toMillis(3));
158158
}
@@ -205,7 +205,7 @@ public void run() {
205205
.withSpotFleetRequestIds(requestSpotFleetResult.getSpotFleetRequestId()))
206206
.getSpotFleetRequestConfigs();
207207
Assert.assertEquals(1, r.size());
208-
Assert.assertEquals(new Integer(0), r.get(0).getSpotFleetRequestConfig().getTargetCapacity());
208+
Assert.assertEquals(Integer.valueOf(0), r.get(0).getSpotFleetRequestConfig().getTargetCapacity());
209209
}
210210
}, TimeUnit.MINUTES.toMillis(3));
211211
}

0 commit comments

Comments
 (0)