diff --git a/build.gradle b/build.gradle index 17e76704..15fb443c 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,9 @@ dependencies { compile 'com.google.guava:guava:11.+' compile 'org.apache.httpcomponents:httpclient:4.2.1' compile 'org.apache.httpcomponents:httpcore:4.2.1' + compile 'org.jclouds:jclouds-all:1.6.0' + compile 'org.jclouds.driver:jclouds-jsch:1.6.0' + compile 'org.jclouds.driver:jclouds-slf4j:1.6.0' testCompile 'org.testng:testng:6.3.1' testCompile 'org.mockito:mockito-core:1.8.5' diff --git a/src/main/java/com/netflix/simianarmy/CloudClient.java b/src/main/java/com/netflix/simianarmy/CloudClient.java index 0dc91efa..03e1d2b6 100644 --- a/src/main/java/com/netflix/simianarmy/CloudClient.java +++ b/src/main/java/com/netflix/simianarmy/CloudClient.java @@ -17,8 +17,12 @@ */ package com.netflix.simianarmy; +import java.util.List; import java.util.Map; +import org.jclouds.compute.ComputeService; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.ssh.SshClient; /** * The CloudClient interface. This abstractions provides the interface that the monkeys need to interact with @@ -88,4 +92,108 @@ public interface CloudClient { */ void createTagsForResources(Map keyValueMap, String... resourceIds); + /** + * Lists all EBS volumes attached to the specified instance. + * + * @param instanceId + * the instance id + * @param includeRoot + * if the root volume is on EBS, should we include it? + * + * @throws NotFoundException + * if the instance no longer exists or was already terminated after the crawler discovered it then you + * should get a NotFoundException + */ + List listAttachedVolumes(String instanceId, boolean includeRoot); + + /** + * Detaches an EBS volumes from the specified instance. + * + * @param instanceId + * the instance id + * @param volumeId + * the volume id + * @param force + * if we should force-detach the volume. Probably best not to use on high-value volumes. + * + * @throws NotFoundException + * if the instance no longer exists or was already terminated after the crawler discovered it then you + * should get a NotFoundException + */ + void detachVolume(String instanceId, String volumeId, boolean force); + + /** + * Returns the jClouds compute service. + */ + ComputeService getJcloudsComputeService(); + + /** + * Returns the jClouds node id for an instance id on this CloudClient. + */ + String getJcloudsId(String instanceId); + + /** + * Opens an SSH connection to an instance. + * + * @param instanceId + * instance id to connect to + * @param credentials + * SSH credentials to use + * @return {@link SshClient}, in connected state + */ + SshClient connectSsh(String instanceId, LoginCredentials credentials); + + /** + * Finds a security group with the given name, that can be applied to the given instance. + * + * For example, if it is a VPC instance, it makes sure that it is in the same VPC group. + * + * @param instanceId + * the instance that the group must be applied to + * @param groupName + * the name of the group to find + * + * @return The group id, or null if not found + */ + String findSecurityGroup(String instanceId, String groupName); + + /** + * Creates an (empty) security group, that can be applied to the given instance. + * + * @param instanceId + * instance that group should be applicable to + * @param groupName + * name for new group + * @param description + * description for new group + * + * @return the id of the security group + */ + String createSecurityGroup(String instanceId, String groupName, String description); + + /** + * Checks if we can change the security groups of an instance. + * + * @param instanceId + * instance to check + * + * @return true iff we can change security groups. + */ + boolean canChangeInstanceSecurityGroups(String instanceId); + + /** + * Sets the security groups for an instance. + * + * Note this is only valid for VPC instances. + * + * @param instanceId + * the instance id + * @param groupIds + * ids of desired new groups + * + * @throws NotFoundException + * if the instance no longer exists or was already terminated after the crawler discovered it then you + * should get a NotFoundException + */ + void setInstanceSecurityGroups(String instanceId, List groupIds); } diff --git a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java index 69d4de2f..abf68b48 100644 --- a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java +++ b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java @@ -257,6 +257,11 @@ public List findEvents(Enum monkeyType, Enum eventType, Map allChaosTypes; + /** * Instantiates a new basic chaos monkey. * @param ctx @@ -81,6 +104,23 @@ public BasicChaosMonkey(ChaosMonkey.Context ctx) { open.set(Calendar.HOUR, monkeyCalendar.openHour()); close.set(Calendar.HOUR, monkeyCalendar.closeHour()); + allChaosTypes = Lists.newArrayList(); + allChaosTypes.add(new ShutdownInstanceChaosType(cfg)); + allChaosTypes.add(new BlockAllNetworkTrafficChaosType(cfg)); + allChaosTypes.add(new DetachVolumesChaosType(cfg)); + allChaosTypes.add(new BurnCpuChaosType(cfg)); + allChaosTypes.add(new BurnIoChaosType(cfg)); + allChaosTypes.add(new KillProcessesChaosType(cfg)); + allChaosTypes.add(new NullRouteChaosType(cfg)); + allChaosTypes.add(new FailEc2ChaosType(cfg)); + allChaosTypes.add(new FailDnsChaosType(cfg)); + allChaosTypes.add(new FailDynamoDbChaosType(cfg)); + allChaosTypes.add(new FailS3ChaosType(cfg)); + allChaosTypes.add(new FillDiskChaosType(cfg)); + allChaosTypes.add(new NetworkCorruptionChaosType(cfg)); + allChaosTypes.add(new NetworkLatencyChaosType(cfg)); + allChaosTypes.add(new NetworkLossChaosType(cfg)); + TimeUnit freqUnit = ctx.scheduler().frequencyUnit(); long units = freqUnit.convert(close.getTimeInMillis() - open.getTimeInMillis(), TimeUnit.MILLISECONDS); runsPerDay = units / ctx.scheduler().frequency(); @@ -102,14 +142,41 @@ public void doMonkeyBusiness() { double prob = getEffectiveProbability(group); Collection instances = context().chaosInstanceSelector().select(group, prob / runsPerDay); for (String inst : instances) { - terminateInstance(group, inst); + ChaosType chaosType = pickChaosType(context().cloudClient(), inst); + if (chaosType == null) { + // This is surprising ... normally we can always just terminate it + LOGGER.warn("No chaos type was applicable to the instance: {}", inst); + continue; + } + terminateInstance(group, inst, chaosType); } } } } + private ChaosType pickChaosType(CloudClient cloudClient, String instanceId) { + Random random = new Random(); + + SshConfig sshConfig = new SshConfig(cfg); + ChaosInstance instance = new ChaosInstance(cloudClient, instanceId, sshConfig); + + List applicable = Lists.newArrayList(); + for (ChaosType chaosType : allChaosTypes) { + if (chaosType.isEnabled() && chaosType.canApply(instance)) { + applicable.add(chaosType); + } + } + + if (applicable.isEmpty()) { + return null; + } + + int index = random.nextInt(applicable.size()); + return applicable.get(index); + } + @Override - public Event terminateNow(String type, String name) + public Event terminateNow(String type, String name, ChaosType chaosType) throws FeatureNotEnabledException, InstanceGroupNotFoundException { Validate.notNull(type); Validate.notNull(name); @@ -129,7 +196,7 @@ public Event terminateNow(String type, String name) Collection instances = context().chaosInstanceSelector().select(group, 1.0); Validate.isTrue(instances.size() <= 1); if (instances.size() == 1) { - return terminateInstance(group, instances.iterator().next()); + return terminateInstance(group, instances.iterator().next(), chaosType); } else { throw new NotFoundException(String.format("No instance is found in group %s [type %s]", name, type)); @@ -155,16 +222,17 @@ private void reportEventForSummary(EventTypes eventType, InstanceGroup group, St * the exception */ protected void handleTerminationError(String instance, Throwable e) { - LOGGER.error("failed to terminate instance " + instance, e.getMessage()); + LOGGER.error("failed to terminate instance " + instance, e); throw new RuntimeException("failed to terminate instance " + instance, e); } /** {@inheritDoc} */ @Override - public Event recordTermination(InstanceGroup group, String instance) { + public Event recordTermination(InstanceGroup group, String instance, ChaosType chaosType) { Event evt = context().recorder().newEvent(Type.CHAOS, EventTypes.CHAOS_TERMINATION, group.region(), instance); evt.addField("groupType", group.type().name()); evt.addField("groupName", group.name()); + evt.addField("chaosType", chaosType.getKey()); context().recorder().recordEvent(evt); return evt; } @@ -304,7 +372,7 @@ private InstanceGroup findInstanceGroup(String type, String name) { return null; } - private Event terminateInstance(InstanceGroup group, String inst) { + private Event terminateInstance(InstanceGroup group, String inst, ChaosType chaosType) { Validate.notNull(group); Validate.notEmpty(inst); String prop = NS + "leashed"; @@ -315,10 +383,13 @@ private Event terminateInstance(InstanceGroup group, String inst) { return null; } else { try { - Event evt = recordTermination(group, inst); - sendTerminationNotification(group, inst); - context().cloudClient().terminateInstance(inst); - LOGGER.info("Terminated {} from group {} [{}]", new Object[]{inst, group.name(), group.type()}); + Event evt = recordTermination(group, inst, chaosType); + sendTerminationNotification(group, inst, chaosType); + SshConfig sshConfig = new SshConfig(cfg); + ChaosInstance chaosInstance = new ChaosInstance(context().cloudClient(), inst, sshConfig); + chaosType.apply(chaosInstance); + LOGGER.info("Terminated {} from group {} [{}] with {}", + new Object[]{inst, group.name(), group.type(), chaosType.getKey() }); reportEventForSummary(EventTypes.CHAOS_TERMINATION, group, inst); return evt; } catch (NotFoundException e) { @@ -371,7 +442,7 @@ protected boolean isMaxTerminationCountExceeded(InstanceGroup group) { } @Override - public void sendTerminationNotification(InstanceGroup group, String instance) { + public void sendTerminationNotification(InstanceGroup group, String instance, ChaosType chaosType) { String propEmailGlobalEnabled = "simianarmy.chaos.notification.global.enabled"; String propEmailGroupEnabled = String.format("%s%s.%s.notification.enabled", NS, group.type(), group.name()); @@ -382,10 +453,18 @@ public void sendTerminationNotification(InstanceGroup group, String instance) { throw new RuntimeException(msg); } if (cfg.getBoolOrElse(propEmailGroupEnabled, false)) { - notifier.sendTerminationNotification(group, instance); + notifier.sendTerminationNotification(group, instance, chaosType); } if (cfg.getBoolOrElse(propEmailGlobalEnabled, false)) { - notifier.sendTerminationGlobalNotification(group, instance); + notifier.sendTerminationGlobalNotification(group, instance, chaosType); } } + + /** + * {@inheritDoc} + */ + @Override + public List getChaosTypes() { + return Lists.newArrayList(allChaosTypes); + } } \ No newline at end of file diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java index 4982e5df..892f5e89 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java @@ -1,6 +1,7 @@ package com.netflix.simianarmy.basic.chaos; import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; +import com.netflix.simianarmy.chaos.ChaosType; /** * The Class CloudFormationChaosMonkey. Strips out the random string generated by the CloudFormation api in @@ -61,9 +62,9 @@ protected long getLastOptInMilliseconds(InstanceGroup group) { * Handle email notifications for no suffix instance groups. */ @Override - public void sendTerminationNotification(InstanceGroup group, String instance) { + public void sendTerminationNotification(InstanceGroup group, String instance, ChaosType chaosType) { InstanceGroup noSuffixGroup = noSuffixInstanceGroup(group); - super.sendTerminationNotification(noSuffixGroup, instance); + super.sendTerminationNotification(noSuffixGroup, instance, chaosType); } /** diff --git a/src/main/java/com/netflix/simianarmy/chaos/BlockAllNetworkTrafficChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/BlockAllNetworkTrafficChaosType.java new file mode 100644 index 00000000..5be2042e --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/BlockAllNetworkTrafficChaosType.java @@ -0,0 +1,96 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Blocks network traffic to/from instance, so it is running but offline. + * + * We actually put the instance into a different security group. First, because AWS requires a SG for some reason. + * Second, because you might well want to continue to allow e.g. SSH inbound. + */ +public class BlockAllNetworkTrafficChaosType extends ChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BlockAllNetworkTrafficChaosType.class); + + private final String blockedSecurityGroupName; + + /** + * Constructor. + * + * @param config + * Configuration to use + */ + public BlockAllNetworkTrafficChaosType(MonkeyConfiguration config) { + super(config, "BlockAllNetworkTraffic"); + + this.blockedSecurityGroupName = config.getStrOrElse(getConfigurationPrefix() + "group", "blocked-network"); + } + + /** + * We can apply the strategy iff the blocked security group is configured. + */ + @Override + public boolean canApply(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + if (!cloudClient.canChangeInstanceSecurityGroups(instanceId)) { + LOGGER.info("Not a VPC instance, can't change security groups"); + return false; + } + + return super.canApply(instance); + } + + /** + * Takes the instance off the network. + */ + @Override + public void apply(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + if (!cloudClient.canChangeInstanceSecurityGroups(instanceId)) { + throw new IllegalStateException("canApply should have returned false"); + } + + String groupId = cloudClient.findSecurityGroup(instance.getInstanceId(), blockedSecurityGroupName); + + if (groupId == null) { + LOGGER.info("Auto-creating security group {}", blockedSecurityGroupName); + + String description = "Empty security group for blocked instances"; + groupId = cloudClient.createSecurityGroup(instance.getInstanceId(), blockedSecurityGroupName, description); + } + + LOGGER.info("Blocking network traffic by applying security group {} to instance {}", groupId, instanceId); + + List groups = Lists.newArrayList(); + groups.add(groupId); + cloudClient.setInstanceSecurityGroups(instanceId, groups); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/BurnCpuChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/BurnCpuChaosType.java new file mode 100644 index 00000000..647a2c1f --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/BurnCpuChaosType.java @@ -0,0 +1,38 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Executes a CPU intensive program on the node, using up all available CPU. + * + * This simulates either a noisy CPU neighbor on the box or just a general issue with the CPU. + */ +public class BurnCpuChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public BurnCpuChaosType(MonkeyConfiguration config) { + super(config, "BurnCpu"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/BurnIoChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/BurnIoChaosType.java new file mode 100644 index 00000000..e957c130 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/BurnIoChaosType.java @@ -0,0 +1,81 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Executes a disk I/O intensive program on the node, reducing I/O capacity. + * + * This simulates either a noisy neighbor on the box or just a general issue with the disk. + */ +public class BurnIoChaosType extends ScriptChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BurnIoChaosType.class); + + /** + * Enhancement: It would be nice to target other devices than the root disk. + * + * Considerations: + * 1) EBS activity costs money. + * 2) The root may be on EBS anyway. + * 3) If it's costing money, we might want to stop after a while to stop runaway charges. + * + * coryb suggested this, and proposed something like this: + * + * tmp=$(mktemp) + * df -hl -x tmpfs | awk '/\//{print $6}' > $tmp + * mount=$(sed -n $((RANDOM%$(wc -l < $tmp)+1))p $tmp) + * rm $tmp + * + * And then of=$mount/burn + * + * An alternative might be to run df over SSH, parse it here, and then pass the desired + * path to the script. This keeps the script simpler. I don't think there's an easy way + * to tell the difference between an EBS volume and an instance volume other than from the + * EC2 API. + */ + + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public BurnIoChaosType(MonkeyConfiguration config) { + super(config, "BurnIO"); + } + + @Override + public boolean canApply(ChaosInstance instance) { + if (!super.canApply(instance)) { + return false; + } + + if (isRootVolumeEbs(instance) && !isBurnMoneyEnabled()) { + LOGGER.debug("Root volume is EBS so BurnIO would cost money; skipping"); + return false; + } + + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java index b74e1f1e..39c561b5 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java @@ -41,15 +41,17 @@ public ChaosEmailNotifier(AmazonSimpleEmailServiceClient sesClient) { * owner's email address. * @param group the instance group * @param instance the instance id + * @param chaosType the chosen chaos strategy */ - public abstract void sendTerminationNotification(InstanceGroup group, String instance); + public abstract void sendTerminationNotification(InstanceGroup group, String instance, ChaosType chaosType); /** * Sends an email notification for a termination of instance to a global * email address. * @param group the instance group * @param instance the instance id + * @param chaosType the chosen chaos strategy */ - public abstract void sendTerminationGlobalNotification(InstanceGroup group, String instance); + public abstract void sendTerminationGlobalNotification(InstanceGroup group, String instance, ChaosType chaosType); } diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosInstance.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosInstance.java new file mode 100644 index 00000000..f3171c96 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosInstance.java @@ -0,0 +1,127 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import org.jclouds.domain.LoginCredentials; +import org.jclouds.ssh.SshClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.CloudClient; + +/** + * Wrapper around an instance on which we are going to cause chaos. + */ +public class ChaosInstance { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ChaosInstance.class); + + private final CloudClient cloudClient; + private final String instanceId; + private final SshConfig sshConfig; + + /** + * Constructor. + * + * @param cloudClient + * client for cloud access + * @param instanceId + * id of instance on cloud + * @param sshConfig + * SSH configuration to access instance + */ + public ChaosInstance(CloudClient cloudClient, String instanceId, SshConfig sshConfig) { + this.cloudClient = cloudClient; + this.instanceId = instanceId; + this.sshConfig = sshConfig; + } + + /** + * Gets the {@link SshConfig} used to SSH to the instance. + * + * @return the {@link SshConfig} + */ + public SshConfig getSshConfig() { + return sshConfig; + } + + /** + * Gets the {@link CloudClient} used to access the cloud. + * + * @return the {@link CloudClient} + */ + public CloudClient getCloudClient() { + return cloudClient; + } + + /** + * Returns the instance id to identify the instance to the cloud client. + * + * @return instance id + */ + public String getInstanceId() { + return instanceId; + } + + /** + * Memoize canConnectSsh function. + */ + private Boolean canConnectSsh = null; + + /** + * Check if the SSH credentials are working. + * + * This is cached for the duration of this object. + * + * @return true iff ssh is configured and able to log on to instance. + */ + public boolean canConnectSsh(ChaosInstance instance) { + if (!sshConfig.isEnabled()) { + return false; + } + + if (canConnectSsh == null) { + try { + // It would be nicer to keep this connection open, but then we'd have to be closed. + SshClient client = connectSsh(); + client.disconnect(); + canConnectSsh = true; + } catch (Exception e) { + LOGGER.warn("Error making SSH connection to instance", e); + canConnectSsh = false; + } + } + return canConnectSsh; + } + + /** + * Connect to the instance over SSH. + * + * @return {@link SshClient} for connection + */ + public SshClient connectSsh() { + if (!sshConfig.isEnabled()) { + throw new IllegalStateException(); + } + + LoginCredentials credentials = sshConfig.getCredentials(); + SshClient ssh = cloudClient.connectSsh(instanceId, credentials); + + return ssh; + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java index 2bfc3f35..2d3518ab 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java @@ -18,6 +18,7 @@ package com.netflix.simianarmy.chaos; import java.util.Date; +import java.util.List; import com.netflix.simianarmy.FeatureNotEnabledException; import com.netflix.simianarmy.InstanceGroupNotFoundException; @@ -132,7 +133,7 @@ public Context context() { * the instance * @return the termination event */ - public abstract Event recordTermination(ChaosCrawler.InstanceGroup group, String instance); + public abstract Event recordTermination(ChaosCrawler.InstanceGroup group, String instance, ChaosType chaosType); /** * Terminates one instance right away from an instance group when there are available instances. @@ -144,7 +145,7 @@ public Context context() { * @throws FeatureNotEnabledException * @throws InstanceGroupNotFoundException */ - public abstract Event terminateNow(String type, String name) + public abstract Event terminateNow(String type, String name, ChaosType chaosType) throws FeatureNotEnabledException, InstanceGroupNotFoundException; /** @@ -154,6 +155,14 @@ public abstract Event terminateNow(String type, String name) * the group * @param instance * the instance + * @param chaosType + * the chaos monkey strategy that was chosen */ - public abstract void sendTerminationNotification(ChaosCrawler.InstanceGroup group, String instance); + public abstract void sendTerminationNotification(ChaosCrawler.InstanceGroup group, String instance, + ChaosType chaosType); + + /** + * Gets a list of all enabled chaos types for this ChaosMonkey. + */ + public abstract List getChaosTypes(); } diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosType.java new file mode 100644 index 00000000..a322a3ef --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosType.java @@ -0,0 +1,147 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * A strategy pattern for different types of chaos the chaos monkey can cause. + */ +public abstract class ChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ChaosType.class); + + /** + * Configuration for this chaos type. + */ + private final MonkeyConfiguration config; + + /** + * The unique key for the ChaosType. + */ + private final String key; + + /** + * Is this strategy enabled? + */ + private final boolean enabled; + + /** + * Protected constructor (abstract class). + * + * @param config + * Configuration to use + * @param key + * Unique key for the ChaosType strategy + */ + protected ChaosType(MonkeyConfiguration config, String key) { + this.config = config; + this.key = key; + this.enabled = config.getBoolOrElse(getConfigurationPrefix() + "enabled", getEnabledDefault()); + + LOGGER.info("ChaosType: {}: enabled={}", key, enabled); + } + + /** + * If not specified, controls whether we default to enabled. + * + * Most ChaosTypes should be disabled by default, not least for legacy compatibility, but we want at least one + * strategy to be available. + */ + protected boolean getEnabledDefault() { + return false; + } + + /** + * Returns the configuration key prefix to use for this strategy. + */ + protected String getConfigurationPrefix() { + return "simianarmy.chaos." + key.toLowerCase() + "."; + } + + /** + * Returns the unique key for the ChaosType. + */ + public String getKey() { + return key; + } + + /** + * Checks if this chaos type can be applied to the given instance. + * + * For example, if the strategy was to detach all the EBS volumes, that only makes sense if there are EBS volumes to + * detach. + */ + public boolean canApply(ChaosInstance instance) { + return isEnabled(); + } + + /** + * Returns whether we are enabled. + */ + public boolean isEnabled() { + return enabled; + } + + /** + * Applies this chaos type to the specified instance. + */ + public abstract void apply(ChaosInstance instance); + + /** + * Returns the ChaosType with the matching key. + */ + public static ChaosType parse(List all, String chaosTypeName) { + for (ChaosType chaosType : all) { + if (chaosType.getKey().equalsIgnoreCase(chaosTypeName)) { + return chaosType; + } + } + throw new IllegalArgumentException("Unknown chaos type value: " + + chaosTypeName); + } + + /** + * Returns whether chaos types that cost money are allowed. + */ + protected boolean isBurnMoneyEnabled() { + return config.getBoolOrElse("simianarmy.chaos.burnmoney", false); + } + + /** + * Checks whether the root volume of the specified instance is on EBS. + * + * @param instance id of instance + * @return true iff root is on EBS + */ + protected boolean isRootVolumeEbs(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + List withRoot = cloudClient.listAttachedVolumes(instanceId, true); + List withoutRoot = cloudClient.listAttachedVolumes(instanceId, false); + + return (withRoot.size() != withoutRoot.size()); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/DetachVolumesChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/DetachVolumesChaosType.java new file mode 100644 index 00000000..c6faebc0 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/DetachVolumesChaosType.java @@ -0,0 +1,81 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey; + +/** + * We force-detach all the EBS volumes. + * + * This is supposed to simulate a catastrophic failure of EBS, however the instance will (possibly) still keep running; + * e.g. it should continue to respond to pings. + */ +public class DetachVolumesChaosType extends ChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicChaosMonkey.class); + + /** + * Constructor. + * + * @param config + * Configuration to use + */ + public DetachVolumesChaosType(MonkeyConfiguration config) { + super(config, "DetachVolumes"); + } + + /** + * Strategy can be applied iff there are any EBS volumes attached. + */ + @Override + public boolean canApply(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + List volumes = cloudClient.listAttachedVolumes(instanceId, false); + if (volumes.isEmpty()) { + LOGGER.debug("Can't apply strategy: no non-root EBS volumes"); + return false; + } + + return super.canApply(instance); + } + + /** + * Force-detaches all attached EBS volumes from the instance. + */ + @Override + public void apply(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + // IDEA: We could have a strategy where we detach some of the volumes... + boolean force = true; + for (String volumeId : cloudClient.listAttachedVolumes(instanceId, false)) { + cloudClient.detachVolume(instanceId, volumeId, force); + } + } + +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/FailDnsChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/FailDnsChaosType.java new file mode 100644 index 00000000..2b6057c1 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/FailDnsChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Blocks TCP & UDP port 53, so DNS resolution fails. + */ +public class FailDnsChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public FailDnsChaosType(MonkeyConfiguration config) { + super(config, "FailDns"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/FailDynamoDbChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/FailDynamoDbChaosType.java new file mode 100644 index 00000000..c76ee079 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/FailDynamoDbChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Adds entries to /etc/hosts so that DynamoDB API endpoints are unreachable. + */ +public class FailDynamoDbChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public FailDynamoDbChaosType(MonkeyConfiguration config) { + super(config, "FailDynamoDb"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/FailEc2ChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/FailEc2ChaosType.java new file mode 100644 index 00000000..77a58a97 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/FailEc2ChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Adds entries to /etc/hosts so that EC2 API endpoints are unreachable. + */ +public class FailEc2ChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public FailEc2ChaosType(MonkeyConfiguration config) { + super(config, "FailEc2"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/FailS3ChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/FailS3ChaosType.java new file mode 100644 index 00000000..ecd97afe --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/FailS3ChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Adds entries to /etc/hosts so that S3 API endpoints are unreachable. + */ +public class FailS3ChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public FailS3ChaosType(MonkeyConfiguration config) { + super(config, "FailS3"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/FillDiskChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/FillDiskChaosType.java new file mode 100644 index 00000000..6b8f9b17 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/FillDiskChaosType.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Creates a huge file on the root device so that the disk fills up. + */ +public class FillDiskChaosType extends ScriptChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(FillDiskChaosType.class); + + /** + * Enhancement: As with BurnIoChaosType, it would be nice to randomize the volume. + * + * coryb suggested this, and proposed this script: + * + * nohup dd if=/dev/urandom of=/burn bs=1M count=$(df -ml /burn | awk '/\//{print $2}') iflag=fullblock & + */ + + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public FillDiskChaosType(MonkeyConfiguration config) { + super(config, "FillDisk"); + } + + @Override + public boolean canApply(ChaosInstance instance) { + if (!super.canApply(instance)) { + return false; + } + + if (isRootVolumeEbs(instance) && !isBurnMoneyEnabled()) { + LOGGER.debug("Root volume is EBS so FillDisk would cost money; skipping"); + return false; + } + + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/KillProcessesChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/KillProcessesChaosType.java new file mode 100644 index 00000000..4ed6924a --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/KillProcessesChaosType.java @@ -0,0 +1,38 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Kills processes on the node. + * + * This simulates the process crashing (for any reason). + */ +public class KillProcessesChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public KillProcessesChaosType(MonkeyConfiguration config) { + super(config, "KillProcesses"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/NetworkCorruptionChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/NetworkCorruptionChaosType.java new file mode 100644 index 00000000..a33ed155 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/NetworkCorruptionChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Introduces network packet corruption using traffic-shaping. + */ +public class NetworkCorruptionChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public NetworkCorruptionChaosType(MonkeyConfiguration config) { + super(config, "NetworkCorruption"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/NetworkLatencyChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/NetworkLatencyChaosType.java new file mode 100644 index 00000000..06c00803 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/NetworkLatencyChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Introduces network latency using traffic-shaping. + */ +public class NetworkLatencyChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public NetworkLatencyChaosType(MonkeyConfiguration config) { + super(config, "NetworkLatency"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/NetworkLossChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/NetworkLossChaosType.java new file mode 100644 index 00000000..c30a2cd8 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/NetworkLossChaosType.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Introduces network packet loss using traffic-shaping. + */ +public class NetworkLossChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public NetworkLossChaosType(MonkeyConfiguration config) { + super(config, "NetworkLoss"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/NullRouteChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/NullRouteChaosType.java new file mode 100644 index 00000000..2ec4b733 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/NullRouteChaosType.java @@ -0,0 +1,41 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Null routes the network, taking a node going offline. + * + * Currently we offline 10.x.x.x (the AWS private network range). + * + * I think the machine will still be publicly accessible, but won't be able to communicate with any other nodes on + * the EC2 network. + */ +public class NullRouteChaosType extends ScriptChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public NullRouteChaosType(MonkeyConfiguration config) { + super(config, "NullRoute"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/ScriptChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/ScriptChaosType.java new file mode 100644 index 00000000..81dc3648 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/ScriptChaosType.java @@ -0,0 +1,94 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.io.IOException; +import java.net.URL; +import org.jclouds.compute.domain.ExecResponse; +import org.jclouds.ssh.SshClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Base class for chaos types that run a script over JClouds/SSH on the node. + */ +public abstract class ScriptChaosType extends ChaosType { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ScriptChaosType.class); + + /** + * Constructor. + * + * @param config + * Configuration to use + * @param key + * Key for the chaos money + * @throws IOException + */ + public ScriptChaosType(MonkeyConfiguration config, String key) { + super(config, key); + } + + /** + * We can apply the strategy iff we can SSH to the instance. + */ + @Override + public boolean canApply(ChaosInstance instance) { + if (!instance.getSshConfig().isEnabled()) { + LOGGER.info("Strategy disabled because SSH credentials not set"); + return false; + } + + if (!instance.canConnectSsh(instance)) { + LOGGER.warn("Strategy disabled because SSH credentials failed"); + return false; + } + + return super.canApply(instance); + } + + /** + * Runs the script. + */ + @Override + public void apply(ChaosInstance instance) { + LOGGER.info("Running script for {} on instance {}", getKey(), instance.getInstanceId()); + + SshClient ssh = instance.connectSsh(); + + String filename = getKey().toLowerCase() + ".sh"; + URL url = Resources.getResource(ScriptChaosType.class, "/scripts/" + filename); + String script; + try { + script = Resources.toString(url, Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Error reading script resource", e); + } + + ssh.put("/tmp/" + filename, script); + ExecResponse response = ssh.exec("/bin/bash /tmp/" + filename); + if (response.getExitStatus() != 0) { + LOGGER.warn("Got non-zero output from running script: {}", response); + } + ssh.disconnect(); + } +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/ShutdownInstanceChaosType.java b/src/main/java/com/netflix/simianarmy/chaos/ShutdownInstanceChaosType.java new file mode 100644 index 00000000..1892c70b --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/ShutdownInstanceChaosType.java @@ -0,0 +1,58 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Shuts down the instance using the cloud instance-termination API. + * + * This is the classic chaos-monkey strategy. + */ +public class ShutdownInstanceChaosType extends ChaosType { + /** + * Constructor. + * + * @param config + * Configuration to use + */ + public ShutdownInstanceChaosType(MonkeyConfiguration config) { + super(config, "ShutdownInstance"); + } + + /** + * Shuts down the instance. + */ + @Override + public void apply(ChaosInstance instance) { + CloudClient cloudClient = instance.getCloudClient(); + String instanceId = instance.getInstanceId(); + + cloudClient.terminateInstance(instanceId); + } + + /** + * We want to default to enabled. + */ + @Override + protected boolean getEnabledDefault() { + return true; + } + +} diff --git a/src/main/java/com/netflix/simianarmy/chaos/SshConfig.java b/src/main/java/com/netflix/simianarmy/chaos/SshConfig.java new file mode 100644 index 00000000..64163c4e --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/chaos/SshConfig.java @@ -0,0 +1,100 @@ +/* + * + * Copyright 2013 Justin Santa Barbara. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.io.File; +import java.io.IOException; +import org.jclouds.domain.LoginCredentials; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Charsets; +import com.google.common.base.Strings; +import com.google.common.io.Files; +import com.netflix.simianarmy.MonkeyConfiguration; + +/** + * Holds SSH connection info, used for script-based chaos types. + */ +public class SshConfig { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(SshConfig.class); + + /** + * The SSH credentials to log on to an instance. + */ + private final LoginCredentials sshCredentials; + + /** + * Constructor. + * + * @param config + * Configuration to use + * @throws IOException + */ + public SshConfig(MonkeyConfiguration config) { + String sshUser = config.getStrOrElse("simianarmy.chaos.ssh.user", "root"); + String privateKey = null; + + String sshKeyPath = config.getStrOrElse("simianarmy.chaos.ssh.key", null); + if (sshKeyPath != null) { + sshKeyPath = sshKeyPath.trim(); + if (sshKeyPath.startsWith("~/")) { + String home = System.getProperty("user.home"); + if (!Strings.isNullOrEmpty(home)) { + if (!home.endsWith("/")) { + home += "/"; + } + sshKeyPath = home + sshKeyPath.substring(2); + } + } + + LOGGER.debug("Reading SSH key from {}", sshKeyPath); + + try { + privateKey = Files.toString(new File(sshKeyPath), Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Unable to read the specified SSH key: " + sshKeyPath, e); + } + } + + if (privateKey == null) { + this.sshCredentials = null; + } else { + this.sshCredentials = LoginCredentials.builder().user(sshUser).privateKey(privateKey).build(); + } + } + + /** + * Get the configured SSH credentials. + * + * @return configured SSH credentials + */ + public LoginCredentials getCredentials() { + return sshCredentials; + } + + /** + * Check if ssh is configured. + * + * @return true if credentials are configured + */ + public boolean isEnabled() { + return sshCredentials != null; + } +} diff --git a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java index 0ad4f8df..a8cea238 100644 --- a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java +++ b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java @@ -33,6 +33,8 @@ import com.amazonaws.services.autoscaling.model.LaunchConfiguration; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest; +import com.amazonaws.services.ec2.model.CreateSecurityGroupResult; import com.amazonaws.services.ec2.model.CreateTagsRequest; import com.amazonaws.services.ec2.model.DeleteSnapshotRequest; import com.amazonaws.services.ec2.model.DeleteVolumeRequest; @@ -41,13 +43,20 @@ import com.amazonaws.services.ec2.model.DescribeImagesResult; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; import com.amazonaws.services.ec2.model.DescribeInstancesResult; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest; +import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult; import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest; import com.amazonaws.services.ec2.model.DescribeSnapshotsResult; import com.amazonaws.services.ec2.model.DescribeVolumesRequest; import com.amazonaws.services.ec2.model.DescribeVolumesResult; +import com.amazonaws.services.ec2.model.DetachVolumeRequest; +import com.amazonaws.services.ec2.model.EbsInstanceBlockDevice; import com.amazonaws.services.ec2.model.Image; import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceBlockDeviceMapping; +import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest; import com.amazonaws.services.ec2.model.Reservation; +import com.amazonaws.services.ec2.model.SecurityGroup; import com.amazonaws.services.ec2.model.Snapshot; import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; @@ -58,17 +67,37 @@ import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription; import com.amazonaws.services.simpledb.AmazonSimpleDB; import com.amazonaws.services.simpledb.AmazonSimpleDBClient; +import com.google.common.base.Objects; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.inject.Module; import com.netflix.simianarmy.CloudClient; import com.netflix.simianarmy.NotFoundException; + import org.apache.commons.lang.Validate; +import org.jclouds.ContextBuilder; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.ComputeServiceContext; +import org.jclouds.compute.Utils; +import org.jclouds.compute.domain.ComputeMetadata; +import org.jclouds.compute.domain.NodeMetadata; +import org.jclouds.compute.domain.NodeMetadataBuilder; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.jclouds.ssh.SshClient; +import org.jclouds.ssh.jsch.config.JschSshClientModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; /** @@ -84,6 +113,8 @@ public class AWSClient implements CloudClient { private final AWSCredentialsProvider awsCredentialsProvider; + private ComputeService jcloudsComputeService; + /** * This constructor will let the AWS SDK obtain the credentials, which will * choose such in the following order: @@ -440,6 +471,23 @@ public void terminateInstance(String instanceId) { } } + /** {@inheritDoc} */ + public void setInstanceSecurityGroups(String instanceId, List groupIds) { + Validate.notEmpty(instanceId); + LOGGER.info(String.format("Removing all security groups from instance %s in region %s.", instanceId, region)); + try { + ModifyInstanceAttributeRequest request = new ModifyInstanceAttributeRequest(); + request.setInstanceId(instanceId); + request.setGroups(groupIds); + ec2Client().modifyInstanceAttribute(request); + } catch (AmazonServiceException e) { + if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) { + throw new NotFoundException("AWS instance " + instanceId + " not found", e); + } + throw e; + } + } + /** * Describe a set of specific EBS volumes. * @@ -531,4 +579,234 @@ public List describeImages(String... imageIds) { LOGGER.info(String.format("Got %d AMIs in region %s.", images.size(), region)); return images; } + + @Override + public void detachVolume(String instanceId, String volumeId, boolean force) { + Validate.notEmpty(instanceId); + LOGGER.info(String.format("Detach volumes from instance %s in region %s.", instanceId, region)); + try { + DetachVolumeRequest detachVolumeRequest = new DetachVolumeRequest(); + detachVolumeRequest.setForce(force); + detachVolumeRequest.setInstanceId(instanceId); + detachVolumeRequest.setVolumeId(volumeId); + ec2Client().detachVolume(detachVolumeRequest); + } catch (AmazonServiceException e) { + if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) { + throw new NotFoundException("AWS instance " + instanceId + " not found", e); + } + throw e; + } + } + + @Override + public List listAttachedVolumes(String instanceId, boolean includeRoot) { + Validate.notEmpty(instanceId); + LOGGER.info(String.format("Listing volumes attached to instance %s in region %s.", instanceId, region)); + try { + List volumeIds = new ArrayList(); + for (Instance instance : describeInstances(instanceId)) { + String rootDeviceName = instance.getRootDeviceName(); + + for (InstanceBlockDeviceMapping ibdm : instance.getBlockDeviceMappings()) { + EbsInstanceBlockDevice ebs = ibdm.getEbs(); + if (ebs == null) { + continue; + } + + String volumeId = ebs.getVolumeId(); + if (Strings.isNullOrEmpty(volumeId)) { + continue; + } + + if (!includeRoot && rootDeviceName != null && rootDeviceName.equals(ibdm.getDeviceName())) { + continue; + } + + volumeIds.add(volumeId); + } + } + return volumeIds; + } catch (AmazonServiceException e) { + if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) { + throw new NotFoundException("AWS instance " + instanceId + " not found", e); + } + throw e; + } + } + + /** + * Describe a set of security groups. + * + * @param groupNames the names of the groups to find + * @return a list of matching groups + */ + public List describeSecurityGroups(String... groupNames) { + AmazonEC2 ec2Client = ec2Client(); + DescribeSecurityGroupsRequest request = new DescribeSecurityGroupsRequest(); + + if (groupNames == null || groupNames.length == 0) { + LOGGER.info(String.format("Getting all EC2 security groups in region %s.", region)); + } else { + LOGGER.info(String.format("Getting EC2 security groups for %d names in region %s.", groupNames.length, + region)); + request.withGroupNames(groupNames); + } + + DescribeSecurityGroupsResult result; + try { + result = ec2Client.describeSecurityGroups(request); + } catch (AmazonServiceException e) { + if (e.getErrorCode().equals("InvalidGroup.NotFound")) { + LOGGER.info("Got InvalidGroup.NotFound error for security groups; returning empty list"); + return Collections.emptyList(); + } + throw e; + } + + List securityGroups = result.getSecurityGroups(); + LOGGER.info(String.format("Got %d EC2 security groups in region %s.", securityGroups.size(), region)); + return securityGroups; + } + + /** {@inheritDoc} */ + public String createSecurityGroup(String instanceId, String name, String description) { + String vpcId = getVpcId(instanceId); + + AmazonEC2 ec2Client = ec2Client(); + CreateSecurityGroupRequest request = new CreateSecurityGroupRequest(); + request.setGroupName(name); + request.setDescription(description); + request.setVpcId(vpcId); + + LOGGER.info(String.format("Creating EC2 security group %s.", name)); + + CreateSecurityGroupResult result = ec2Client.createSecurityGroup(request); + return result.getGroupId(); + } + + /** + * Convenience wrapper around describeInstances, for a single instance id. + * + * @param instanceId id of instance to find + * @return the instance info, or null if instance not found + */ + public Instance describeInstance(String instanceId) { + Instance instance = null; + for (Instance i : describeInstances(instanceId)) { + if (instance != null) { + throw new IllegalStateException("Duplicate instance: " + instanceId); + } + instance = i; + } + return instance; + } + + /** {@inheritDoc} */ + @Override + public synchronized ComputeService getJcloudsComputeService() { + if (jcloudsComputeService == null) { + String username = awsCredentialsProvider.getCredentials().getAWSAccessKeyId(); + String password = awsCredentialsProvider.getCredentials().getAWSSecretKey(); + ComputeServiceContext jcloudsContext = ContextBuilder.newBuilder("ec2").credentials(username, password) + .modules(ImmutableSet.of(new SLF4JLoggingModule(), new JschSshClientModule())) + .buildView(ComputeServiceContext.class); + + this.jcloudsComputeService = jcloudsContext.getComputeService(); + } + + return jcloudsComputeService; + } + + /** {@inheritDoc} */ + @Override + public String getJcloudsId(String instanceId) { + return this.region + "/" + instanceId; + } + + @Override + public SshClient connectSsh(String instanceId, LoginCredentials credentials) { + ComputeService computeService = getJcloudsComputeService(); + + String jcloudsId = getJcloudsId(instanceId); + NodeMetadata node = getJcloudsNode(computeService, jcloudsId); + + node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(credentials).build(); + + Utils utils = computeService.getContext().getUtils(); + SshClient ssh = utils.sshForNode().apply(node); + + ssh.connect(); + + return ssh; + } + + private NodeMetadata getJcloudsNode(ComputeService computeService, String jcloudsId) { + // Work around a jclouds bug / documentation issue... + // TODO: Figure out what's broken, and eliminate this function + + // This should work (?): + // Set nodes = computeService.listNodesByIds(Collections.singletonList(jcloudsId)); + + Set nodes = Sets.newHashSet(); + for (ComputeMetadata n : computeService.listNodes()) { + if (jcloudsId.equals(n.getId())) { + nodes.add((NodeMetadata) n); + } + } + + if (nodes.isEmpty()) { + LOGGER.warn("Unable to find jclouds node: {}", jcloudsId); + for (ComputeMetadata n : computeService.listNodes()) { + LOGGER.info("Did find node: {}", n); + } + throw new IllegalStateException("Unable to find node using jclouds: " + jcloudsId); + } + NodeMetadata node = Iterables.getOnlyElement(nodes); + return node; + } + + /** {@inheritDoc} */ + @Override + public String findSecurityGroup(String instanceId, String groupName) { + String vpcId = getVpcId(instanceId); + + SecurityGroup found = null; + List securityGroups = describeSecurityGroups(vpcId, groupName); + for (SecurityGroup sg : securityGroups) { + if (Objects.equal(vpcId, sg.getVpcId())) { + if (found != null) { + throw new IllegalStateException("Duplicate security groups found"); + } + found = sg; + } + } + if (found == null) { + return null; + } + return found.getGroupId(); + } + + /** + * Gets the VPC id for the given instance. + * + * @param instanceId + * instance we're checking + * @return vpc id, or null if not a vpc instance + */ + String getVpcId(String instanceId) { + Instance awsInstance = describeInstance(instanceId); + + String vpcId = awsInstance.getVpcId(); + if (Strings.isNullOrEmpty(vpcId)) { + return null; + } + + return vpcId; + } + + /** {@inheritDoc} */ + @Override + public boolean canChangeInstanceSecurityGroups(String instanceId) { + return null != getVpcId(instanceId); + } } diff --git a/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java b/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java index f52aa73a..ab77b2e1 100644 --- a/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java +++ b/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java @@ -32,8 +32,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import com.google.common.base.Strings; import com.netflix.simianarmy.Monkey; import com.sun.jersey.spi.resource.Singleton; + import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.JsonEncoding; import org.codehaus.jackson.JsonGenerator; @@ -49,6 +51,8 @@ import com.netflix.simianarmy.MonkeyRunner; import com.netflix.simianarmy.NotFoundException; import com.netflix.simianarmy.chaos.ChaosMonkey; +import com.netflix.simianarmy.chaos.ChaosType; +import com.netflix.simianarmy.chaos.ShutdownInstanceChaosType; /** * The Class ChaosMonkeyResource for json REST apis. @@ -164,6 +168,14 @@ public Response addEvent(String content) throws IOException { String eventType = getStringField(input, "eventType"); String groupType = getStringField(input, "groupType"); String groupName = getStringField(input, "groupName"); + String chaosTypeName = getStringField(input, "chaosType"); + + ChaosType chaosType; + if (!Strings.isNullOrEmpty(chaosTypeName)) { + chaosType = ChaosType.parse(this.monkey.getChaosTypes(), chaosTypeName); + } else { + chaosType = new ShutdownInstanceChaosType(monkey.context().configuration()); + } Response.Status responseStatus; ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -172,13 +184,14 @@ public Response addEvent(String content) throws IOException { gen.writeStringField("eventType", eventType); gen.writeStringField("groupType", groupType); gen.writeStringField("groupName", groupName); + gen.writeStringField("chaosType", chaosType.getKey()); if (StringUtils.isEmpty(eventType) || StringUtils.isEmpty(groupType) || StringUtils.isEmpty(groupName)) { responseStatus = Response.Status.BAD_REQUEST; gen.writeStringField("message", "eventType, groupType, and groupName parameters are all required"); } else { if (eventType.equals("CHAOS_TERMINATION")) { - responseStatus = addTerminationEvent(groupType, groupName, gen); + responseStatus = addTerminationEvent(groupType, groupName, chaosType, gen); } else { responseStatus = Response.Status.BAD_REQUEST; gen.writeStringField("message", String.format("Unrecognized event type: %s", eventType)); @@ -190,13 +203,14 @@ public Response addEvent(String content) throws IOException { return Response.status(responseStatus).entity(baos.toString("UTF-8")).build(); } - private Response.Status addTerminationEvent(String groupType, String groupName, JsonGenerator gen) + private Response.Status addTerminationEvent(String groupType, + String groupName, ChaosType chaosType, JsonGenerator gen) throws IOException { LOGGER.info("Running on-demand termination for instance group type '{}' and name '{}'", groupType, groupName); Response.Status responseStatus; try { - Event evt = monkey.terminateNow(groupType, groupName); + Event evt = monkey.terminateNow(groupType, groupName, chaosType); if (evt != null) { responseStatus = Response.Status.OK; gen.writeStringField("monkeyType", evt.monkeyType().name()); diff --git a/src/main/resources/chaos.properties b/src/main/resources/chaos.properties index c7518623..aad28d50 100644 --- a/src/main/resources/chaos.properties +++ b/src/main/resources/chaos.properties @@ -17,6 +17,34 @@ simianarmy.chaos.ASG.probability = 1.0 # increase or decrease the termination limit simianarmy.chaos.ASG.maxTerminationsPerDay = 1.0 +# Strategies +simianarmy.chaos.shutdowninstance.enabled = true +simianarmy.chaos.blockallnetworktraffic.enabled = false +simianarmy.chaos.burncpu.enabled = false +simianarmy.chaos.killprocesses.enabled = false +simianarmy.chaos.nullroute.enabled = false +simianarmy.chaos.failapi.enabled = false +simianarmy.chaos.faildns.enabled = false +simianarmy.chaos.faildynamodb.enabled = false +simianarmy.chaos.fails3.enabled = false +simianarmy.chaos.networkcorruption.enabled = false +simianarmy.chaos.networklatency.enabled = false +simianarmy.chaos.networkloss.enabled = false + +# Force-detaching EBS volumes may cause data loss +simianarmy.chaos.detachvolumes.enabled = false + +# FillDisk fills the root disk. +# NOTE: This may incur charges for an EBS root volume. See burnmoney option. +simianarmy.chaos.burnio.enabled = false +# BurnIO causes disk activity on the root disk. +# NOTE: This may incur charges for an EBS root volume. See burnmoney option. +simianarmy.chaos.filldisk.enabled = false + +# Where we know the chaos strategy will incur charges, we won't run it unless burnmoney is true. +simianarmy.chaos.burnmoney = false + + # enable a specific ASG # simianarmy.chaos.ASG..enabled = true # simianarmy.chaos.ASG..probability = 1.0 diff --git a/src/main/resources/scripts/burncpu.sh b/src/main/resources/scripts/burncpu.sh new file mode 100644 index 00000000..69115345 --- /dev/null +++ b/src/main/resources/scripts/burncpu.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script for BurnCpu Chaos Monkey + +cat << EOF > /tmp/infiniteburn.sh +#!/bin/bash +while true; + do openssl speed; +done +EOF + +# 32 parallel 100% CPU tasks should hit even the biggest EC2 instances +for i in {1..32} +do + nohup /bin/bash /tmp/infiniteburn.sh & +done \ No newline at end of file diff --git a/src/main/resources/scripts/burnio.sh b/src/main/resources/scripts/burnio.sh new file mode 100644 index 00000000..13b3363a --- /dev/null +++ b/src/main/resources/scripts/burnio.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Script for BurnIO Chaos Monkey + +cat << EOF > /tmp/loopburnio.sh +#!/bin/bash +while true; +do + dd if=/dev/urandom of=/burn bs=1M count=1024 iflag=fullblock +done +EOF + +nohup /bin/bash /tmp/loopburnio.sh & diff --git a/src/main/resources/scripts/faildns.sh b/src/main/resources/scripts/faildns.sh new file mode 100644 index 00000000..ffe0abfe --- /dev/null +++ b/src/main/resources/scripts/faildns.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Script for FailDns Chaos Monkey + +# Block all traffic on port 53 +iptables -A INPUT -p tcp -m tcp --dport 53 -j DROP +iptables -A INPUT -p udp -m udp --dport 53 -j DROP diff --git a/src/main/resources/scripts/faildynamodb.sh b/src/main/resources/scripts/faildynamodb.sh new file mode 100644 index 00000000..16cc1262 --- /dev/null +++ b/src/main/resources/scripts/faildynamodb.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script for FailDynamoDb Chaos Monkey + +# Block well-known Amazon DynamoDB API endpoints +echo "127.0.0.1 dynamodb.us-east-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.us-northeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.us-gov-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.us-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.us-west-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.sa-east-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.ap-southeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.ap-southeast-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 dynamodb.eu-west-1.amazonaws.com" >> /etc/hosts + + diff --git a/src/main/resources/scripts/failec2.sh b/src/main/resources/scripts/failec2.sh new file mode 100644 index 00000000..de35cc49 --- /dev/null +++ b/src/main/resources/scripts/failec2.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script for FailEc2 Chaos Monkey + +# Block well-known Amazon EC2 API endpoints +echo "127.0.0.1 ec2.us-east-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.us-northeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.us-gov-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.us-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.us-west-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.sa-east-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.ap-southeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.ap-southeast-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 ec2.eu-west-1.amazonaws.com" >> /etc/hosts + + diff --git a/src/main/resources/scripts/fails3.sh b/src/main/resources/scripts/fails3.sh new file mode 100644 index 00000000..2615e1c7 --- /dev/null +++ b/src/main/resources/scripts/fails3.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Script for FailS3 Chaos Monkey + +# See http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + +echo "127.0.0.1 s3.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-external-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-us-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-us-west-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-eu-west-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-ap-southeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-ap-southeast-2.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-ap-northeast-1.amazonaws.com" >> /etc/hosts +echo "127.0.0.1 s3-sa-east-1.amazonaws.com" >> /etc/hosts + diff --git a/src/main/resources/scripts/filldisk.sh b/src/main/resources/scripts/filldisk.sh new file mode 100644 index 00000000..2525b04d --- /dev/null +++ b/src/main/resources/scripts/filldisk.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Script for FillDisk Chaos Monkey + +# 65 GB should be enough to fill up all EC2 root disks! +nohup dd if=/dev/urandom of=/burn bs=1M count=65536 iflag=fullblock & diff --git a/src/main/resources/scripts/killprocesses.sh b/src/main/resources/scripts/killprocesses.sh new file mode 100644 index 00000000..408ae9a7 --- /dev/null +++ b/src/main/resources/scripts/killprocesses.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Script for KillProcesses Chaos Monkey + +cat << EOF > /tmp/kill_loop.sh +#!/bin/bash +while true; +do + pkill -KILL -f java + pkill -KILL -f python + sleep 1 +done +EOF + +nohup /bin/bash /tmp/kill_loop.sh & diff --git a/src/main/resources/scripts/networkcorruption.sh b/src/main/resources/scripts/networkcorruption.sh new file mode 100644 index 00000000..c46198f8 --- /dev/null +++ b/src/main/resources/scripts/networkcorruption.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Script for NetworkCorruption Chaos Monkey + +# Corrupts 5% of packets +tc qdisc add dev eth0 root netem corrupt 5% diff --git a/src/main/resources/scripts/networklatency.sh b/src/main/resources/scripts/networklatency.sh new file mode 100644 index 00000000..b40b13c8 --- /dev/null +++ b/src/main/resources/scripts/networklatency.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Script for NetworkLatency Chaos Monkey + +# Adds 1000ms +- 500ms of latency to each packet +tc qdisc add dev eth0 root latency delay 1000ms 500ms + diff --git a/src/main/resources/scripts/networkloss.sh b/src/main/resources/scripts/networkloss.sh new file mode 100644 index 00000000..e2d9860f --- /dev/null +++ b/src/main/resources/scripts/networkloss.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Script for NetworkLoss Chaos Monkey + +# Drops 7% of packets, with 25% correlation with previous packet loss +# 7% is high, but it isn't high enough that TCP will fail entirely +tc qdisc add dev eth0 root netem loss 7% 25% + + diff --git a/src/main/resources/scripts/nullroute.sh b/src/main/resources/scripts/nullroute.sh new file mode 100644 index 00000000..03cbd5bf --- /dev/null +++ b/src/main/resources/scripts/nullroute.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Script for NullRoute Chaos Monkey + +ip route add blackhole 10.0.0.0/8 diff --git a/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java b/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java index b4bbd59d..ccc291ea 100644 --- a/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java @@ -26,6 +26,9 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; +import org.jclouds.compute.ComputeService; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.ssh.SshClient; import org.testng.Assert; import com.netflix.simianarmy.MonkeyRecorder.Event; @@ -132,6 +135,52 @@ public void deleteImage(String imageId) { @Override public void deleteLaunchConfiguration(String launchConfigName) { } + + @Override + public List listAttachedVolumes(String instanceId, boolean includeRoot) { + throw new UnsupportedOperationException(); + } + + @Override + public void detachVolume(String instanceId, String volumeId, + boolean force) { + throw new UnsupportedOperationException(); + } + + @Override + public ComputeService getJcloudsComputeService() { + throw new UnsupportedOperationException(); + } + + @Override + public String getJcloudsId(String instanceId) { + throw new UnsupportedOperationException(); + } + + @Override + public SshClient connectSsh(String instanceId, LoginCredentials credentials) { + throw new UnsupportedOperationException(); + } + + @Override + public String findSecurityGroup(String instanceId, String groupName) { + throw new UnsupportedOperationException(); + } + + @Override + public String createSecurityGroup(String instanceId, String groupName, String description) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean canChangeInstanceSecurityGroups(String instanceId) { + throw new UnsupportedOperationException(); + } + + @Override + public void setInstanceSecurityGroups(String instanceId, List groupIds) { + throw new UnsupportedOperationException(); + } }; } diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosEmailNotifier.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosEmailNotifier.java index 08fbac9b..128948d9 100644 --- a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosEmailNotifier.java +++ b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosEmailNotifier.java @@ -101,7 +101,7 @@ public void testbuildEmailSubjectWithSubjectPrefixSuffix() { @Test public void testbuildEmailBody() { basicChaosEmailNotifier = new BasicChaosEmailNotifier(new BasicConfiguration(properties), sesClient, null); - String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId); + String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId, null); Assert.assertEquals(subject, defaultBody); } @@ -109,7 +109,7 @@ public void testbuildEmailBody() { public void testbuildEmailBodyPrefix() { properties.setProperty("simianarmy.chaos.notification.body.prefix", bodyPrefix); basicChaosEmailNotifier = new BasicChaosEmailNotifier(new BasicConfiguration(properties), sesClient, null); - String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId); + String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId, null); Assert.assertEquals(subject, bodyPrefix + defaultBody); } @@ -117,7 +117,7 @@ public void testbuildEmailBodyPrefix() { public void testbuildEmailBodySuffix() { properties.setProperty("simianarmy.chaos.notification.body.suffix", bodySuffix); basicChaosEmailNotifier = new BasicChaosEmailNotifier(new BasicConfiguration(properties), sesClient, null); - String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId); + String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId, null); Assert.assertEquals(subject, defaultBody + bodySuffix); } @@ -126,7 +126,7 @@ public void testbuildEmailBodyPrefixSuffix() { properties.setProperty("simianarmy.chaos.notification.body.prefix", bodyPrefix); properties.setProperty("simianarmy.chaos.notification.body.suffix", bodySuffix); basicChaosEmailNotifier = new BasicChaosEmailNotifier(new BasicConfiguration(properties), sesClient, null); - String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId); + String subject = basicChaosEmailNotifier.buildEmailBody(testInstanceGroup, instanceId, null); Assert.assertEquals(subject, bodyPrefix + defaultBody + bodySuffix); } @@ -136,7 +136,7 @@ public void testBuildAndSendEmail() { BasicChaosEmailNotifier spyBasicChaosEmailNotifier = spy(new BasicChaosEmailNotifier(new BasicConfiguration( properties), sesClient, null)); doNothing().when(spyBasicChaosEmailNotifier).sendEmail(to, defaultSubject, defaultBody); - spyBasicChaosEmailNotifier.buildAndSendEmail(to, testInstanceGroup, instanceId); + spyBasicChaosEmailNotifier.buildAndSendEmail(to, testInstanceGroup, instanceId, null); verify(spyBasicChaosEmailNotifier).sendEmail(to, defaultSubject, defaultBody); } @@ -147,7 +147,7 @@ public void testBuildAndSendEmailSubjectIsBody() { BasicChaosEmailNotifier spyBasicChaosEmailNotifier = spy(new BasicChaosEmailNotifier(new BasicConfiguration( properties), sesClient, null)); doNothing().when(spyBasicChaosEmailNotifier).sendEmail(to, defaultBody, defaultBody); - spyBasicChaosEmailNotifier.buildAndSendEmail(to, testInstanceGroup, instanceId); + spyBasicChaosEmailNotifier.buildAndSendEmail(to, testInstanceGroup, instanceId, null); verify(spyBasicChaosEmailNotifier).sendEmail(to, defaultBody, defaultBody); } diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyArmy.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyArmy.java new file mode 100644 index 00000000..885d139f --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyArmy.java @@ -0,0 +1,328 @@ +// CHECKSTYLE IGNORE Javadoc +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.chaos; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey; +import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; +import com.netflix.simianarmy.chaos.TestChaosMonkeyContext.Notification; +import com.netflix.simianarmy.chaos.TestChaosMonkeyContext.SshAction; + +// CHECKSTYLE IGNORE MagicNumberCheck +public class TestChaosMonkeyArmy { + private File sshKey; + + @BeforeTest + public void createSshKey() throws IOException { + sshKey = File.createTempFile("tmp", "key"); + Files.write("fakekey", sshKey, Charsets.UTF_8); + sshKey.deleteOnExit(); + } + + private TestChaosMonkeyContext runChaosMonkey(String key) { + return runChaosMonkey(key, true); + } + + private TestChaosMonkeyContext runChaosMonkey(String key, boolean burnMoney) { + Properties properties = new Properties(); + properties.setProperty("simianarmy.chaos.enabled", "true"); + properties.setProperty("simianarmy.chaos.leashed", "false"); + properties.setProperty("simianarmy.chaos.TYPE_A.enabled", "true"); + properties.setProperty("simianarmy.chaos.notification.global.enabled", "true"); + + properties.setProperty("simianarmy.chaos.burnmoney", Boolean.toString(burnMoney)); + + properties.setProperty("simianarmy.chaos.shutdowninstance.enabled", "false"); + properties.setProperty("simianarmy.chaos." + key.toLowerCase() + ".enabled", "true"); + + properties.setProperty("simianarmy.chaos.ssh.key", sshKey.getAbsolutePath()); + + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext(properties); + + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + return ctx; + } + + private void checkSelected(TestChaosMonkeyContext ctx) { + List selectedOn = ctx.selectedOn(); + Assert.assertEquals(selectedOn.size(), 2); + Assert.assertEquals(selectedOn.get(0).type(), TestChaosMonkeyContext.CrawlerTypes.TYPE_A); + Assert.assertEquals(selectedOn.get(0).name(), "name0"); + Assert.assertEquals(selectedOn.get(1).type(), TestChaosMonkeyContext.CrawlerTypes.TYPE_A); + Assert.assertEquals(selectedOn.get(1).name(), "name1"); + } + + private void checkNotifications(TestChaosMonkeyContext ctx, String key) { + List notifications = ctx.getGloballyNotifiedList(); + Assert.assertEquals(notifications.size(), 2); + Assert.assertEquals(notifications.get(0).getInstance(), "0:i-123456780"); + Assert.assertEquals(notifications.get(0).getChaosType().getKey(), key); + Assert.assertEquals(notifications.get(1).getInstance(), "1:i-123456781"); + Assert.assertEquals(notifications.get(1).getChaosType().getKey(), key); + } + + private void checkSshActions(TestChaosMonkeyContext ctx, String key) { + List sshActions = ctx.getSshActions(); + Assert.assertEquals(sshActions.size(), 4); + + Assert.assertEquals(sshActions.get(0).getMethod(), "put"); + Assert.assertEquals(sshActions.get(0).getInstanceId(), "0:i-123456780"); + + // We require that each script include the name of the chaos type + // This makes testing easier, and also means the scripts show where they came from + Assert.assertTrue(sshActions.get(0).getContents().toLowerCase().contains(key.toLowerCase())); + + Assert.assertEquals(sshActions.get(1).getMethod(), "exec"); + Assert.assertEquals(sshActions.get(1).getInstanceId(), "0:i-123456780"); + + Assert.assertEquals(sshActions.get(2).getMethod(), "put"); + Assert.assertEquals(sshActions.get(2).getInstanceId(), "1:i-123456781"); + Assert.assertTrue(sshActions.get(2).getContents().contains(key)); + + Assert.assertEquals(sshActions.get(3).getMethod(), "exec"); + Assert.assertEquals(sshActions.get(3).getInstanceId(), "1:i-123456781"); + } + + @Test + public void testShutdownInstance() { + String key = "ShutdownInstance"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + + checkNotifications(ctx, key); + + List terminated = ctx.terminated(); + Assert.assertEquals(terminated.size(), 2); + Assert.assertEquals(terminated.get(0), "0:i-123456780"); + Assert.assertEquals(terminated.get(1), "1:i-123456781"); + } + + @Test + public void testBlockAllNetworkTraffic() { + String key = "BlockAllNetworkTraffic"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + + checkNotifications(ctx, key); + + List cloudActions = ctx.getCloudActions(); + Assert.assertEquals(cloudActions.size(), 3); + Assert.assertEquals(cloudActions.get(0), "createSecurityGroup:0:i-123456780:blocked-network"); + Assert.assertEquals(cloudActions.get(1), "setInstanceSecurityGroups:0:i-123456780:sg-1"); + Assert.assertEquals(cloudActions.get(2), "setInstanceSecurityGroups:1:i-123456781:sg-1"); + } + + @Test + public void testDetachVolumes() { + String key = "DetachVolumes"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + + checkNotifications(ctx, key); + + List cloudActions = ctx.getCloudActions(); + Assert.assertEquals(cloudActions.size(), 4); + Assert.assertEquals(cloudActions.get(0), "detach:0:i-123456780:volume-1"); + Assert.assertEquals(cloudActions.get(1), "detach:0:i-123456780:volume-2"); + Assert.assertEquals(cloudActions.get(2), "detach:1:i-123456781:volume-1"); + Assert.assertEquals(cloudActions.get(3), "detach:1:i-123456781:volume-2"); + } + + @Test + public void testBurnCpu() { + String key = "BurnCpu"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testBurnIo() { + String key = "BurnIO"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testBurnIoWithoutBurnMoney() { + String key = "BurnIO"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key, false); + + checkSelected(ctx); + + List notifications = ctx.getGloballyNotifiedList(); + Assert.assertEquals(notifications.size(), 0); + + List sshActions = ctx.getSshActions(); + Assert.assertEquals(sshActions.size(), 0); + } + + @Test + public void testFillDisk() { + String key = "FillDisk"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testFillDiskWithoutBurnMoney() { + String key = "FillDisk"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key, false); + + checkSelected(ctx); + + List notifications = ctx.getGloballyNotifiedList(); + Assert.assertEquals(notifications.size(), 0); + + List sshActions = ctx.getSshActions(); + Assert.assertEquals(sshActions.size(), 0); + } + + + @Test + public void testFailDns() { + String key = "FailDns"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testFailDynamoDb() { + String key = "FailDynamoDb"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testFailEc2() { + String key = "FailEc2"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testFailS3() { + String key = "FailS3"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testKillProcess() { + String key = "KillProcesses"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testNetworkCorruption() { + String key = "NetworkCorruption"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testNetworkLatency() { + String key = "NetworkLatency"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testNetworkLoss() { + String key = "NetworkLoss"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + + @Test + public void testNullRoute() { + String key = "NullRoute"; + + TestChaosMonkeyContext ctx = runChaosMonkey(key); + + checkSelected(ctx); + checkNotifications(ctx, key); + checkSshActions(ctx, key); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java index e38ccf8a..f60e95c9 100644 --- a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java @@ -30,9 +30,18 @@ import java.util.Map; import java.util.Properties; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.domain.ExecChannel; +import org.jclouds.compute.domain.ExecResponse; +import org.jclouds.domain.LoginCredentials; +import org.jclouds.io.Payload; +import org.jclouds.ssh.SshClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.netflix.simianarmy.CloudClient; import com.netflix.simianarmy.MonkeyConfiguration; import com.netflix.simianarmy.TestMonkeyContext; @@ -45,8 +54,12 @@ public class TestChaosMonkeyContext extends TestMonkeyContext implements ChaosMo private final BasicConfiguration cfg; public TestChaosMonkeyContext() { + this(new Properties()); + } + + protected TestChaosMonkeyContext(Properties properties) { super(ChaosMonkey.Type.CHAOS); - cfg = new BasicConfiguration(new Properties()); + cfg = new BasicConfiguration(properties); } public TestChaosMonkeyContext(String propFile) { @@ -158,8 +171,8 @@ public List groups(String... names) { if (ig == null) { continue; } - for (String instanceId : terminated) { - // Remove terminated instances from crawler list + for (String instanceId : selected) { + // Remove selected instances from crawler list TestInstanceGroup testIg = (TestInstanceGroup) ig; testIg.deleteInstance(instanceId); } @@ -169,6 +182,7 @@ public List groups(String... names) { } }; } + private final List selectedOn = new LinkedList(); public List selectedOn() { @@ -181,17 +195,23 @@ public ChaosInstanceSelector chaosInstanceSelector() { @Override public Collection select(InstanceGroup group, double probability) { selectedOn.add(group); - return super.select(group, probability); + Collection instances = super.select(group, probability); + selected.addAll(instances); + return instances; } }; } private final List terminated = new LinkedList(); + private final List selected = Lists.newArrayList(); + private final List cloudActions = Lists.newArrayList(); public List terminated() { return terminated; } + final Map securityGroupNames = Maps.newHashMap(); + @Override public CloudClient cloudClient() { return new CloudClient() { @@ -223,11 +243,180 @@ public void deleteImage(String imageId) { @Override public void deleteLaunchConfiguration(String launchConfigName) { } + + @Override + public List listAttachedVolumes(String instanceId, boolean includeRoot) { + List volumes = Lists.newArrayList(); + if (includeRoot) { + volumes.add("volume-0"); + } + volumes.add("volume-1"); + volumes.add("volume-2"); + return volumes; + } + + @Override + public void detachVolume(String instanceId, String volumeId, boolean force) { + cloudActions.add("detach:" + instanceId + ":" + volumeId); + } + + @Override + public ComputeService getJcloudsComputeService() { + throw new UnsupportedOperationException(); + } + + @Override + public String getJcloudsId(String instanceId) { + throw new UnsupportedOperationException(); + } + + @Override + public SshClient connectSsh(String instanceId, LoginCredentials credentials) { + return new MockSshClient(instanceId, credentials); + } + + @Override + public String findSecurityGroup(String instanceId, String groupName) { + return securityGroupNames.get(groupName); + } + + @Override + public String createSecurityGroup(String instanceId, String groupName, String description) { + String id = "sg-" + (securityGroupNames.size() + 1); + securityGroupNames.put(groupName, id); + cloudActions.add("createSecurityGroup:" + instanceId + ":" + groupName); + return id; + } + + @Override + public boolean canChangeInstanceSecurityGroups(String instanceId) { + return true; + } + + @Override + public void setInstanceSecurityGroups(String instanceId, List groupIds) { + cloudActions.add("setInstanceSecurityGroups:" + instanceId + ":" + Joiner.on(',').join(groupIds)); + } }; } - private int groupNotified = 0; - private int globallyNotified = 0; + private final List sshActions = Lists.newArrayList(); + + public static class SshAction { + private String instanceId; + private String method; + private String path; + private String contents; + private String command; + + public String getInstanceId() { + return instanceId; + } + + public String getMethod() { + return method; + } + + public String getPath() { + return path; + } + + public String getContents() { + return contents; + } + + public String getCommand() { + return command; + } + } + + private class MockSshClient implements SshClient { + private final String instanceId; + private final LoginCredentials credentials; + + public MockSshClient(String instanceId, LoginCredentials credentials) { + this.instanceId = instanceId; + this.credentials = credentials; + } + + @Override + public String getUsername() { + return credentials.getUser(); + } + + @Override + public String getHostAddress() { + throw new UnsupportedOperationException(); + } + + @Override + public void put(String path, Payload contents) { + throw new UnsupportedOperationException(); + } + + @Override + public Payload get(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public ExecResponse exec(String command) { + SshAction action = new SshAction(); + action.method = "exec"; + action.instanceId = instanceId; + action.command = command; + sshActions.add(action); + + String output = ""; + String error = ""; + int exitStatus = 0; + return new ExecResponse(output, error, exitStatus); + } + + @Override + public ExecChannel execChannel(String command) { + throw new UnsupportedOperationException(); + } + + @Override + public void connect() { + } + + @Override + public void disconnect() { + } + + @Override + public void put(String path, String contents) { + SshAction action = new SshAction(); + action.method = "put"; + action.instanceId = instanceId; + action.path = path; + action.contents = contents; + sshActions.add(action); + } + } + + private List groupNotified = Lists.newArrayList(); + private List globallyNotified = Lists.newArrayList(); + + static class Notification { + private final String instance; + private final ChaosType chaosType; + + public Notification(String instance, ChaosType chaosType) { + this.instance = instance; + this.chaosType = chaosType; + } + + public String getInstance() { + return instance; + } + + public ChaosType getChaosType() { + return chaosType; + } + } @Override public ChaosEmailNotifier chaosEmailNotifier() { @@ -248,23 +437,38 @@ public String buildEmailSubject(String to) { } @Override - public void sendTerminationNotification(InstanceGroup group, String instance) { - groupNotified++; + public void sendTerminationNotification(InstanceGroup group, String instance, ChaosType chaosType) { + groupNotified.add(new Notification(instance, chaosType)); } @Override - public void sendTerminationGlobalNotification(InstanceGroup group, String instance) { - globallyNotified++; + public void sendTerminationGlobalNotification(InstanceGroup group, String instance, ChaosType chaosType) { + globallyNotified.add(new Notification(instance, chaosType)); } }; } public int getNotified() { - return groupNotified; + return groupNotified.size(); } public int getGloballyNotified() { + return globallyNotified.size(); + } + + public List getNotifiedList() { + return groupNotified; + } + + public List getGloballyNotifiedList() { return globallyNotified; } + public List getSshActions() { + return sshActions; + } + + public List getCloudActions() { + return cloudActions; + } }