diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java index 3c783290..29f10b7d 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.List; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,16 +58,40 @@ public BasicChaosEmailNotifier(MonkeyConfiguration cfg, AmazonSimpleEmailService } /** - * Sends an email notification for a termination of instance. + * Sends an email notification for a termination of instance to a global + * email address. + * @param group the instance group + * @param instanceId the instance id + */ + @Override + public void sendTerminationGlobalNotification(InstanceGroup group, String instanceId) { + String to = cfg.getStr("simianarmy.chaos.notification.global.receiverEmail"); + + if (StringUtils.isBlank(to)) { + LOGGER.warn("Global email address was not set, but global email notification was enabled!"); + return; + } + + String body = String.format("Instance %s of %s %s is being terminated by Chaos monkey.", + instanceId, group.type(), group.name()); + String subject = buildEmailSubject(to); + LOGGER.info("sending termination notification to global email address {}", to); + sendEmail(to, subject, body); + } + + /** + * Sends an email notification for a termination of instance to the group + * owner's email address. * @param group the instance group * @param instanceId the instance id */ @Override public void sendTerminationNotification(InstanceGroup group, String instanceId) { String to = getOwnerEmail(group); - String subject = buildEmailSubject(to); String body = String.format("Instance %s of %s %s is being terminated by Chaos monkey.", - instanceId, group.type(), group.name()); + instanceId, group.type(), group.name()); + String subject = buildEmailSubject(to); + LOGGER.info("sending termination notification to group owner email address {}", to); sendEmail(to, subject, body); } @@ -109,4 +134,4 @@ public String getSourceAddress(String to) { } return sourceEmail; } -} +} \ No newline at end of file diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java index 31582151..0f8b8b2c 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java @@ -372,19 +372,20 @@ protected boolean isMaxTerminationCountExceeded(InstanceGroup group) { @Override public void sendTerminationNotification(InstanceGroup group, String instance) { - String prop = String.format("%s%s.%s.notification.enabled", NS, group.type(), group.name()); - if (!cfg.getBoolOrElse(prop, false)) { - LOGGER.debug(String.format("Group %s [type %s] does not turn on termination notification, " - + "set %s=true to enable it.", - group.name(), group.type(), prop)); - return; - } + String propEmailGlobalEnabled = "simianarmy.chaos.notification.global.enabled"; + String propEmailGroupEnabled = String.format("%s%s.%s.notification.enabled", NS, group.type(), group.name()); + ChaosEmailNotifier notifier = context().chaosEmailNotifier(); if (notifier == null) { String msg = "Chaos email notifier is not set."; LOGGER.error(msg); throw new RuntimeException(msg); } - notifier.sendTerminationNotification(group, instance); + if (cfg.getBoolOrElse(propEmailGroupEnabled, false)) { + notifier.sendTerminationNotification(group, instance); + } + if (cfg.getBoolOrElse(propEmailGlobalEnabled, false)) { + notifier.sendTerminationGlobalNotification(group, instance); + } } -} +} \ No newline at end of file diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java index 179b4a04..4982e5df 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java @@ -57,6 +57,15 @@ protected long getLastOptInMilliseconds(InstanceGroup group) { return super.getLastOptInMilliseconds(noSuffixGroup); } + /** + * Handle email notifications for no suffix instance groups. + */ + @Override + public void sendTerminationNotification(InstanceGroup group, String instance) { + InstanceGroup noSuffixGroup = noSuffixInstanceGroup(group); + super.sendTerminationNotification(noSuffixGroup, instance); + } + /** * Return a copy of the instance group removing the randomly generated suffix from * its name. diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java index 7c43a225..b74e1f1e 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java @@ -37,10 +37,19 @@ public ChaosEmailNotifier(AmazonSimpleEmailServiceClient sesClient) { } /** - * Sends an email notification for a termination of instance. + * Sends an email notification for a termination of instance to group + * owner's email address. * @param group the instance group * @param instance the instance id */ public abstract void sendTerminationNotification(InstanceGroup group, String instance); + /** + * Sends an email notification for a termination of instance to a global + * email address. + * @param group the instance group + * @param instance the instance id + */ + public abstract void sendTerminationGlobalNotification(InstanceGroup group, String instance); + } diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java index 206a7096..17baf684 100644 --- a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java +++ b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java @@ -394,6 +394,18 @@ public void testNotificationEnabled() { Assert.assertEquals(ctx.getNotified(), 2); } + @Test + public void testGlobalNotificationEnabled() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("globalNotificationEnabled.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 4); + Assert.assertEquals(ctx.terminated().size(), 4); + Assert.assertEquals(ctx.getNotified(), 1); + Assert.assertEquals(ctx.getGloballyNotified(), 4); + } + private void terminateOnDemand(TestChaosMonkeyContext ctx, String groupType, String groupName) { String input = String.format("{\"eventType\":\"CHAOS_TERMINATION\",\"groupType\":\"%s\",\"groupName\":\"%s\"}", groupType, groupName); diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java index 38946076..26b3b1c5 100644 --- a/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java +++ b/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java @@ -88,5 +88,6 @@ public void testCloudFormationChaosMonkeyIntegration() { chaos.stop(); Assert.assertEquals(ctx.selectedOn().size(), 1); Assert.assertEquals(ctx.terminated().size(), 1); + Assert.assertEquals(ctx.getNotified(), 1); } } diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java index 695ab785..e38ccf8a 100644 --- a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java @@ -226,7 +226,8 @@ public void deleteLaunchConfiguration(String launchConfigName) { }; } - private int notified = 0; + private int groupNotified = 0; + private int globallyNotified = 0; @Override public ChaosEmailNotifier chaosEmailNotifier() { @@ -248,21 +249,22 @@ public String buildEmailSubject(String to) { @Override public void sendTerminationNotification(InstanceGroup group, String instance) { - String prop = String.format("simianarmy.chaos.%s.%s.notification.enabled", - group.type(), group.name()); - if (!cfg.getBoolOrElse(prop, false)) { - LOGGER.debug(String.format("Group %s [type %s] does not turn on termination notification, " - + "set %s=true to enable it.", - group.name(), group.type(), prop)); - return; - } - notified++; + groupNotified++; + } + + @Override + public void sendTerminationGlobalNotification(InstanceGroup group, String instance) { + globallyNotified++; } }; } public int getNotified() { - return notified; + return groupNotified; + } + + public int getGloballyNotified() { + return globallyNotified; } } diff --git a/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties b/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties index 8a9d088b..799f4cfe 100644 --- a/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties +++ b/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties @@ -3,4 +3,7 @@ simianarmy.chaos.leashed = false simianarmy.chaos.TYPE_D.new-group-TestGroup1.enabled = true simianarmy.chaos.TYPE_D.new-group-TestGroup1.probability = 1.0 simianarmy.chaos.TYPE_D.new-group-TestGroup1.maxTerminationsPerDay = 100 -simianarmy.chaos.TYPE_D.new-group-TestGroup1.lastOptInTimeInMilliseconds=2000 \ No newline at end of file +simianarmy.chaos.TYPE_D.new-group-TestGroup1.lastOptInTimeInMilliseconds=2000 + +simianarmy.chaos.TYPE_D.new-group-TestGroup1.notification.enabled = true +simianarmy.chaos.TYPE_D.new-group-TestGroup1.ownerEmail = foo@bar.com diff --git a/src/test/resources/com/netflix/simianarmy/chaos/globalNotificationEnabled.properties b/src/test/resources/com/netflix/simianarmy/chaos/globalNotificationEnabled.properties new file mode 100644 index 00000000..e7f7e3fa --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/globalNotificationEnabled.properties @@ -0,0 +1,8 @@ +simianarmy.chaos.enabled = true +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_A.enabled = true +simianarmy.chaos.TYPE_B.enabled = true +simianarmy.chaos.TYPE_A.name0.notification.enabled = false +simianarmy.chaos.TYPE_A.name1.notification.enabled = true +simianarmy.chaos.notification.global.enabled = true +simianarmy.chaos.notification.global.receiverEmail = test@email.com \ No newline at end of file