Netflix · michaelnflx · Jul 16, 2013 · Jun 26, 2013 · Jul 11, 2013 · Jul 12, 2013
diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosEmailNotifier.java
@@ -20,6 +20,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,16 +58,40 @@ public BasicChaosEmailNotifier(MonkeyConfiguration cfg, AmazonSimpleEmailService
     }
 
     /**
-     * Sends an email notification for a termination of instance.
+     * Sends an email notification for a termination of instance to a global
+     * email address.
+     * @param group the instance group
+     * @param instanceId the instance id
+     */
+    @Override
+    public void sendTerminationGlobalNotification(InstanceGroup group, String instanceId) {
+        String to = cfg.getStr("simianarmy.chaos.notification.global.receiverEmail");
+
+        if (StringUtils.isBlank(to)) {
+            LOGGER.warn("Global email address was not set, but global email notification was enabled!");
+            return;
+        }
+
+        String body = String.format("Instance %s of %s %s is being terminated by Chaos monkey.",
+                    instanceId, group.type(), group.name());
+        String subject = buildEmailSubject(to);
+        LOGGER.info("sending termination notification to global email address {}", to);
+        sendEmail(to, subject, body);
+    }
+
+    /**
+     * Sends an email notification for a termination of instance to the group
+     * owner's email address.
      * @param group the instance group
      * @param instanceId the instance id
      */
     @Override
     public void sendTerminationNotification(InstanceGroup group, String instanceId) {
         String to = getOwnerEmail(group);
-        String subject = buildEmailSubject(to);
         String body = String.format("Instance %s of %s %s is being terminated by Chaos monkey.",
-                instanceId, group.type(), group.name());
+                    instanceId, group.type(), group.name());
+        String subject = buildEmailSubject(to);
+        LOGGER.info("sending termination notification to group owner email address {}", to);
         sendEmail(to, subject, body);
     }
 
@@ -109,4 +134,4 @@ public String getSourceAddress(String to) {
         }
         return sourceEmail;
     }
-}
+}
diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java
@@ -372,19 +372,20 @@ protected boolean isMaxTerminationCountExceeded(InstanceGroup group) {
 
     @Override
     public void sendTerminationNotification(InstanceGroup group, String instance) {
-        String prop = String.format("%s%s.%s.notification.enabled", NS, group.type(), group.name());
-        if (!cfg.getBoolOrElse(prop, false)) {
-            LOGGER.debug(String.format("Group %s [type %s] does not turn on termination notification, "
-                    + "set %s=true to enable it.",
-                    group.name(), group.type(), prop));
-            return;
-        }
+        String propEmailGlobalEnabled = "simianarmy.chaos.notification.global.enabled";
+        String propEmailGroupEnabled = String.format("%s%s.%s.notification.enabled", NS, group.type(), group.name());
+
         ChaosEmailNotifier notifier = context().chaosEmailNotifier();
         if (notifier == null) {
             String msg = "Chaos email notifier is not set.";
             LOGGER.error(msg);
             throw new RuntimeException(msg);
         }
-        notifier.sendTerminationNotification(group, instance);
+        if (cfg.getBoolOrElse(propEmailGroupEnabled, false)) {
+            notifier.sendTerminationNotification(group, instance);
+        }
+        if (cfg.getBoolOrElse(propEmailGlobalEnabled, false)) {
+            notifier.sendTerminationGlobalNotification(group, instance);
+        }
     }
-}
+}
diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/CloudFormationChaosMonkey.java
@@ -57,6 +57,15 @@ protected long getLastOptInMilliseconds(InstanceGroup group) {
         return super.getLastOptInMilliseconds(noSuffixGroup);
     }
 
+    /**
+     * Handle email notifications for no suffix instance groups.
+     */
+    @Override
+    public void sendTerminationNotification(InstanceGroup group, String instance) {
+        InstanceGroup noSuffixGroup = noSuffixInstanceGroup(group);
+        super.sendTerminationNotification(noSuffixGroup, instance);
+    }
+
     /**
      * Return a copy of the instance group removing the randomly generated suffix from
      * its name.

diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosEmailNotifier.java
@@ -37,10 +37,19 @@ public ChaosEmailNotifier(AmazonSimpleEmailServiceClient sesClient) {
     }
 
     /**
-     * Sends an email notification for a termination of instance.
+     * Sends an email notification for a termination of instance to group
+     * owner's email address.
      * @param group the instance group
      * @param instance the instance id
      */
     public abstract void sendTerminationNotification(InstanceGroup group, String instance);
 
+    /**
+     * Sends an email notification for a termination of instance to a global
+     * email address.
+     * @param group the instance group
+     * @param instance the instance id
+     */
+    public abstract void sendTerminationGlobalNotification(InstanceGroup group, String instance);
+
 }
diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java
@@ -394,6 +394,18 @@ public void testNotificationEnabled() {
         Assert.assertEquals(ctx.getNotified(), 2);
     }
 
+    @Test
+    public void testGlobalNotificationEnabled() {
+        TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("globalNotificationEnabled.properties");
+        ChaosMonkey chaos = new BasicChaosMonkey(ctx);
+        chaos.start();
+        chaos.stop();
+        Assert.assertEquals(ctx.selectedOn().size(), 4);
+        Assert.assertEquals(ctx.terminated().size(), 4);
+        Assert.assertEquals(ctx.getNotified(), 1);
+        Assert.assertEquals(ctx.getGloballyNotified(), 4);
+    }
+
     private void terminateOnDemand(TestChaosMonkeyContext ctx, String groupType, String groupName) {
         String input = String.format("{\"eventType\":\"CHAOS_TERMINATION\",\"groupType\":\"%s\",\"groupName\":\"%s\"}",
                 groupType, groupName);

diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestCloudFormationChaosMonkey.java
@@ -88,5 +88,6 @@ public void testCloudFormationChaosMonkeyIntegration() {
         chaos.stop();
         Assert.assertEquals(ctx.selectedOn().size(), 1);
         Assert.assertEquals(ctx.terminated().size(), 1);
+        Assert.assertEquals(ctx.getNotified(), 1);
     }
 }
diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java
@@ -226,7 +226,8 @@ public void deleteLaunchConfiguration(String launchConfigName) {
         };
     }
 
-    private int notified = 0;
+    private int groupNotified = 0;
+    private int globallyNotified = 0;
 
     @Override
     public ChaosEmailNotifier chaosEmailNotifier() {
@@ -248,21 +249,22 @@ public String buildEmailSubject(String to) {
 
             @Override
             public void sendTerminationNotification(InstanceGroup group, String instance) {
-                String prop = String.format("simianarmy.chaos.%s.%s.notification.enabled",
-                        group.type(), group.name());
-                if (!cfg.getBoolOrElse(prop, false)) {
-                    LOGGER.debug(String.format("Group %s [type %s] does not turn on termination notification, "
-                            + "set %s=true to enable it.",
-                            group.name(), group.type(), prop));
-                    return;
-                }
-                notified++;
+                groupNotified++;
+            }
+
+            @Override
+            public void sendTerminationGlobalNotification(InstanceGroup group, String instance) {
+                globallyNotified++;
             }
         };
     }
 
     public int getNotified() {
-        return notified;
+        return groupNotified;
+    }
+
+    public int getGloballyNotified() {
+        return globallyNotified;
     }
 
 }
diff --git a/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties b/src/test/resources/com/netflix/simianarmy/chaos/cloudformation.properties
@@ -3,4 +3,7 @@ simianarmy.chaos.leashed = false
 simianarmy.chaos.TYPE_D.new-group-TestGroup1.enabled = true
 simianarmy.chaos.TYPE_D.new-group-TestGroup1.probability = 1.0
 simianarmy.chaos.TYPE_D.new-group-TestGroup1.maxTerminationsPerDay = 100
-simianarmy.chaos.TYPE_D.new-group-TestGroup1.lastOptInTimeInMilliseconds=2000
+simianarmy.chaos.TYPE_D.new-group-TestGroup1.lastOptInTimeInMilliseconds=2000
+
+simianarmy.chaos.TYPE_D.new-group-TestGroup1.notification.enabled = true
+simianarmy.chaos.TYPE_D.new-group-TestGroup1.ownerEmail = foo@bar.com
diff --git a/src/test/resources/com/netflix/simianarmy/chaos/globalNotificationEnabled.properties b/src/test/resources/com/netflix/simianarmy/chaos/globalNotificationEnabled.properties
@@ -0,0 +1,8 @@
+simianarmy.chaos.enabled = true
+simianarmy.chaos.leashed = false
+simianarmy.chaos.TYPE_A.enabled = true
+simianarmy.chaos.TYPE_B.enabled = true
+simianarmy.chaos.TYPE_A.name0.notification.enabled = false
+simianarmy.chaos.TYPE_A.name1.notification.enabled = true
+simianarmy.chaos.notification.global.enabled = true
+simianarmy.chaos.notification.global.receiverEmail = test@email.com