From 862035c53b5b1cd3f714f6cf6ebb659a9bdbf107 Mon Sep 17 00:00:00 2001 From: michaelf Date: Wed, 15 Aug 2012 15:56:52 -0700 Subject: [PATCH 1/3] Support Maximum Termination Per Period --- .../netflix/simianarmy/MonkeyRecorder.java | 6 +- .../basic/chaos/BasicChaosMonkey.java | 82 ++++++++++++----- .../chaos/ChaosInstanceSelector.java | 13 +-- .../netflix/simianarmy/chaos/ChaosMonkey.java | 23 ++++- src/main/resources/simianarmy.properties | 6 +- .../basic/chaos/TestBasicChaosMonkey.java | 89 +++++++++++++++++-- .../chaos/TestChaosMonkeyContext.java | 53 ++++++----- ...erminationPerDayAsBiggerThanOne.properties | 6 ++ .../terminationPerDayAsNegative.properties | 5 ++ .../chaos/terminationPerDayAsOne.properties | 5 ++ ...rminationPerDayAsSmallerThanOne.properties | 6 ++ .../terminationPerDayAsVerySmall.properties | 5 ++ .../chaos/terminationPerDayAsZero.properties | 6 ++ 13 files changed, 244 insertions(+), 61 deletions(-) create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsBiggerThanOne.properties create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsNegative.properties create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsOne.properties create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsSmallerThanOne.properties create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsVerySmall.properties create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsZero.properties diff --git a/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java b/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java index cf114459..6881e632 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java @@ -17,9 +17,9 @@ */ package com.netflix.simianarmy; -import java.util.Map; -import java.util.List; import java.util.Date; +import java.util.List; +import java.util.Map; /** * The Interface MonkeyRecorder. This is use to store and find events in some datastore. @@ -102,7 +102,7 @@ public interface Event { * @param eventType * the event type * @param region - * the region the event occured + * the region the event occurred * @param id * the id * @return the event diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java index 05e40c63..7c1889bc 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java @@ -17,21 +17,22 @@ */ package com.netflix.simianarmy.basic.chaos; -import com.netflix.simianarmy.chaos.ChaosMonkey; -import com.netflix.simianarmy.MonkeyConfiguration; -import com.netflix.simianarmy.MonkeyRecorder.Event; -import com.netflix.simianarmy.NotFoundException; -import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; - -import java.util.Map; +import java.util.Calendar; +import java.util.Date; import java.util.HashMap; import java.util.List; -import java.util.Calendar; +import java.util.Map; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.NotFoundException; +import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; +import com.netflix.simianarmy.chaos.ChaosMonkey; + /** * The Class BasicChaosMonkey. */ @@ -44,10 +45,13 @@ public class BasicChaosMonkey extends ChaosMonkey { private static final String NS = "simianarmy.chaos."; /** The cfg. */ - private MonkeyConfiguration cfg; + private final MonkeyConfiguration cfg; /** The runs per day. */ - private long runsPerDay; + private final long runsPerDay; + + /** The minimum value of the maxTerminationCountPerday property to be considered non-zero. **/ + private static final double MIN_MAX_TERMINATION_COUNT_PER_DAY = 0.001; /** * Instantiates a new basic chaos monkey. @@ -71,6 +75,7 @@ public BasicChaosMonkey(ChaosMonkey.Context ctx) { } /** {@inheritDoc} */ + @Override public void doMonkeyBusiness() { cfg.reload(); String prop = NS + "enabled"; @@ -83,6 +88,9 @@ public void doMonkeyBusiness() { prop = NS + group.type() + "." + group.name() + ".enabled"; String defaultProp = NS + group.type(); if (cfg.getBoolOrElse(prop, cfg.getBool(defaultProp + ".enabled"))) { + if (isMaxTerminationCountExceeded(group)) { + continue; + } String probProp = NS + group.type() + "." + group.name() + ".probability"; double prob = cfg.getNumOrElse(probProp, cfg.getNumOrElse(defaultProp + ".probability", 1.0)); LOGGER.info("Group {} [type {}] enabled [prob {}]", new Object[] {group.name(), group.type(), prob}); @@ -93,11 +101,6 @@ public void doMonkeyBusiness() { LOGGER.info("leashed ChaosMonkey prevented from killing {} from group {} [{}], set {}=false", new Object[] {inst, group.name(), group.type(), prop}); } else { - if (hasPreviousTerminations(group)) { - LOGGER.info("ChaosMonkey takes pity on group {} [{}] since it was attacked ealier today", - group.name(), group.type()); - continue; - } try { recordTermination(group, inst); context().cloudClient().terminateInstance(inst); @@ -132,26 +135,63 @@ protected void handleTerminationError(String instance, Throwable e) { } /** {@inheritDoc} */ + @Override + @Deprecated public boolean hasPreviousTerminations(InstanceGroup group) { - Map query = new HashMap(); - query.put("groupType", group.type().name()); - query.put("groupName", group.name()); Calendar today = Calendar.getInstance(); // set to midnight today.set(Calendar.HOUR_OF_DAY, 0); today.set(Calendar.MINUTE, 0); today.set(Calendar.SECOND, 0); today.set(Calendar.MILLISECOND, 0); - List evts = context().recorder().findEvents(Type.CHAOS, EventTypes.CHAOS_TERMINATION, query, - today.getTime()); - return !evts.isEmpty(); + return getPreviousTerminationCount(group, today.getTime()) != 0; } /** {@inheritDoc} */ + @Override public void recordTermination(InstanceGroup group, String instance) { Event evt = context().recorder().newEvent(Type.CHAOS, EventTypes.CHAOS_TERMINATION, group.region(), instance); evt.addField("groupType", group.type().name()); evt.addField("groupName", group.name()); context().recorder().recordEvent(evt); } + + /** {@inheritDoc} */ + @Override + public int getPreviousTerminationCount(InstanceGroup group, Date after) { + Map query = new HashMap(); + query.put("groupType", group.type().name()); + query.put("groupName", group.name()); + List evts = context().recorder().findEvents(Type.CHAOS, EventTypes.CHAOS_TERMINATION, query, after); + return evts.size(); + } + + private boolean isMaxTerminationCountExceeded(InstanceGroup group) { + String prop = NS + group.type() + "." + group.name() + ".maxTerminationsPerDay"; + double maxTerminationsPerDay = cfg.getNumOrElse(prop, 1.0); + if (maxTerminationsPerDay <= MIN_MAX_TERMINATION_COUNT_PER_DAY) { + LOGGER.info("ChaosMonkey is configured to not allow any killing from group {} [{}] " + + "with max daily count set as {}", + new Object[] {group.name(), group.type(), prop}); + return true; + } else { + int daysBack = 1; + int maxCount = (int) maxTerminationsPerDay; + if (maxTerminationsPerDay < 1.0) { + daysBack = (int) Math.ceil(1 / maxTerminationsPerDay); + maxCount = 1; + } + Calendar after = Calendar.getInstance(); + after.add(Calendar.DATE, -1 * daysBack); + // Check if the group has exceeded the maximum terminations for the last period + int terminationCount = getPreviousTerminationCount(group, after.getTime()); + if (terminationCount >= maxCount) { + LOGGER.info("The count of terminations in the last {} days is {}, equal or greater than" + + " the max count threshold {}", + new Object[] {daysBack, terminationCount, maxCount}); + return true; + } + } + return false; + } } diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosInstanceSelector.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosInstanceSelector.java index adba06fa..492f850e 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosInstanceSelector.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosInstanceSelector.java @@ -25,19 +25,20 @@ public interface ChaosInstanceSelector { /** - * Select. Pick a random instance out of the group with provided probabilty. Chaos will draw a random number and if + * Select. Pick a random instance out of the group with provided probability. Chaos will draw a random number and if * that random number is lower than probability then it will proceed to select an instance (at random) out of the - * group. If the random number is higher than the provide probability then no instance will be selected and + * group. If the random number is higher than the provided probability then no instance will be selected and * null will be returned. * * The probability is the run probability. If Chaos is running hourly between 9am and 3pm with an overall configured - * probabilty of "1.0" then the probabilty provided to this routine would be 1.0/6 (6 hours in 9am-3pm). So the + * probability of "1.0" then the probability provided to this routine would be 1.0/6 (6 hours in 9am-3pm). So the * typical probability here would be .1666. For Chaos to select an instance it will pick a random number between 0 - * and 1. If that random number is less than the .1666 it will proced to select an instance and return it, otherwise - * it will return null. Over 6 runs it is likely that the random number be less than .1666, but it is not certain. + * and 1. If that random number is less than the .1666 it will proceed to select an instance and return it, + * otherwise it will return null. Over 6 runs it is likely that the random number be less than .1666, but it is not + * certain. * * To make Chaos select an instance with 100% certainty it would have to be configured to run only once a day and - * the instance group would have to be configured for "1.0" daily probabilty. + * the instance group would have to be configured for "1.0" daily probability. * * @param group * the group diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java index 7810ed3a..b794e889 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java @@ -17,6 +17,8 @@ */ package com.netflix.simianarmy.chaos; +import java.util.Date; + import com.netflix.simianarmy.Monkey; import com.netflix.simianarmy.MonkeyConfiguration; @@ -53,7 +55,7 @@ public interface Context extends Monkey.Context { } /** The context. */ - private Context ctx; + private final Context ctx; /** * Instantiates a new chaos monkey. @@ -85,28 +87,43 @@ public enum EventTypes { } /** {@inheritDoc} */ + @Override public final Enum type() { return Type.CHAOS; } /** {@inheritDoc} */ + @Override public Context context() { return ctx; } /** {@inheritDoc} */ + @Override public abstract void doMonkeyBusiness(); /** - * Checks for previous terminations. Chaos should probably not continue to beat up an instance group if it has - * already been thrashed today. + * Checks for previous terminations for today from the midnight. Chaos should probably not continue to beat + * up an instance group if it has already been thrashed today. * * @param group * the group * @return true, if successful + * @deprecated Should use getPreviousTerminationCount */ + @Deprecated public abstract boolean hasPreviousTerminations(ChaosCrawler.InstanceGroup group); + /** + * Gets the count of terminations since a specific time. Chaos should probably not continue to beat up an + * instance group if the count exceeds a threshold. + * + * @param group + * the group + * @return true, if successful + */ + public abstract int getPreviousTerminationCount(ChaosCrawler.InstanceGroup group, Date after); + /** * Record termination. This is used to notify system owners of terminations and to record terminations so that Chaos * does not continue to thrash the instance groups on later runs. diff --git a/src/main/resources/simianarmy.properties b/src/main/resources/simianarmy.properties index b36463c2..8bed0b2e 100644 --- a/src/main/resources/simianarmy.properties +++ b/src/main/resources/simianarmy.properties @@ -14,15 +14,15 @@ simianarmy.calendar.timezone = America/Los_Angeles # let chaos run simianarmy.chaos.enabled = true -# dont allow chaos to kill (ie dryrun mode) +# don't allow chaos to kill (ie dryrun mode) simianarmy.chaos.leashed = true # set to "false" for Opt-In behavior, "true" for Opt-Out behavior simianarmy.chaos.ASG.enabled = false -# default probabily for all ASGs +# default probability for all ASGs simianarmy.chaos.ASG.probability = 1.0 # enable a specific ASG -# simianarmy.chaos.ASG..enabled = true +# simianarmy.chaos.ASG..enabled = true # simianarmy.chaos.ASG..probability = 1.0 \ No newline at end of file diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java index 92b01d1b..30fc6b1a 100644 --- a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java +++ b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java @@ -19,13 +19,13 @@ package com.netflix.simianarmy.basic.chaos; import java.util.List; -import com.netflix.simianarmy.chaos.ChaosMonkey; -import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; - -import com.netflix.simianarmy.chaos.TestChaosMonkeyContext; -import org.testng.annotations.Test; import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; +import com.netflix.simianarmy.chaos.ChaosMonkey; +import com.netflix.simianarmy.chaos.TestChaosMonkeyContext; // CHECKSTYLE IGNORE MagicNumberCheck public class TestBasicChaosMonkey { @@ -202,4 +202,83 @@ public void testNoProbabilityByName() { Assert.assertEquals(selectedOn.get(3).name(), "name3"); Assert.assertEquals(terminated.size(), 0); } + + @Test + public void testMaxTerminationCountPerDayAsZero() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsZero.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 0); + Assert.assertEquals(ctx.terminated().size(), 0); + } + + @Test + public void testMaxTerminationCountPerDayAsOne() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsOne.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 1); + Assert.assertEquals(ctx.terminated().size(), 1); + + // Run the chaos the second time will NOT trigger another termination + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 1); + Assert.assertEquals(ctx.terminated().size(), 1); + } + + @Test + public void testMaxTerminationCountPerDayAsBiggerThanOne() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsBiggerThanOne.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 1); + Assert.assertEquals(ctx.terminated().size(), 1); + + // Run the chaos the second time will trigger another termination + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 2); + Assert.assertEquals(ctx.terminated().size(), 2); + } + + @Test + public void testMaxTerminationCountPerDayAsSmallerThanOne() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsSmallerThanOne.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 1); + Assert.assertEquals(ctx.terminated().size(), 1); + + // Run the chaos the second time will NOT trigger another termination + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 1); + Assert.assertEquals(ctx.terminated().size(), 1); + } + + @Test + public void testMaxTerminationCountPerDayAsNegative() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsNegative.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 0); + Assert.assertEquals(ctx.terminated().size(), 0); + } + + @Test + public void testMaxTerminationCountPerDayAsVerySmall() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayAsVerySmall.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 0); + Assert.assertEquals(ctx.terminated().size(), 0); + } + } diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java index 8fcd86c3..c0b322b5 100644 --- a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java @@ -18,27 +18,26 @@ */ package com.netflix.simianarmy.chaos; -import com.netflix.simianarmy.TestMonkeyContext; -import com.netflix.simianarmy.MonkeyConfiguration; -import com.netflix.simianarmy.CloudClient; -import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; -import com.netflix.simianarmy.basic.BasicConfiguration; -import com.netflix.simianarmy.basic.chaos.BasicChaosInstanceSelector; - -import java.util.Properties; import java.io.InputStream; - -import java.util.List; -import java.util.LinkedList; import java.util.Arrays; import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.TestMonkeyContext; +import com.netflix.simianarmy.basic.BasicConfiguration; +import com.netflix.simianarmy.basic.chaos.BasicChaosInstanceSelector; +import com.netflix.simianarmy.chaos.ChaosCrawler.InstanceGroup; + public class TestChaosMonkeyContext extends TestMonkeyContext implements ChaosMonkey.Context { private static final Logger LOGGER = LoggerFactory.getLogger(TestChaosMonkeyContext.class); - private BasicConfiguration cfg; + private final BasicConfiguration cfg; public TestChaosMonkeyContext() { super(ChaosMonkey.Type.CHAOS); @@ -61,6 +60,7 @@ public TestChaosMonkeyContext(String propFile) { cfg = new BasicConfiguration(props); } + @Override public MonkeyConfiguration configuration() { return cfg; } @@ -69,63 +69,74 @@ public static class TestInstanceGroup implements InstanceGroup { private final Enum type; private final String name; private final String region; - private final String instance; + private final String[] instances; - public TestInstanceGroup(Enum type, String name, String region, String instance) { + public TestInstanceGroup(Enum type, String name, String region, String... instances) { this.type = type; this.name = name; this.region = region; - this.instance = instance; + this.instances = instances; } + @Override public Enum type() { return type; } + @Override public String name() { return name; } + @Override public String region() { return region; } + @Override public List instances() { - return Arrays.asList(instance); + return Arrays.asList(instances); } + @Override public void addInstance(String ignored) { } } public enum CrawlerTypes { - TYPE_A, TYPE_B + TYPE_A, TYPE_B, TYPE_C }; + @Override public ChaosCrawler chaosCrawler() { return new ChaosCrawler() { + @Override public EnumSet groupTypes() { return EnumSet.allOf(CrawlerTypes.class); } + @Override public List groups() { InstanceGroup gA0 = new TestInstanceGroup(CrawlerTypes.TYPE_A, "name0", "reg1", "0:i-123456780"); InstanceGroup gA1 = new TestInstanceGroup(CrawlerTypes.TYPE_A, "name1", "reg1", "1:i-123456781"); InstanceGroup gB2 = new TestInstanceGroup(CrawlerTypes.TYPE_B, "name2", "reg1", "2:i-123456782"); InstanceGroup gB3 = new TestInstanceGroup(CrawlerTypes.TYPE_B, "name3", "reg1", "3:i-123456783"); - return Arrays.asList(gA0, gA1, gB2, gB3); + InstanceGroup gC1 = new TestInstanceGroup(CrawlerTypes.TYPE_C, "name4", "reg1", "3:i-123456784", "3:i-123456785"); + return Arrays.asList(gA0, gA1, gB2, gB3, gC1); } }; } - private List selectedOn = new LinkedList(); + private final List selectedOn = new LinkedList(); public List selectedOn() { return selectedOn; } + @Override public ChaosInstanceSelector chaosInstanceSelector() { return new BasicChaosInstanceSelector() { + @Override public String select(InstanceGroup group, double probability) { selectedOn.add(group); return super.select(group, probability); @@ -133,14 +144,16 @@ public String select(InstanceGroup group, double probability) { }; } - private List terminated = new LinkedList(); + private final List terminated = new LinkedList(); public List terminated() { return terminated; } + @Override public CloudClient cloudClient() { return new CloudClient() { + @Override public void terminateInstance(String instanceId) { terminated.add(instanceId); } diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsBiggerThanOne.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsBiggerThanOne.properties new file mode 100644 index 00000000..ff2e8ffd --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsBiggerThanOne.properties @@ -0,0 +1,6 @@ +simianarmy.chaos.enabled = true +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = 2 \ No newline at end of file diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsNegative.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsNegative.properties new file mode 100644 index 00000000..08dcbd21 --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsNegative.properties @@ -0,0 +1,5 @@ +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = -1.0 \ No newline at end of file diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsOne.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsOne.properties new file mode 100644 index 00000000..30d9c6c3 --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsOne.properties @@ -0,0 +1,5 @@ +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = 1.0 \ No newline at end of file diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsSmallerThanOne.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsSmallerThanOne.properties new file mode 100644 index 00000000..84665837 --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsSmallerThanOne.properties @@ -0,0 +1,6 @@ +simianarmy.chaos.enabled = true +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = 0.5 \ No newline at end of file diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsVerySmall.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsVerySmall.properties new file mode 100644 index 00000000..e25c7970 --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsVerySmall.properties @@ -0,0 +1,5 @@ +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = 0.0005 \ No newline at end of file diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsZero.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsZero.properties new file mode 100644 index 00000000..138b054f --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayAsZero.properties @@ -0,0 +1,6 @@ +simianarmy.chaos.enabled = true +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.name4.maxTerminationsPerDay = 0 \ No newline at end of file From 5dff0c6cd0bf574872ee9021792b64432201c13a Mon Sep 17 00:00:00 2001 From: michaelf Date: Wed, 15 Aug 2012 17:02:17 -0700 Subject: [PATCH 2/3] Remove the derecated method --- .../simianarmy/basic/chaos/BasicChaosMonkey.java | 13 ------------- .../com/netflix/simianarmy/chaos/ChaosMonkey.java | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java index 7c1889bc..9e3fbf27 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java @@ -134,19 +134,6 @@ protected void handleTerminationError(String instance, Throwable e) { throw new RuntimeException("failed to terminate instance " + instance, e); } - /** {@inheritDoc} */ - @Override - @Deprecated - public boolean hasPreviousTerminations(InstanceGroup group) { - Calendar today = Calendar.getInstance(); - // set to midnight - today.set(Calendar.HOUR_OF_DAY, 0); - today.set(Calendar.MINUTE, 0); - today.set(Calendar.SECOND, 0); - today.set(Calendar.MILLISECOND, 0); - return getPreviousTerminationCount(group, today.getTime()) != 0; - } - /** {@inheritDoc} */ @Override public void recordTermination(InstanceGroup group, String instance) { diff --git a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java index b794e889..8deb01be 100644 --- a/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/chaos/ChaosMonkey.java @@ -102,18 +102,6 @@ public Context context() { @Override public abstract void doMonkeyBusiness(); - /** - * Checks for previous terminations for today from the midnight. Chaos should probably not continue to beat - * up an instance group if it has already been thrashed today. - * - * @param group - * the group - * @return true, if successful - * @deprecated Should use getPreviousTerminationCount - */ - @Deprecated - public abstract boolean hasPreviousTerminations(ChaosCrawler.InstanceGroup group); - /** * Gets the count of terminations since a specific time. Chaos should probably not continue to beat up an * instance group if the count exceeds a threshold. From 111abfbc930ba9dbbbcbb6724aab728c13e13970 Mon Sep 17 00:00:00 2001 From: michaelf Date: Wed, 15 Aug 2012 17:39:30 -0700 Subject: [PATCH 3/3] Added group level property support --- .../basic/chaos/BasicChaosMonkey.java | 6 ++++-- .../basic/chaos/TestBasicChaosMonkey.java | 18 ++++++++++++++++++ .../chaos/TestChaosMonkeyContext.java | 1 + .../terminationPerDayGroupLevel.properties | 6 ++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayGroupLevel.properties diff --git a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java index 9e3fbf27..a22dac4b 100644 --- a/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/chaos/BasicChaosMonkey.java @@ -154,8 +154,10 @@ public int getPreviousTerminationCount(InstanceGroup group, Date after) { } private boolean isMaxTerminationCountExceeded(InstanceGroup group) { - String prop = NS + group.type() + "." + group.name() + ".maxTerminationsPerDay"; - double maxTerminationsPerDay = cfg.getNumOrElse(prop, 1.0); + String propName = "maxTerminationsPerDay"; + String defaultProp = String.format("%s%s.%s", NS, group.type(), propName); + String prop = String.format("%s%s.%s.%s", NS, group.type(), group.name(), propName); + double maxTerminationsPerDay = cfg.getNumOrElse(prop, cfg.getNumOrElse(defaultProp, 1.0)); if (maxTerminationsPerDay <= MIN_MAX_TERMINATION_COUNT_PER_DAY) { LOGGER.info("ChaosMonkey is configured to not allow any killing from group {} [{}] " + "with max daily count set as {}", diff --git a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java index 30fc6b1a..a5aebeb6 100644 --- a/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java +++ b/src/test/java/com/netflix/simianarmy/basic/chaos/TestBasicChaosMonkey.java @@ -281,4 +281,22 @@ public void testMaxTerminationCountPerDayAsVerySmall() { Assert.assertEquals(ctx.terminated().size(), 0); } + @Test + public void testMaxTerminationCountPerDayGroupLevel() { + TestChaosMonkeyContext ctx = new TestChaosMonkeyContext("terminationPerDayGroupLevel.properties"); + ChaosMonkey chaos = new BasicChaosMonkey(ctx); + + for (int i=1; i<=3; i++) { + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), i); + Assert.assertEquals(ctx.terminated().size(), i); + } + // Run the chaos the second time will NOT trigger another termination + chaos.start(); + chaos.stop(); + Assert.assertEquals(ctx.selectedOn().size(), 3); + Assert.assertEquals(ctx.terminated().size(), 3); + } + } diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java index c0b322b5..e3e7e3fb 100644 --- a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java @@ -122,6 +122,7 @@ public List groups() { InstanceGroup gB2 = new TestInstanceGroup(CrawlerTypes.TYPE_B, "name2", "reg1", "2:i-123456782"); InstanceGroup gB3 = new TestInstanceGroup(CrawlerTypes.TYPE_B, "name3", "reg1", "3:i-123456783"); InstanceGroup gC1 = new TestInstanceGroup(CrawlerTypes.TYPE_C, "name4", "reg1", "3:i-123456784", "3:i-123456785"); + InstanceGroup gC2 = new TestInstanceGroup(CrawlerTypes.TYPE_C, "name5", "reg1", "3:i-123456786", "3:i-123456787"); return Arrays.asList(gA0, gA1, gB2, gB3, gC1); } }; diff --git a/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayGroupLevel.properties b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayGroupLevel.properties new file mode 100644 index 00000000..9674e603 --- /dev/null +++ b/src/test/resources/com/netflix/simianarmy/chaos/terminationPerDayGroupLevel.properties @@ -0,0 +1,6 @@ +simianarmy.chaos.enabled = true +simianarmy.chaos.leashed = false +simianarmy.chaos.TYPE_C.name4.enabled = true +simianarmy.chaos.TYPE_C.name4.probability = 1.0 + +simianarmy.chaos.TYPE_C.maxTerminationsPerDay = 3.0 \ No newline at end of file