From 86d384f1d5328eda312ce44f0b918df1669c744c Mon Sep 17 00:00:00 2001 From: michaelf Date: Wed, 2 Jan 2013 18:24:38 -0800 Subject: [PATCH 1/2] Janitor Monkey Changes --- build.gradle | 10 +- gradle.properties | 2 +- .../simianarmy/AbstractEmailBuilder.java | 83 +++ .../com/netflix/simianarmy/CloudClient.java | 41 +- .../com/netflix/simianarmy/EmailBuilder.java | 28 + .../java/com/netflix/simianarmy/Monkey.java | 14 +- .../netflix/simianarmy/MonkeyCalendar.java | 9 + .../simianarmy/MonkeyEmailNotifier.java | 1 + .../netflix/simianarmy/MonkeyRecorder.java | 2 +- .../com/netflix/simianarmy/MonkeyRunner.java | 16 +- .../java/com/netflix/simianarmy/Resource.java | 385 +++++++++++ .../simianarmy/aws/AWSEmailNotifier.java | 2 +- .../netflix/simianarmy/aws/AWSResource.java | 512 ++++++++++++++ .../simianarmy/aws/AWSResourceType.java | 39 ++ .../simianarmy/aws/SimpleDBRecorder.java | 84 +-- .../simianarmy/aws/janitor/ASGJanitor.java | 64 ++ .../aws/janitor/EBSSnapshotJanitor.java | 64 ++ .../aws/janitor/EBSVolumeJanitor.java | 64 ++ .../aws/janitor/InstanceJanitor.java | 64 ++ .../SimpleDBJanitorResourceTracker.java | 195 ++++++ .../aws/janitor/VolumeTaggingMonkey.java | 307 +++++++++ .../janitor/crawler/ASGJanitorCrawler.java | 181 +++++ .../crawler/AbstractAWSJanitorCrawler.java | 61 ++ .../crawler/EBSSnapshotJanitorCrawler.java | 151 +++++ .../crawler/EBSVolumeJanitorCrawler.java | 117 ++++ .../crawler/InstanceJanitorCrawler.java | 124 ++++ .../rule/asg/ASGInstanceValidator.java | 110 +++ .../aws/janitor/rule/asg/OldEmptyASGRule.java | 129 ++++ .../janitor/rule/asg/SuspendedASGRule.java | 111 ++++ .../rule/instance/OrphanedInstanceRule.java | 123 ++++ .../rule/snapshot/NoGeneratedAMIRule.java | 142 ++++ .../rule/volume/OldDetachedVolumeRule.java | 142 ++++ .../simianarmy/basic/BasicCalendar.java | 63 +- .../basic/BasicChaosMonkeyContext.java | 98 +++ .../simianarmy/basic/BasicContext.java | 135 ---- .../simianarmy/basic/BasicContextShell.java | 218 ------ .../simianarmy/basic/BasicMonkeyServer.java | 28 +- .../basic/BasicSimianArmyContext.java | 296 +++++++++ .../janitor/BasicJanitorEmailBuilder.java | 141 ++++ .../basic/janitor/BasicJanitorMonkey.java | 216 ++++++ .../janitor/BasicJanitorMonkeyContext.java | 367 ++++++++++ .../basic/janitor/BasicJanitorRuleEngine.java | 98 +++ .../BasicVolumeTaggingMonkeyContext.java | 33 + .../simianarmy/client/aws/AWSClient.java | 284 +++++++- .../PropertyBasedTerminationStrategy.java | 16 +- .../client/vsphere/VSphereContext.java | 10 +- .../vsphere/VSphereServiceConnection.java | 12 +- .../simianarmy/janitor/AbstractJanitor.java | 384 +++++++++++ .../netflix/simianarmy/janitor/Janitor.java | 43 ++ .../simianarmy/janitor/JanitorCrawler.java | 62 ++ .../janitor/JanitorEmailBuilder.java | 35 + .../janitor/JanitorEmailNotifier.java | 290 ++++++++ .../simianarmy/janitor/JanitorMonkey.java | 147 ++++ .../janitor/JanitorResourceTracker.java | 54 ++ .../simianarmy/janitor/JanitorRuleEngine.java | 47 ++ .../com/netflix/simianarmy/janitor/Rule.java | 37 ++ .../resources/chaos/ChaosMonkeyResource.java | 2 + .../janitor/JanitorMonkeyResource.java | 153 +++++ src/main/resources/chaos.properties | 44 ++ src/main/resources/client.properties | 2 +- src/main/resources/janitor.properties | 97 +++ src/main/resources/simianarmy.properties | 46 +- src/main/resources/volumeTagging.properties | 18 + .../netflix/simianarmy/TestMonkeyContext.java | 56 +- .../simianarmy/aws/TestSimpleDBRecorder.java | 59 +- .../aws/janitor/TestAWSResource.java | 155 +++++ .../TestSimpleDBJanitorResourceTracker.java | 220 ++++++ .../crawler/TestASGJanitorCrawler.java | 124 ++++ .../TestEBSSnapshotJanitorCrawler.java | 119 ++++ .../crawler/TestEBSVolumeJanitorCrawler.java | 119 ++++ .../crawler/TestInstanceJanitorCrawler.java | 149 +++++ .../aws/janitor/rule/TestMonkeyCalendar.java | 61 ++ .../janitor/rule/asg/TestOldEmptyASGRule.java | 181 +++++ .../rule/asg/TestSuspendedASGRule.java | 182 +++++ .../instance/TestOrphanedInstanceRule.java | 179 +++++ .../rule/snapshot/TestNoGeneratedAMIRule.java | 190 ++++++ .../volume/TestOldDetachedVolumeRule.java | 206 ++++++ .../simianarmy/basic/TestBasicCalendar.java | 99 ++- .../simianarmy/basic/TestBasicContext.java | 13 +- .../basic/TestBasicMonkeyServer.java | 8 +- .../janitor/TestBasicJanitorRuleEngine.java | 106 +++ .../chaos/TestChaosMonkeyContext.java | 16 + .../client/vsphere/TestVSphereContext.java | 2 +- .../janitor/TestAbstractJanitor.java | 628 ++++++++++++++++++ .../chaos/TestChaosMonkeyResource.java | 3 +- src/test/resources/chaos.properties | 1 + src/test/resources/simianarmy.properties | 3 +- 87 files changed, 9153 insertions(+), 579 deletions(-) create mode 100644 src/main/java/com/netflix/simianarmy/AbstractEmailBuilder.java create mode 100644 src/main/java/com/netflix/simianarmy/EmailBuilder.java create mode 100644 src/main/java/com/netflix/simianarmy/Resource.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/AWSResource.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/AWSResourceType.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/ASGJanitor.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/EBSSnapshotJanitor.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/EBSVolumeJanitor.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/InstanceJanitor.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/VolumeTaggingMonkey.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/crawler/ASGJanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/crawler/AbstractAWSJanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSSnapshotJanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSVolumeJanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/crawler/InstanceJanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/ASGInstanceValidator.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/OldEmptyASGRule.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/SuspendedASGRule.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/instance/OrphanedInstanceRule.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/NoGeneratedAMIRule.java create mode 100644 src/main/java/com/netflix/simianarmy/aws/janitor/rule/volume/OldDetachedVolumeRule.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/BasicChaosMonkeyContext.java delete mode 100644 src/main/java/com/netflix/simianarmy/basic/BasicContext.java delete mode 100644 src/main/java/com/netflix/simianarmy/basic/BasicContextShell.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorEmailBuilder.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorRuleEngine.java create mode 100644 src/main/java/com/netflix/simianarmy/basic/janitor/BasicVolumeTaggingMonkeyContext.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/Janitor.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorCrawler.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorEmailBuilder.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/JanitorRuleEngine.java create mode 100644 src/main/java/com/netflix/simianarmy/janitor/Rule.java create mode 100644 src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java create mode 100644 src/main/resources/chaos.properties create mode 100644 src/main/resources/janitor.properties create mode 100644 src/main/resources/volumeTagging.properties create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/TestAWSResource.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/TestSimpleDBJanitorResourceTracker.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestASGJanitorCrawler.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSSnapshotJanitorCrawler.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSVolumeJanitorCrawler.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestInstanceJanitorCrawler.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/TestMonkeyCalendar.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestOldEmptyASGRule.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestSuspendedASGRule.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/instance/TestOrphanedInstanceRule.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/TestNoGeneratedAMIRule.java create mode 100644 src/test/java/com/netflix/simianarmy/aws/janitor/rule/volume/TestOldDetachedVolumeRule.java create mode 100644 src/test/java/com/netflix/simianarmy/basic/janitor/TestBasicJanitorRuleEngine.java create mode 100644 src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java create mode 100644 src/test/resources/chaos.properties diff --git a/build.gradle b/build.gradle index 2123afea..8d158adc 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ apply plugin:'eclipse-wtp' buildscript { repositories { mavenCentral() } - apply from: file('gradle/buildscript.gradle'), to: buildscript + apply from: file('gradle/buildscript.gradle'), to: buildscript } allprojects { @@ -23,7 +23,7 @@ apply plugin: 'jetty' apply plugin: 'eclipse' dependencies { - // for the VMWareClient + // for the VMWareClient compile 'com.cloudbees.thirdparty:vijava:5.0.0' compile 'dom4j:dom4j:1.6.1' @@ -34,10 +34,12 @@ dependencies { compile 'org.slf4j:slf4j-api:1.6.4' compile 'org.codehaus.jackson:jackson-core-asl:1.9.2' compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2' + compile 'com.netflix.eureka:eureka-client:1.1.22' compile('com.amazonaws:aws-java-sdk:1.3.11') { exclude group:'org.codehaus.jackson' } compile 'commons-lang:commons-lang:2.6' + compile 'joda-time:joda-time:2.1' testCompile 'org.testng:testng:6.3.1' testCompile 'org.mockito:mockito-core:1.8.5' @@ -53,12 +55,12 @@ dependencies { } test { - useTestNG() + useTestNG() } tasks.withType(Compile) { options.compilerArgs << "-Xlint" << "-Werror" -} +} import nl.javadude.gradle.plugins.license.License tasks.withType(License).each { licenseTask -> diff --git a/gradle.properties b/gradle.properties index 67f32e54..d70e3778 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=2.3-SNAPSHOT +version=2.4-SNAPSHOT diff --git a/src/main/java/com/netflix/simianarmy/AbstractEmailBuilder.java b/src/main/java/com/netflix/simianarmy/AbstractEmailBuilder.java new file mode 100644 index 00000000..107eff43 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/AbstractEmailBuilder.java @@ -0,0 +1,83 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy; + +/** The abstract email builder. */ +public abstract class AbstractEmailBuilder implements EmailBuilder { + + @Override + public String buildEmailBody(String emailAddress) { + StringBuilder body = new StringBuilder(); + String header = getHeader(); + if (header != null) { + body.append(header); + } + String entryTable = getEntryTable(emailAddress); + if (entryTable != null) { + body.append(entryTable); + } + String footer = getFooter(); + if (footer != null) { + body.append(footer); + } + return body.toString(); + } + + /** + * Gets the header to the email body. + */ + protected abstract String getHeader(); + + /** + * Gets the table of entries in the email body. + * @param emailAddress the email address to notify + * @return the HTML string representing the table for the resources to send to the + * email address + */ + protected abstract String getEntryTable(String emailAddress); + + /** + * Gets the footer of the email body. + */ + protected abstract String getFooter(); + + /** + * Gets the HTML cell in the table of a string value. + * @param value the string to put in the table + * @return the HTML text + */ + protected String getHtmlCell(String value) { + return "" + value + ""; + } + + /** + * Gets the HTML string displaying the table header with the specified column names. + * @param columns the column names for the table + */ + protected String getHtmlTableHeader(String[] columns) { + StringBuilder tableHeader = new StringBuilder(); + tableHeader.append( + ""); + tableHeader.append(""); + for (String col : columns) { + tableHeader.append(getHtmlCell(col)); + } + tableHeader.append(""); + return tableHeader.toString(); + } +} diff --git a/src/main/java/com/netflix/simianarmy/CloudClient.java b/src/main/java/com/netflix/simianarmy/CloudClient.java index ac9b2f95..c651a968 100644 --- a/src/main/java/com/netflix/simianarmy/CloudClient.java +++ b/src/main/java/com/netflix/simianarmy/CloudClient.java @@ -17,6 +17,9 @@ */ package com.netflix.simianarmy; +import java.util.Map; + + /** * The CloudClient interface. This abstractions provides the interface that the monkeys need to interact with * "the cloud". @@ -24,7 +27,7 @@ public interface CloudClient { /** - * Terminate instance. + * Terminates instance. * * @param instanceId * the instance id @@ -34,4 +37,40 @@ public interface CloudClient { * should get a NotFoundException */ void terminateInstance(String instanceId); + + /** + * Deletes an auto scaling group. + * + * @param asgName + * the auto scaling group name + */ + void deleteAutoScalingGroup(String asgName); + + /** + * Deletes a volume. + * + * @param volumeId + * the volume id + */ + void deleteVolume(String volumeId); + + /** + * Deletes a snapshot. + * + * @param snapshotId + * the snapshot id. + */ + void deleteSnapshot(String snapshotId); + + /** + * Adds or overwrites tags for the specified resources. + * + * @param keyValueMap + * the new tags in the form of map from key to value + * + * @param resourceIds + * the list of resource ids + */ + void createTagsForResources(Map keyValueMap, String... resourceIds); + } diff --git a/src/main/java/com/netflix/simianarmy/EmailBuilder.java b/src/main/java/com/netflix/simianarmy/EmailBuilder.java new file mode 100644 index 00000000..4a753031 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/EmailBuilder.java @@ -0,0 +1,28 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy; + +/** Interface for build the email body. */ +public interface EmailBuilder { + /** + * Builds an email body for an email address. + * @param emailAddress the email address to send notification to + * @return the email body + */ + String buildEmailBody(String emailAddress); +} diff --git a/src/main/java/com/netflix/simianarmy/Monkey.java b/src/main/java/com/netflix/simianarmy/Monkey.java index e9fef2b0..b06e00fa 100644 --- a/src/main/java/com/netflix/simianarmy/Monkey.java +++ b/src/main/java/com/netflix/simianarmy/Monkey.java @@ -82,10 +82,16 @@ public interface Context { */ String getEventReport(); + /** + * Configuration. + * + * @return the monkey configuration + */ + MonkeyConfiguration configuration(); } /** The context. */ - private Context ctx; + private final Context ctx; /** * Instantiates a new monkey. @@ -127,7 +133,10 @@ public void run() { try { this.doMonkeyBusiness(); } finally { - LOGGER.info("Reporting what I did...\n" + context().getEventReport()); + String eventReport = context().getEventReport(); + if (eventReport != null) { + LOGGER.info("Reporting what I did...\n" + eventReport); + } } } else { LOGGER.info("Not Time for " + this.type().name() + " Monkey"); @@ -140,6 +149,7 @@ public void run() { public void start() { final Monkey me = this; ctx.scheduler().start(this, new Runnable() { + @Override public void run() { try { me.run(); diff --git a/src/main/java/com/netflix/simianarmy/MonkeyCalendar.java b/src/main/java/com/netflix/simianarmy/MonkeyCalendar.java index fe89f2e9..061212f7 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyCalendar.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyCalendar.java @@ -18,6 +18,7 @@ package com.netflix.simianarmy; import java.util.Calendar; +import java.util.Date; /** * The Interface MonkeyCalendar used to tell if a monkey should be running or now. We only want monkeys to run during @@ -54,4 +55,12 @@ public interface MonkeyCalendar { * @return the calendar */ Calendar now(); + + /** Gets the next business day from the start date after n business days. + * + * @param date the start date + * @param n the number of business days from now + * @return the business day after n business days + */ + Date getBusinessDay(Date date, int n); } diff --git a/src/main/java/com/netflix/simianarmy/MonkeyEmailNotifier.java b/src/main/java/com/netflix/simianarmy/MonkeyEmailNotifier.java index b633d88e..e837e81e 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyEmailNotifier.java @@ -20,6 +20,7 @@ /** The interface for the email notifier used by monkeys. */ public interface MonkeyEmailNotifier { + /** * Determines if a email address is valid. * @param email the email diff --git a/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java b/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java index 14fdb0e7..bcab334e 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyRecorder.java @@ -79,7 +79,7 @@ public interface Event { * * @param name * the name - * @return the string assiciated with that field + * @return the string associated with that field */ String field(String name); diff --git a/src/main/java/com/netflix/simianarmy/MonkeyRunner.java b/src/main/java/com/netflix/simianarmy/MonkeyRunner.java index a534c5a2..bb79876a 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyRunner.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyRunner.java @@ -72,11 +72,11 @@ public void stop() { * The monkey map. Maps the monkey class to the context class that is registered. This is so we can create new * monkeys in factory() that have the same context types as the registered ones. */ - // SUPPRESS CHECKSTYLE LineLength - private Map, Class> monkeyMap = new HashMap, Class>(); + private final Map, Class> monkeyMap = + new HashMap, Class>(); /** The monkeys. */ - private List monkeys = new LinkedList(); + private final List monkeys = new LinkedList(); /** * Gets the registered monkeys. @@ -171,12 +171,20 @@ public void removeMonkey(Class monkeyClass) { * Example: * * 
+<<<<<<< HEAD
+     *         {@code
+     *         MonkeyRunner.getInstance().addMonkey(BasicChaosMonkey.class, BasicMonkeyContext.class);
+     *         // This will actually return a BasicChaosMonkey since that is the only subclass that was registered
+     *         ChaosMonkey monkey = MonkeyRunner.getInstance().factory(ChaosMonkey.class);
+     *}
+=======
      *          {@code
      *          MonkeyRunner.getInstance().addMonkey(BasicChaosMonkey.class, BasicMonkeyContext.class);
      *
      *          // This will actualy return a BasicChaosMonkey since that is the only subclass that was registered
      *          ChaosMonkey monkey = MonkeyRunner.getInstance().factory(ChaosMonkey.class);
      * }
+>>>>>>> fde87ebfbda161188f1dac96e5ea8e34bcc684f6
      *
* * @param @@ -240,7 +248,7 @@ public T factory(Class monkeyClass, Class getFieldToValueMap(); + + /** Adds or sets an additional field with the specified name and value to the resource. + * + * @param fieldName the field name + * @param fieldValue the field value + * @return the resource itself for chaining + */ + Resource setAdditionalField(String fieldName, String fieldValue); + + /** Gets the value of an additional field with the specified name of the resource. + * + * @param fieldName the field name + * @return the field value + */ + String getAdditionalField(String fieldName); + + /** + * Gets all additional field names in the resource. + * @return a collection of names of all additional fields + */ + Collection getAdditionalFieldNames(); + + /** + * Adds a tag with the specified key and value to the resource. + * @param key the key of the tag + * @param value the value of the tag + */ + void setTag(String key, String value); + + /** + * Gets the tag value for a specific key of the resource. + * @param key the key of the tag + * @return the value of the tag + */ + String getTag(String key); + + /** + * Gets all the keys of tags. + * @return collection of keys of all tags + */ + Collection getAllTagKeys(); + + + /** Clone a resource with the exact field values of the current object. + * + * @return the clone of the resource + */ + Resource cloneResource(); +} diff --git a/src/main/java/com/netflix/simianarmy/aws/AWSEmailNotifier.java b/src/main/java/com/netflix/simianarmy/aws/AWSEmailNotifier.java index 67e7fefa..2adfb86f 100644 --- a/src/main/java/com/netflix/simianarmy/aws/AWSEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/aws/AWSEmailNotifier.java @@ -34,7 +34,7 @@ /** * The class implements the monkey email notifier using AWS simple email service - * for sending emails. + * for sending email. */ public abstract class AWSEmailNotifier implements MonkeyEmailNotifier { /** The Constant LOGGER. */ diff --git a/src/main/java/com/netflix/simianarmy/aws/AWSResource.java b/src/main/java/com/netflix/simianarmy/aws/AWSResource.java new file mode 100644 index 00000000..604e1e84 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/AWSResource.java @@ -0,0 +1,512 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws; + +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang.Validate; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import com.netflix.simianarmy.Resource; + +/** + * The class represents general AWS resources that are managed by janitor monkey. + */ +public class AWSResource implements Resource { + private String id; + private Enum resourceType; + private String region; + private String ownerEmail; + private String description; + private String terminationReason; + private CleanupState state; + private Date expectedTerminationTime; + private Date actualTerminationTime; + private Date notificationTime; + private Date launchTime; + private Date markTime; + private boolean optOutOfJanitor; + private String awsResourceState; + + /** The field name for resourceId. **/ + public static final String FIELD_RESOURCE_ID = "resourceId"; + /** The field name for resourceType. **/ + public static final String FIELD_RESOURCE_TYPE = "resourceType"; + /** The field name for region. **/ + public static final String FIELD_REGION = "region"; + /** The field name for owner email. **/ + public static final String FIELD_OWNER_EMAIL = "ownerEmail"; + /** The field name for description. **/ + public static final String FIELD_DESCRIPTION = "description"; + /** The field name for state. **/ + public static final String FIELD_STATE = "state"; + /** The field name for terminationReason. **/ + public static final String FIELD_TERMINATION_REASON = "terminationReason"; + /** The field name for expectedTerminationTime. **/ + public static final String FIELD_EXPECTED_TERMINATION_TIME = "expectedTerminationTime"; + /** The field name for actualTerminationTime. **/ + public static final String FIELD_ACTUAL_TERMINATION_TIME = "actualTerminationTime"; + /** The field name for notificationTime. **/ + public static final String FIELD_NOTIFICATION_TIME = "notificationTime"; + /** The field name for launchTime. **/ + public static final String FIELD_LAUNCH_TIME = "launchTime"; + /** The field name for markTime. **/ + public static final String FIELD_MARK_TIME = "markTime"; + /** The field name for isOptOutOfJanitor. **/ + public static final String FIELD_OPT_OUT_OF_JANITOR = "optOutOfJanitor"; + /** The field name for awsResourceState. **/ + public static final String FIELD_AWS_RESOURCE_STATE = "awsResourceState"; + + /** The date format used to print or parse a Date value. **/ + public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); + + /** The map from name to value for additional fields used by the resource. **/ + private final Map additionalFields = new HashMap(); + + /** The map from AWS tag key to value for the resource. **/ + private final Map tags = new HashMap(); + + /** {@inheritDoc} */ + @Override + public Map getFieldToValueMap() { + Map fieldToValue = new HashMap(); + + putToMapIfNotNull(fieldToValue, FIELD_RESOURCE_ID, getId()); + putToMapIfNotNull(fieldToValue, FIELD_RESOURCE_TYPE, getResourceType()); + putToMapIfNotNull(fieldToValue, FIELD_REGION, getRegion()); + putToMapIfNotNull(fieldToValue, FIELD_OWNER_EMAIL, getOwnerEmail()); + putToMapIfNotNull(fieldToValue, FIELD_DESCRIPTION, getDescription()); + putToMapIfNotNull(fieldToValue, FIELD_STATE, getState()); + putToMapIfNotNull(fieldToValue, FIELD_TERMINATION_REASON, getTerminationReason()); + putToMapIfNotNull(fieldToValue, FIELD_EXPECTED_TERMINATION_TIME, printDate(getExpectedTerminationTime())); + putToMapIfNotNull(fieldToValue, FIELD_ACTUAL_TERMINATION_TIME, printDate(getActualTerminationTime())); + putToMapIfNotNull(fieldToValue, FIELD_NOTIFICATION_TIME, printDate(getNotificationTime())); + putToMapIfNotNull(fieldToValue, FIELD_LAUNCH_TIME, printDate(getLaunchTime())); + putToMapIfNotNull(fieldToValue, FIELD_MARK_TIME, printDate(getMarkTime())); + putToMapIfNotNull(fieldToValue, FIELD_AWS_RESOURCE_STATE, getAWSResourceState()); + + // Additional fields are serialized while tags are not. So if any tags need to be + // serialized as well, put them to additional fields. + fieldToValue.put(FIELD_OPT_OUT_OF_JANITOR, String.valueOf(isOptOutOfJanitor())); + + fieldToValue.putAll(additionalFields); + + return fieldToValue; + } + + /** + * Parse a map from field name to value to a resource. + * @param fieldToValue the map from field name to value + * @return the resource that is de-serialized from the map + */ + public static AWSResource parseFieldtoValueMap(Map fieldToValue) { + AWSResource resource = new AWSResource(); + for (Map.Entry field : fieldToValue.entrySet()) { + String name = field.getKey(); + String value = field.getValue(); + if (name.equals(FIELD_RESOURCE_ID)) { + resource.setId(value); + } else if (name.equals(FIELD_RESOURCE_TYPE)) { + resource.setResourceType(AWSResourceType.valueOf(value)); + } else if (name.equals(FIELD_REGION)) { + resource.setRegion(value); + } else if (name.equals(FIELD_OWNER_EMAIL)) { + resource.setOwnerEmail(value); + } else if (name.equals(FIELD_DESCRIPTION)) { + resource.setDescription(value); + } else if (name.equals(FIELD_STATE)) { + resource.setState(CleanupState.valueOf(value)); + } else if (name.equals(FIELD_TERMINATION_REASON)) { + resource.setTerminationReason(value); + } else if (name.equals(FIELD_EXPECTED_TERMINATION_TIME)) { + resource.setExpectedTerminationTime(new Date(DATE_FORMATTER.parseDateTime(value).getMillis())); + } else if (name.equals(FIELD_ACTUAL_TERMINATION_TIME)) { + resource.setActualTerminationTime(new Date(DATE_FORMATTER.parseDateTime(value).getMillis())); + } else if (name.equals(FIELD_NOTIFICATION_TIME)) { + resource.setNotificationTime(new Date(DATE_FORMATTER.parseDateTime(value).getMillis())); + } else if (name.equals(FIELD_LAUNCH_TIME)) { + resource.setLaunchTime(new Date(DATE_FORMATTER.parseDateTime(value).getMillis())); + } else if (name.equals(FIELD_MARK_TIME)) { + resource.setMarkTime(new Date(DATE_FORMATTER.parseDateTime(value).getMillis())); + } else if (name.equals(FIELD_AWS_RESOURCE_STATE)) { + resource.setAWSResourceState(value); + } else if (name.equals(FIELD_OPT_OUT_OF_JANITOR)) { + resource.setOptOutOfJanitor("true".equals(value)); + } else { + // put all other fields into additional fields + resource.setAdditionalField(name, value); + } + } + return resource; + } + + public String getAWSResourceState() { + return awsResourceState; + } + + public void setAWSResourceState(String awsState) { + this.awsResourceState = awsState; + } + + /** {@inheritDoc} */ + @Override + public String getId() { + return id; + } + + /** {@inheritDoc} */ + @Override + public void setId(String id) { + this.id = id; + } + + /** {@inheritDoc} */ + @Override + public Resource withId(String resourceId) { + setId(resourceId); + return this; + } + + /** {@inheritDoc} */ + @Override + public Enum getResourceType() { + return resourceType; + } + + /** {@inheritDoc} */ + @Override + public void setResourceType(Enum resourceType) { + this.resourceType = resourceType; + } + + /** {@inheritDoc} */ + @Override + public Resource withResourceType(Enum type) { + setResourceType(type); + return this; + } + + /** {@inheritDoc} */ + @Override + public String getRegion() { + return region; + } + + /** {@inheritDoc} */ + @Override + public void setRegion(String region) { + this.region = region; + } + + /** {@inheritDoc} */ + @Override + public Resource withRegion(String resourceRegion) { + setRegion(resourceRegion); + return this; + } + + /** {@inheritDoc} */ + @Override + public String getOwnerEmail() { + return ownerEmail; + } + + /** {@inheritDoc} */ + @Override + public void setOwnerEmail(String ownerEmail) { + this.ownerEmail = ownerEmail; + } + + /** {@inheritDoc} */ + @Override + public Resource withOwnerEmail(String resourceOwner) { + setOwnerEmail(resourceOwner); + return this; + } + + /** {@inheritDoc} */ + @Override + public String getDescription() { + return description; + } + + /** {@inheritDoc} */ + @Override + public void setDescription(String description) { + this.description = description; + } + + /** {@inheritDoc} */ + @Override + public Resource withDescription(String resourceDescription) { + setDescription(resourceDescription); + return this; + } + + /** {@inheritDoc} */ + @Override + public Date getLaunchTime() { + return getCopyOfDate(launchTime); + } + + /** {@inheritDoc} */ + @Override + public void setLaunchTime(Date launchTime) { + this.launchTime = getCopyOfDate(launchTime); + } + + /** {@inheritDoc} */ + @Override + public Resource withLaunchTime(Date resourceLaunchTime) { + setLaunchTime(resourceLaunchTime); + return this; + } + + /** {@inheritDoc} */ + @Override + public Date getMarkTime() { + return getCopyOfDate(markTime); + } + + /** {@inheritDoc} */ + @Override + public void setMarkTime(Date markTime) { + this.markTime = getCopyOfDate(markTime); + } + + /** {@inheritDoc} */ + @Override + public Resource withMarkTime(Date resourceMarkTime) { + setMarkTime(resourceMarkTime); + return this; + } + + /** {@inheritDoc} */ + @Override + public Date getExpectedTerminationTime() { + return getCopyOfDate(expectedTerminationTime); + } + + /** {@inheritDoc} */ + @Override + public void setExpectedTerminationTime(Date expectedTerminationTime) { + this.expectedTerminationTime = getCopyOfDate(expectedTerminationTime); + } + + /** {@inheritDoc} */ + @Override + public Resource withExpectedTerminationTime(Date resourceExpectedTerminationTime) { + setExpectedTerminationTime(resourceExpectedTerminationTime); + return this; + } + + /** {@inheritDoc} */ + @Override + public Date getActualTerminationTime() { + return getCopyOfDate(actualTerminationTime); + } + + /** {@inheritDoc} */ + @Override + public void setActualTerminationTime(Date actualTerminationTime) { + this.actualTerminationTime = getCopyOfDate(actualTerminationTime); + } + + /** {@inheritDoc} */ + @Override + public Resource withActualTerminationTime(Date resourceActualTerminationTime) { + setActualTerminationTime(resourceActualTerminationTime); + return this; + } + + /** {@inheritDoc} */ + @Override + public Date getNotificationTime() { + return getCopyOfDate(notificationTime); + } + + /** {@inheritDoc} */ + @Override + public void setNotificationTime(Date notificationTime) { + this.notificationTime = getCopyOfDate(notificationTime); + } + + /** {@inheritDoc} */ + @Override + public Resource withNnotificationTime(Date resourceNotificationTime) { + setNotificationTime(resourceNotificationTime); + return this; + } + + /** {@inheritDoc} */ + @Override + public CleanupState getState() { + return state; + } + + /** {@inheritDoc} */ + @Override + public void setState(CleanupState state) { + this.state = state; + } + + /** {@inheritDoc} */ + @Override + public Resource withState(CleanupState resourceState) { + setState(resourceState); + return this; + } + + /** {@inheritDoc} */ + @Override + public String getTerminationReason() { + return terminationReason; + } + + /** {@inheritDoc} */ + @Override + public void setTerminationReason(String terminationReason) { + this.terminationReason = terminationReason; + } + + /** {@inheritDoc} */ + @Override + public Resource withTerminationReason(String resourceTerminationReason) { + setTerminationReason(resourceTerminationReason); + return this; + } + + /** {@inheritDoc} */ + @Override + public boolean isOptOutOfJanitor() { + return optOutOfJanitor; + } + + /** {@inheritDoc} */ + @Override + public void setOptOutOfJanitor(boolean optOutOfJanitor) { + this.optOutOfJanitor = optOutOfJanitor; + } + + /** {@inheritDoc} */ + @Override + public Resource withOptOutOfJanitor(boolean optOut) { + setOptOutOfJanitor(optOut); + return this; + } + + private static Date getCopyOfDate(Date date) { + if (date == null) { + return null; + } + return new Date(date.getTime()); + } + + private static void putToMapIfNotNull(Map map, String key, String value) { + Validate.notNull(map); + Validate.notNull(key); + if (value != null) { + map.put(key, value); + } + } + + private static void putToMapIfNotNull(Map map, String key, Enum value) { + Validate.notNull(map); + Validate.notNull(key); + if (value != null) { + map.put(key, value.name()); + } + } + + private static String printDate(Date date) { + if (date == null) { + return null; + } + + return DATE_FORMATTER.print(date.getTime()); + } + + @Override + public Resource setAdditionalField(String fieldName, String fieldValue) { + Validate.notNull(fieldName); + Validate.notNull(fieldValue); + putToMapIfNotNull(additionalFields, fieldName, fieldValue); + return this; + } + + @Override + public String getAdditionalField(String fieldName) { + return additionalFields.get(fieldName); + } + + @Override + public Collection getAdditionalFieldNames() { + return additionalFields.keySet(); + } + + @Override + public Resource cloneResource() { + Resource clone = new AWSResource() + .withActualTerminationTime(getActualTerminationTime()) + .withDescription(getDescription()) + .withExpectedTerminationTime(getExpectedTerminationTime()) + .withId(getId()) + .withLaunchTime(getLaunchTime()) + .withMarkTime(getMarkTime()) + .withNnotificationTime(getNotificationTime()) + .withOwnerEmail(getOwnerEmail()) + .withRegion(getRegion()) + .withResourceType(getResourceType()) + .withState(getState()) + .withTerminationReason(getTerminationReason()) + .withOptOutOfJanitor(isOptOutOfJanitor()); + ((AWSResource) clone).setAWSResourceState(awsResourceState); + + ((AWSResource) clone).additionalFields.putAll(additionalFields); + + for (String key : this.getAllTagKeys()) { + clone.setTag(key, this.getTag(key)); + } + + return clone; + } + + /** {@inheritDoc} */ + @Override + public void setTag(String key, String value) { + tags.put(key, value); + } + + /** {@inheritDoc} */ + @Override + public String getTag(String key) { + return tags.get(key); + } + + /** {@inheritDoc} */ + @Override + public Collection getAllTagKeys() { + return tags.keySet(); + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/AWSResourceType.java b/src/main/java/com/netflix/simianarmy/aws/AWSResourceType.java new file mode 100644 index 00000000..e81856ac --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/AWSResourceType.java @@ -0,0 +1,39 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws; + +/** + * The enum of resource types of AWS. + */ +public enum AWSResourceType { + /** AWS instance. */ + INSTANCE, + /** AWS EBS volume. */ + EBS_VOLUME, + /** AWS EBS snapshot. */ + EBS_SNAPSHOT, + /** AWS auto scaling group. */ + ASG, + /** AWS launch configuration. */ + LAUNCH_CONFIG, + /** AWS S3 bucket. */ + S3_BUCKET, + /** AWS security group. */ + SECURITY_GROUP +} diff --git a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java index 0b7efd96..a0a19d04 100644 --- a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java +++ b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java @@ -27,11 +27,9 @@ import java.util.Map; import java.util.Set; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import org.apache.commons.lang.Validate; + import com.amazonaws.services.simpledb.AmazonSimpleDB; -import com.amazonaws.services.simpledb.AmazonSimpleDBClient; import com.amazonaws.services.simpledb.model.Attribute; import com.amazonaws.services.simpledb.model.Item; import com.amazonaws.services.simpledb.model.PutAttributesRequest; @@ -40,6 +38,7 @@ import com.amazonaws.services.simpledb.model.SelectResult; import com.netflix.simianarmy.MonkeyRecorder; import com.netflix.simianarmy.basic.BasicRecorderEvent; +import com.netflix.simianarmy.client.aws.AWSClient; /** * The Class SimpleDBRecorder. Records events to and fetched events from a Amazon SimpleDB table (default SIMIAN_ARMY) @@ -47,14 +46,12 @@ @SuppressWarnings("serial") public class SimpleDBRecorder implements MonkeyRecorder { - /** The cred. */ - private AWSCredentials cred; + private final AmazonSimpleDB simpleDBClient; - /** The region. */ - private String region; + private final String region; /** The domain. */ - private String domain; + private final String domain; /** * The Enum Keys. @@ -87,65 +84,26 @@ private enum Keys { /** * Instantiates a new simple db recorder. * - * @param accessKey - * the access key - * @param secretKey - * the secret key - * @param region - * the region - * @param domain - * the domain - */ - public SimpleDBRecorder(String accessKey, String secretKey, String region, String domain) { - this.cred = new BasicAWSCredentials(accessKey, secretKey); - this.region = region; - this.domain = domain; - } - - /** - * Instantiates a new simple db recorder. - * - * @param cred - * the cred - * @param region - * the region + * @param awsClient + * the AWS client * @param domain * the domain */ - public SimpleDBRecorder(AWSCredentials cred, String region, String domain) { - this.cred = cred; - this.region = region; + public SimpleDBRecorder(AWSClient awsClient, String domain) { + Validate.notNull(awsClient); + Validate.notNull(domain); + this.simpleDBClient = awsClient.sdbClient(); + this.region = awsClient.region(); this.domain = domain; } - /** - * Use {@link DefaultAWSCredentialsProviderChain} to provide credentials. - * - * @param region - * the region - * @param domain - * the domain - */ - public SimpleDBRecorder(String region, String domain) { - this(new DefaultAWSCredentialsProviderChain().getCredentials(), region, domain); - } - /** * simple client. abstracted to aid testing * * @return the amazon simple db */ protected AmazonSimpleDB sdbClient() { - AmazonSimpleDB client = new AmazonSimpleDBClient(cred); - - // us-east-1 has special naming - // http://docs.amazonwebservices.com/general/latest/gr/rande.html#sdb_region - if (region.equals("us-east-1")) { - client.setEndpoint("sdb.amazonaws.com"); - } else { - client.setEndpoint("sdb." + region + ".amazonaws.com"); - } - return client; + return simpleDBClient; } /** @@ -188,15 +146,18 @@ private static Enum valueToEnum(String value) { } /** {@inheritDoc} */ + @Override public Event newEvent(Enum monkeyType, Enum eventType, String reg, String id) { return new BasicRecorderEvent(monkeyType, eventType, reg, id); } /** {@inheritDoc} */ + @Override public void recordEvent(Event evt) { + String evtTime = String.valueOf(evt.eventTime().getTime()); List attrs = new LinkedList(); attrs.add(new ReplaceableAttribute(Keys.id.name(), evt.id(), true)); - attrs.add(new ReplaceableAttribute(Keys.eventTime.name(), String.valueOf(evt.eventTime().getTime()), true)); + attrs.add(new ReplaceableAttribute(Keys.eventTime.name(), evtTime, true)); attrs.add(new ReplaceableAttribute(Keys.region.name(), evt.region(), true)); attrs.add(new ReplaceableAttribute(Keys.recordType.name(), "MonkeyEvent", true)); attrs.add(new ReplaceableAttribute(Keys.monkeyType.name(), enumToValue(evt.monkeyType()), true)); @@ -207,7 +168,8 @@ public void recordEvent(Event evt) { } attrs.add(new ReplaceableAttribute(pair.getKey(), pair.getValue(), true)); } - String pk = String.format("%s-%s-%s", evt.monkeyType().name(), evt.id(), region); + // Let pk contain the timestamp so that the same resource can have multiple events. + String pk = String.format("%s-%s-%s-%s", evt.monkeyType().name(), evt.id(), region, evtTime); PutAttributesRequest putReq = new PutAttributesRequest(domain, pk, attrs); sdbClient().putAttributes(putReq); @@ -223,7 +185,8 @@ public void recordEvent(Event evt) { * @return the list */ protected List findEvents(Map queryMap, long after) { - StringBuilder query = new StringBuilder(String.format("select * from %s where region = '%s'", domain, region)); + StringBuilder query = new StringBuilder( + String.format("select * from %s where region = '%s'", domain, region)); for (Map.Entry pair : queryMap.entrySet()) { query.append(String.format(" and %s = '%s'", pair.getKey(), pair.getValue())); } @@ -260,11 +223,13 @@ protected List findEvents(Map queryMap, long after) { } /** {@inheritDoc} */ + @Override public List findEvents(Map query, Date after) { return findEvents(query, after.getTime()); } /** {@inheritDoc} */ + @Override public List findEvents(Enum monkeyType, Map query, Date after) { Map copy = new LinkedHashMap(query); copy.put(Keys.monkeyType.name(), enumToValue(monkeyType)); @@ -272,6 +237,7 @@ public List findEvents(Enum monkeyType, Map query, Date a } /** {@inheritDoc} */ + @Override public List findEvents(Enum monkeyType, Enum eventType, Map query, Date after) { Map copy = new LinkedHashMap(query); copy.put(Keys.monkeyType.name(), enumToValue(monkeyType)); diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/ASGJanitor.java b/src/main/java/com/netflix/simianarmy/aws/janitor/ASGJanitor.java new file mode 100644 index 00000000..a925fdac --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/ASGJanitor.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.AbstractJanitor; + +/** + * The Janitor responsible for ASG cleanup. + */ +public class ASGJanitor extends AbstractJanitor { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJanitor.class); + + private final AWSClient awsClient; + + /** + * Constructor. + * @param awsClient the AWS client + * @param ctx the context + */ + public ASGJanitor(AWSClient awsClient, AbstractJanitor.Context ctx) { + super(ctx, AWSResourceType.ASG); + Validate.notNull(awsClient); + this.awsClient = awsClient; + } + + @Override + protected void postMark(Resource resource) { + } + + @Override + protected void cleanup(Resource resource) { + LOGGER.info(String.format("Deleting ASG %s", resource.getId())); + awsClient.deleteAutoScalingGroup(resource.getId()); + } + + @Override + protected void postCleanup(Resource resource) { + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/EBSSnapshotJanitor.java b/src/main/java/com/netflix/simianarmy/aws/janitor/EBSSnapshotJanitor.java new file mode 100644 index 00000000..e6309db3 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/EBSSnapshotJanitor.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.AbstractJanitor; + +/** + * The Janitor responsible for EBS snapshot cleanup. + */ +public class EBSSnapshotJanitor extends AbstractJanitor { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(EBSSnapshotJanitor.class); + + private final AWSClient awsClient; + + /** + * Constructor. + * @param awsClient the AWS client + * @param ctx the context + */ + public EBSSnapshotJanitor(AWSClient awsClient, AbstractJanitor.Context ctx) { + super(ctx, AWSResourceType.EBS_SNAPSHOT); + Validate.notNull(awsClient); + this.awsClient = awsClient; + } + + @Override + protected void postMark(Resource resource) { + } + + @Override + protected void cleanup(Resource resource) { + LOGGER.info(String.format("Deleting EBS snapshot %s", resource.getId())); + awsClient.deleteSnapshot(resource.getId()); + } + + @Override + protected void postCleanup(Resource resource) { + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/EBSVolumeJanitor.java b/src/main/java/com/netflix/simianarmy/aws/janitor/EBSVolumeJanitor.java new file mode 100644 index 00000000..90effebe --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/EBSVolumeJanitor.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.AbstractJanitor; + +/** + * The Janitor responsible for EBS volume cleanup. + */ +public class EBSVolumeJanitor extends AbstractJanitor { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(EBSVolumeJanitor.class); + + private final AWSClient awsClient; + + /** + * Constructor. + * @param awsClient the AWS client + * @param ctx the context + */ + public EBSVolumeJanitor(AWSClient awsClient, AbstractJanitor.Context ctx) { + super(ctx, AWSResourceType.EBS_VOLUME); + Validate.notNull(awsClient); + this.awsClient = awsClient; + } + + @Override + protected void postMark(Resource resource) { + } + + @Override + protected void cleanup(Resource resource) { + LOGGER.info(String.format("Deleting EBS volume %s", resource.getId())); + awsClient.deleteVolume(resource.getId()); + } + + @Override + protected void postCleanup(Resource resource) { + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/InstanceJanitor.java b/src/main/java/com/netflix/simianarmy/aws/janitor/InstanceJanitor.java new file mode 100644 index 00000000..a9478c52 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/InstanceJanitor.java @@ -0,0 +1,64 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.AbstractJanitor; + +/** + * The Janitor responsible for auto scaling instance cleanup. + */ +public class InstanceJanitor extends AbstractJanitor { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceJanitor.class); + + private final AWSClient awsClient; + + /** + * Constructor. + * @param awsClient the AWS client + * @param ctx the context + */ + public InstanceJanitor(AWSClient awsClient, AbstractJanitor.Context ctx) { + super(ctx, AWSResourceType.INSTANCE); + Validate.notNull(awsClient); + this.awsClient = awsClient; + } + + @Override + protected void postMark(Resource resource) { + } + + @Override + protected void cleanup(Resource resource) { + LOGGER.info(String.format("Terminating instance %s", resource.getId())); + awsClient.terminateInstance(resource.getId()); + } + + @Override + protected void postCleanup(Resource resource) { + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java new file mode 100644 index 00000000..51496637 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java @@ -0,0 +1,195 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.simpledb.AmazonSimpleDB; +import com.amazonaws.services.simpledb.model.Attribute; +import com.amazonaws.services.simpledb.model.Item; +import com.amazonaws.services.simpledb.model.PutAttributesRequest; +import com.amazonaws.services.simpledb.model.ReplaceableAttribute; +import com.amazonaws.services.simpledb.model.SelectRequest; +import com.amazonaws.services.simpledb.model.SelectResult; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.JanitorResourceTracker; + +/** + * The JanitorResourceTracker implementation in SimpleDB. + */ +public class SimpleDBJanitorResourceTracker implements JanitorResourceTracker { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleDBJanitorResourceTracker.class); + + /** The domain. */ + private final String domain; + + /** The SimpleDB client. */ + private final AmazonSimpleDB simpleDBClient; + + /** + * Instantiates a new simple db resource tracker. + * + * @param awsClient + * the AWS Client + * @param domain + * the domain + */ + public SimpleDBJanitorResourceTracker(AWSClient awsClient, String domain) { + this.domain = domain; + this.simpleDBClient = awsClient.sdbClient(); + } + + /** + * Gets the SimpleDB client. + * @return the SimpleDB client + */ + protected AmazonSimpleDB getSimpleDBClient() { + return simpleDBClient; + } + + /** {@inheritDoc} */ + @Override + public void addOrUpdate(Resource resource) { + List attrs = new ArrayList(); + Map fieldToValueMap = resource.getFieldToValueMap(); + for (Map.Entry entry : fieldToValueMap.entrySet()) { + attrs.add(new ReplaceableAttribute(entry.getKey(), entry.getValue(), true)); + } + PutAttributesRequest putReqest = new PutAttributesRequest(domain, getSimpleDBItemName(resource), attrs); + LOGGER.debug(String.format("Saving resource %s to SimpleDB domain %s", + resource.getId(), domain)); + this.simpleDBClient.putAttributes(putReqest); + LOGGER.debug("Successfully saved."); + } + + /** + * Returns a list of AWSResource objects. You need to override this method if more + * specific resource types (e.g. subtypes of AWSResource) need to be obtained from + * the SimpleDB. + */ + @Override + public List getResources(Enum resourceType, CleanupState state, String resourceRegion) { + Validate.notEmpty(resourceRegion); + List resources = new ArrayList(); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where ", domain)); + if (resourceType != null) { + query.append(String.format("resourceType='%s' and ", resourceType)); + } + if (state != null) { + query.append(String.format("state='%s' and ", state)); + } + query.append(String.format("region='%s'", resourceRegion)); + + LOGGER.debug(String.format("Query is '%s'", query)); + + List items = querySimpleDBItems(query.toString()); + for (Item item : items) { + try { + resources.add(parseResource(item)); + } catch (Exception e) { + // Ignore the item that cannot be parsed. + LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a resource.", item)); + } + } + LOGGER.info(String.format("Retrieved %d resources from SimpleDB in domain %s for resource type %s" + + " and state %s and region %s", + resources.size(), domain, resourceType, state, resourceRegion)); + return resources; + } + + @Override + public Resource getResource(String resourceId) { + Validate.notEmpty(resourceId); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where resourceId = '%s'", domain, resourceId)); + + LOGGER.debug(String.format("Query is '%s'", query)); + + List items = querySimpleDBItems(query.toString()); + Validate.isTrue(items.size() <= 1); + if (items.size() == 0) { + LOGGER.info(String.format("Not found resource with id %s", resourceId)); + return null; + } else { + Resource resource = null; + try { + resource = parseResource(items.get(0)); + } catch (Exception e) { + // Ignore the item that cannot be parsed. + LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a resource.", items.get(0))); + } + return resource; + } + } + + /** + * Parses a SimpleDB item into an AWS resource. + * @param item the item from SimpleDB + * @return the AWSResource object for the SimpleDB item + */ + protected Resource parseResource(Item item) { + Map fieldToValue = new HashMap(); + for (Attribute attr : item.getAttributes()) { + String name = attr.getName(); + String value = attr.getValue(); + if (name != null && value != null) { + fieldToValue.put(name, value); + } + } + return AWSResource.parseFieldtoValueMap(fieldToValue); + } + + /** + * Gets the unique SimpleDB item name for a resource. The subclass can override this + * method to generate the item name differently. + * @param resource + * @return the SimpleDB item name for the resource + */ + protected String getSimpleDBItemName(Resource resource) { + return String.format("%s-%s-%s", resource.getResourceType().name(), resource.getId(), resource.getRegion()); + } + + private List querySimpleDBItems(String query) { + Validate.notNull(query); + String nextToken = null; + List items = new ArrayList(); + do { + SelectRequest request = new SelectRequest(query); + request.setNextToken(nextToken); + request.setConsistentRead(Boolean.TRUE); + SelectResult result = this.simpleDBClient.select(request); + items.addAll(result.getItems()); + nextToken = result.getNextToken(); + } while (nextToken != null); + + return items; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/VolumeTaggingMonkey.java b/src/main/java/com/netflix/simianarmy/aws/janitor/VolumeTaggingMonkey.java new file mode 100644 index 00000000..0dc871ee --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/VolumeTaggingMonkey.java @@ -0,0 +1,307 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Tag; +import com.amazonaws.services.ec2.model.Volume; +import com.amazonaws.services.ec2.model.VolumeAttachment; +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.JanitorMonkey; + +/** + * A companion monkey of Janitor Monkey for tagging EBS volumes with the last attachment information. + * In many scenarios, EBS volumes generated by applications remain unattached to instances. Amazon + * does not keep track of last unattached time, which makes it difficult to determine its usage. + * To solve this, this monkey will tag all EBS volumes with last owner and instance to which they are attached + * and the time they got detached from instance. The monkey will poll and monitor EBS volumes hourly (by default). + * + */ +public class VolumeTaggingMonkey extends Monkey { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(VolumeTaggingMonkey.class); + + /** + * The Interface Context. + */ + public interface Context extends Monkey.Context { + /** + * Configuration. + * + * @return the monkey configuration + */ + @Override + MonkeyConfiguration configuration(); + + /** + * AWS client. + * + * @return the AWS client + */ + AWSClient awsClient(); + } + + private final MonkeyConfiguration config; + private final AWSClient awsClient; + private final MonkeyCalendar calendar; + + /** We cache the global map from instance id to its owner when starting the monkey. */ + private final Map instanceToOwner; + + /** + * The constructor. + * @param ctx the context + */ + public VolumeTaggingMonkey(Context ctx) { + super(ctx); + this.config = ctx.configuration(); + this.awsClient = ctx.awsClient(); + this.calendar = ctx.calendar(); + instanceToOwner = new HashMap(); + for (Instance instance : awsClient.describeInstances()) { + for (Tag tag : instance.getTags()) { + if (tag.getKey().equals(JanitorMonkey.OWNER_TAG_KEY)) { + instanceToOwner.put(instance.getInstanceId(), tag.getValue()); + } + } + } + } + + /** + * The monkey Type. + */ + public enum Type { + /** Volume tagging monkey. */ + VOLUME_TAGGING + } + + /** + * The event types that this monkey causes. + */ + public enum EventTypes { + /** The event type for tagging the volume with Janitor meta data information. */ + TAGGING_JANITOR + } + + @Override + public Enum type() { + return Type.VOLUME_TAGGING; + } + + @Override + public void doMonkeyBusiness() { + String prop = "simianarmy.volumeTagging.enabled"; + if (config.getBoolOrElse(prop, false)) { + tagVolumesWithLatestAttachment(awsClient.describeVolumes()); + } else { + LOGGER.info(String.format("Volume tagging monkey is not enabled. You can set %s to true to enalbe it.", + prop)); + } + } + + private void tagVolumesWithLatestAttachment(List volumes) { + LOGGER.info(String.format("Trying to tag %d volumes for Janitor Monkey meta data.", + volumes.size())); + Date now = calendar.now().getTime(); + for (Volume volume : volumes) { + String owner = null, instanceId = null; + Date lastDetachTime = null; + List attachments = volume.getAttachments(); + List tags = volume.getTags(); + + // The volume can have a special tag is it does not want to be changed/tagged + // by Jantior monkey. + if ("donotmark".equals(getTagValue(JanitorMonkey.JANITOR_TAG, tags))) { + LOGGER.info(String.format("The volume %s is tagged as not handled by Janitor", + volume.getVolumeId())); + continue; + } + + Map janitorMetadata = parseJanitorTag(tags); + // finding the instance attached most recently. + VolumeAttachment latest = null; + for (VolumeAttachment attachment : attachments) { + if (latest == null || latest.getAttachTime().before(attachment.getAttachTime())) { + latest = attachment; + } + } + if (latest != null) { + instanceId = latest.getInstanceId(); + owner = getOwnerEmail(instanceId, janitorMetadata, tags); + } + + if (latest == null || "detached".equals(latest.getState())) { + if (janitorMetadata.get(JanitorMonkey.DETACH_TIME_TAG_KEY) == null) { + // There is no attached instance and the last detached time is not set. + // Use the current time as the last detached time. + LOGGER.info(String.format("Setting the last detached time to %s for volume %s", + now, volume.getVolumeId())); + lastDetachTime = now; + } else { + LOGGER.debug(String.format("The volume %s was already marked as detached at time %s", + volume.getVolumeId(), janitorMetadata.get(JanitorMonkey.DETACH_TIME_TAG_KEY))); + } + } else { + // The volume is currently attached to an instance + lastDetachTime = null; + } + String existingOwner = janitorMetadata.get(JanitorMonkey.OWNER_TAG_KEY); + if (owner == null && existingOwner != null) { + // Save the current owner in the tag when we are not able to find a owner. + owner = existingOwner; + } + if (needsUpdate(janitorMetadata, owner, instanceId, lastDetachTime)) { + Event evt = updateJanitorMetaTag(volume, instanceId, owner, lastDetachTime); + if (evt != null) { + context().recorder().recordEvent(evt); + } + } + } + } + + private String getOwnerEmail(String instanceId, Map janitorMetadata, List tags) { + // The owner of the volume is set as the owner of the last instance attached to it. + String owner = instanceToOwner.get(instanceId); + if (owner == null) { + owner = janitorMetadata.get(JanitorMonkey.OWNER_TAG_KEY); + } + if (owner == null) { + owner = getTagValue(JanitorMonkey.OWNER_TAG_KEY, tags); + } + String emailDomain = getOwnerEmailDomain(); + if (owner != null && !owner.contains("@") + && StringUtils.isNotBlank(emailDomain)) { + owner = String.format("%s@%s", owner, emailDomain); + } + return owner; + } + + /** + * Parses the Janitor meta tag set by this monkey and gets a map from key + * to value for the tag values. + * @param tags the tags of the volumes + * @return the map from the Janitor meta tag key to value + */ + private static Map parseJanitorTag(List tags) { + String janitorTag = getTagValue(JanitorMonkey.JANITOR_META_TAG, tags); + return parseJanitorMetaTag(janitorTag); + } + + /** + * Parses the string of Janitor meta-data tag value to get a key value map. + * @param janitorMetaTag the value of the Janitor meta-data tag + * @return the key value map in the Janitor meta-data tag + */ + public static Map parseJanitorMetaTag(String janitorMetaTag) { + Map metadata = new HashMap(); + if (janitorMetaTag != null) { + for (String keyValue : janitorMetaTag.split(";")) { + String[] meta = keyValue.split("="); + if (meta.length == 2) { + metadata.put(meta[0], meta[1]); + } + } + } + return metadata; + } + + /** Gets the domain name for the owner email. The method can be overriden in subclasses. + * + * @return the domain name for the owner email. + */ + protected String getOwnerEmailDomain() { + return config.getStrOrElse("simianarmy.volumeTagging.ownerEmailDomain", ""); + } + + private Event updateJanitorMetaTag(Volume volume, String instance, String owner, Date lastDetachTime) { + String meta = makeMetaTag(instance, owner, lastDetachTime); + Map janitorTags = new HashMap(); + janitorTags.put(JanitorMonkey.JANITOR_META_TAG, meta); + LOGGER.info(String.format("Setting tag %s to '%s' for volume %s", + JanitorMonkey.JANITOR_META_TAG, meta, volume.getVolumeId())); + String prop = "simianarmy.volumeTagging.leashed"; + Event evt = null; + if (config.getBoolOrElse(prop, true)) { + LOGGER.info("Volume tagging monkey is leashed. No real change is made to the volume."); + } else { + try { + awsClient.createTagsForResources(janitorTags, volume.getVolumeId()); + evt = context().recorder().newEvent(type(), EventTypes.TAGGING_JANITOR, + awsClient.region(), volume.getVolumeId()); + evt.addField(JanitorMonkey.JANITOR_META_TAG, meta); + } catch (Exception e) { + LOGGER.error(String.format("Failed to update the tag for volume %s", volume.getVolumeId())); + } + } + return evt; + } + + /** + * Makes the Janitor meta tag for volumes to track the last attachment/detachment information. + * The method is intentionally made public for testing. + * @param instance the last attached instance + * @param owner the last owner + * @param lastDetachTime the detach time + * @return the meta tag of Janitor Monkey + */ + public static String makeMetaTag(String instance, String owner, Date lastDetachTime) { + StringBuilder meta = new StringBuilder(); + meta.append(String.format("%s=%s;", + JanitorMonkey.INSTANCE_TAG_KEY, instance == null ? "" : instance)); + meta.append(String.format("%s=%s;", JanitorMonkey.OWNER_TAG_KEY, owner == null ? "" : owner)); + meta.append(String.format("%s=%s", JanitorMonkey.DETACH_TIME_TAG_KEY, + lastDetachTime == null ? "" : AWSResource.DATE_FORMATTER.print(lastDetachTime.getTime()))); + return meta.toString(); + } + + private static String getTagValue(String key, List tags) { + for (Tag tag : tags) { + if (tag.getKey().equals(key)) { + return tag.getValue(); + } + } + return null; + } + + /** Needs to update tags of the volume if + * 1) owner or instance attached changed or + * 2) the last detached status is changed. + */ + private static boolean needsUpdate(Map metadata, + String owner, String instance, Date lastDetachTime) { + return (owner != null && !StringUtils.equals(metadata.get(JanitorMonkey.OWNER_TAG_KEY), owner)) + || (instance != null && !StringUtils.equals(metadata.get(JanitorMonkey.INSTANCE_TAG_KEY), instance)) + || lastDetachTime != null; + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/ASGJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/ASGJanitorCrawler.java new file mode 100644 index 00000000..7c41bb86 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/ASGJanitorCrawler.java @@ -0,0 +1,181 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.autoscaling.model.AutoScalingGroup; +import com.amazonaws.services.autoscaling.model.Instance; +import com.amazonaws.services.autoscaling.model.LaunchConfiguration; +import com.amazonaws.services.autoscaling.model.SuspendedProcess; +import com.amazonaws.services.autoscaling.model.TagDescription; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +/** + * The crawler to crawl AWS auto scaling groups for janitor monkey. + */ +public class ASGJanitorCrawler extends AbstractAWSJanitorCrawler { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ASGJanitorCrawler.class); + + /** The name representing the additional field name of instance ids. */ + public static final String ASG_FIELD_INSTANCES = "INSTANCES"; + + /** The name representing the additional field name of max ASG size. */ + public static final String ASG_FIELD_MAX_SIZE = "MAX_SIZE"; + + /** The name representing the additional field name of ELB names. */ + public static final String ASG_FIELD_ELBS = "ELBS"; + + /** The name representing the additional field name of launch configuration name. */ + public static final String ASG_FIELD_LC_NAME = "LAUNCH_CONFIGURATION_NAME"; + + /** The name representing the additional field name of launch configuration creation time. */ + public static final String ASG_FIELD_LC_CREATION_TIME = "LAUNCH_CONFIGURATION_CREATION_TIME"; + + /** The name representing the additional field name of ASG suspension time from ELB. */ + public static final String ASG_FIELD_SUSPENSION_TIME = "ASG_SUSPENSION_TIME"; + + private final Map nameToLaunchConfig = new HashMap(); + + /** The regular expression patter below is for the termination reason added by AWS when + * an ASG is suspended from ELB's traffic. + */ + private static final Pattern SUSPENSION_REASON_PATTERN = + Pattern.compile("User suspended at (\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}).*"); + + /** The date format used to print or parse the suspension time value. **/ + public static final DateTimeFormatter SUSPENSION_TIME_FORMATTER = + DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); + + /** + * Instantiates a new basic ASG crawler. + * @param awsClient + * the aws client + */ + public ASGJanitorCrawler(AWSClient awsClient) { + super(awsClient); + } + + @Override + public EnumSet resourceTypes() { + return EnumSet.of(AWSResourceType.ASG); + } + + @Override + public List resources(Enum resourceType) { + if ("ASG".equals(resourceType.name())) { + return getASGResources(); + } + return Collections.emptyList(); + } + + @Override + public List resources(String... asgNames) { + return getASGResources(asgNames); + } + + private List getASGResources(String... asgNames) { + AWSClient awsClient = getAWSClient(); + + List launchConfigurations = awsClient.describeLaunchConfigurations(); + for (LaunchConfiguration lc : launchConfigurations) { + nameToLaunchConfig.put(lc.getLaunchConfigurationName(), lc); + } + + List resources = new LinkedList(); + for (AutoScalingGroup asg : awsClient.describeAutoScalingGroups(asgNames)) { + Resource asgResource = new AWSResource().withId(asg.getAutoScalingGroupName()) + .withResourceType(AWSResourceType.ASG).withRegion(awsClient.region()) + .withLaunchTime(asg.getCreatedTime()); + for (TagDescription tag : asg.getTags()) { + asgResource.setTag(tag.getKey(), tag.getValue()); + } + asgResource.setDescription(String.format("%d instances", asg.getInstances().size())); + asgResource.setOwnerEmail(getOwnerEmailForResource(asgResource)); + if (asg.getStatus() != null) { + ((AWSResource) asgResource).setAWSResourceState(asg.getStatus()); + } + Integer maxSize = asg.getMaxSize(); + if (maxSize != null) { + asgResource.setAdditionalField(ASG_FIELD_MAX_SIZE, String.valueOf(maxSize)); + } + // Adds instances and ELBs as additional fields. + List instances = new ArrayList(); + for (Instance instance : asg.getInstances()) { + instances.add(instance.getInstanceId()); + } + asgResource.setAdditionalField(ASG_FIELD_INSTANCES, StringUtils.join(instances, ",")); + asgResource.setAdditionalField(ASG_FIELD_ELBS, + StringUtils.join(asg.getLoadBalancerNames(), ",")); + String lcName = asg.getLaunchConfigurationName(); + LaunchConfiguration lc = nameToLaunchConfig.get(lcName); + if (lc != null) { + asgResource.setAdditionalField(ASG_FIELD_LC_NAME, lcName); + } + if (lc != null && lc.getCreatedTime() != null) { + asgResource.setAdditionalField(ASG_FIELD_LC_CREATION_TIME, + String.valueOf(lc.getCreatedTime().getTime())); + } + // sets the field for the time when the ASG's traffic is suspended from ELB + for (SuspendedProcess sp : asg.getSuspendedProcesses()) { + if ("AddToLoadBalancer".equals(sp.getProcessName())) { + String suspensionTime = getSuspensionTimeString(sp.getSuspensionReason()); + if (suspensionTime != null) { + LOGGER.info(String.format("Suspension time of ASG %s is %s", + asg.getAutoScalingGroupName(), suspensionTime)); + asgResource.setAdditionalField(ASG_FIELD_SUSPENSION_TIME, suspensionTime); + break; + } + } + } + resources.add(asgResource); + } + return resources; + } + + private String getSuspensionTimeString(String suspensionReason) { + if (suspensionReason == null) { + return null; + } + Matcher matcher = SUSPENSION_REASON_PATTERN.matcher(suspensionReason); + if (matcher.matches()) { + return matcher.group(1); + } + return null; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/AbstractAWSJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/AbstractAWSJanitorCrawler.java new file mode 100644 index 00000000..8466386a --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/AbstractAWSJanitorCrawler.java @@ -0,0 +1,61 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.crawler; + +import org.apache.commons.lang.Validate; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.JanitorCrawler; + +/** + * The abstract class for crawler of AWS resources. + */ +public abstract class AbstractAWSJanitorCrawler implements JanitorCrawler { + /** The AWS client. */ + private final AWSClient awsClient; + + /** + * The constructor. + * @param awsClient the AWS client used by the crawler. + */ + public AbstractAWSJanitorCrawler(AWSClient awsClient) { + Validate.notNull(awsClient); + this.awsClient = awsClient; + } + + /** + * Gets the owner email from the resource's tag "owner". + * @param resource the resource + * @return the owner email specified in the resource's tags + */ + @Override + public String getOwnerEmailForResource(Resource resource) { + Validate.notNull(resource); + return resource.getTag("owner"); + } + + /** + * Gets the AWS client used by the crawler. + * @return the AWS client used by the crawler. + */ + protected AWSClient getAWSClient() { + return awsClient; + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSSnapshotJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSSnapshotJanitorCrawler.java new file mode 100644 index 00000000..8c3b86f5 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSSnapshotJanitorCrawler.java @@ -0,0 +1,151 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.crawler; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.ec2.model.BlockDeviceMapping; +import com.amazonaws.services.ec2.model.EbsBlockDevice; +import com.amazonaws.services.ec2.model.Image; +import com.amazonaws.services.ec2.model.Snapshot; +import com.amazonaws.services.ec2.model.Tag; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +/** + * The crawler to crawl AWS EBS snapshots for janitor monkey. + */ +public class EBSSnapshotJanitorCrawler extends AbstractAWSJanitorCrawler { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(EBSSnapshotJanitorCrawler.class); + + /** The name representing the additional field name of AMIs genreated using the snapshot. */ + public static final String SNAPSHOT_FIELD_AMIS = "AMIs"; + + + /** The map from snapshot id to the AMI ids that are generated using the snapshot. */ + private final Map> snapshotToAMIs = + new HashMap>(); + + /** + * The constructor. + * @param awsClient the AWS client + */ + public EBSSnapshotJanitorCrawler(AWSClient awsClient) { + super(awsClient); + } + + @Override + public EnumSet resourceTypes() { + return EnumSet.of(AWSResourceType.EBS_SNAPSHOT); + } + + @Override + public List resources(Enum resourceType) { + if ("EBS_SNAPSHOT".equals(resourceType.name())) { + return getSnapshotResources(); + } + return Collections.emptyList(); + } + + @Override + public List resources(String... resourceIds) { + return getSnapshotResources(resourceIds); + } + + private List getSnapshotResources(String... snapshotIds) { + refreshSnapshotToAMIs(); + + List resources = new LinkedList(); + AWSClient awsClient = getAWSClient(); + + for (Snapshot snapshot : awsClient.describeSnapshots(snapshotIds)) { + Resource snapshotResource = new AWSResource().withId(snapshot.getSnapshotId()) + .withRegion(getAWSClient().region()).withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(snapshot.getStartTime()).withDescription(snapshot.getDescription()); + for (Tag tag : snapshot.getTags()) { + LOGGER.debug(String.format("Adding tag %s = %s to resource %s", + tag.getKey(), tag.getValue(), snapshotResource.getId())); + snapshotResource.setTag(tag.getKey(), tag.getValue()); + } + snapshotResource.setOwnerEmail(getOwnerEmailForResource(snapshotResource)); + ((AWSResource) snapshotResource).setAWSResourceState(snapshot.getState()); + Collection amis = snapshotToAMIs.get(snapshotResource.getId()); + if (amis != null) { + snapshotResource.setAdditionalField(SNAPSHOT_FIELD_AMIS, StringUtils.join(amis, ",")); + } + resources.add(snapshotResource); + } + return resources; + } + + @Override + public String getOwnerEmailForResource(Resource resource) { + String owner = resource.getTag("creator"); + if (owner == null) { + owner = super.getOwnerEmailForResource(resource); + } + return owner; + } + + /** + * Gets the collection of AMIs that are created using a specific snapshot. + * @param snapshotId the snapshot id + */ + protected Collection getAMIsForSnapshot(String snapshotId) { + Collection amis = snapshotToAMIs.get(snapshotId); + if (amis != null) { + return Collections.unmodifiableCollection(amis); + } else { + return Collections.emptyList(); + } + } + + private void refreshSnapshotToAMIs() { + snapshotToAMIs.clear(); + for (Image image : getAWSClient().describeImages()) { + for (BlockDeviceMapping bdm : image.getBlockDeviceMappings()) { + EbsBlockDevice ebd = bdm.getEbs(); + if (ebd != null && ebd.getSnapshotId() != null) { + LOGGER.debug(String.format("Snapshot %s is used to generate AMI %s", + ebd.getSnapshotId(), image.getImageId())); + Collection amis = snapshotToAMIs.get(ebd.getSnapshotId()); + if (amis == null) { + amis = new ArrayList(); + snapshotToAMIs.put(ebd.getSnapshotId(), amis); + } + amis.add(image.getImageId()); + } + } + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSVolumeJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSVolumeJanitorCrawler.java new file mode 100644 index 00000000..d41face6 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/EBSVolumeJanitorCrawler.java @@ -0,0 +1,117 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.crawler; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.ec2.model.Tag; +import com.amazonaws.services.ec2.model.Volume; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.VolumeTaggingMonkey; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.janitor.JanitorMonkey; + +/** + * The crawler to crawl AWS EBS volumes for janitor monkey. + */ +public class EBSVolumeJanitorCrawler extends AbstractAWSJanitorCrawler { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(EBSVolumeJanitorCrawler.class); + + /** + * The constructor. + * @param awsClient the AWS client + */ + public EBSVolumeJanitorCrawler(AWSClient awsClient) { + super(awsClient); + } + + @Override + public EnumSet resourceTypes() { + return EnumSet.of(AWSResourceType.EBS_VOLUME); + } + + @Override + public List resources(Enum resourceType) { + if ("EBS_VOLUME".equals(resourceType.name())) { + return getVolumeResources(); + } + return Collections.emptyList(); + } + + @Override + public List resources(String... resourceIds) { + return getVolumeResources(resourceIds); + } + + private List getVolumeResources(String... volumeIds) { + List resources = new LinkedList(); + + AWSClient awsClient = getAWSClient(); + + for (Volume volume : awsClient.describeVolumes(volumeIds)) { + Resource volumeResource = new AWSResource().withId(volume.getVolumeId()) + .withRegion(getAWSClient().region()).withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(volume.getCreateTime()); + for (Tag tag : volume.getTags()) { + LOGGER.info(String.format("Adding tag %s = %s to resource %s", + tag.getKey(), tag.getValue(), volumeResource.getId())); + volumeResource.setTag(tag.getKey(), tag.getValue()); + } + volumeResource.setOwnerEmail(getOwnerEmailForResource(volumeResource)); + volumeResource.setDescription(getVolumeDescription(volume)); + ((AWSResource) volumeResource).setAWSResourceState(volume.getState()); + resources.add(volumeResource); + } + return resources; + } + + private String getVolumeDescription(Volume volume) { + StringBuilder description = new StringBuilder(); + Integer size = volume.getSize(); + description.append(String.format("size=%s", size == null ? "unknown" : size)); + for (Tag tag : volume.getTags()) { + description.append(String.format("; %s=%s", tag.getKey(), tag.getValue())); + } + return description.toString(); + } + + @Override + public String getOwnerEmailForResource(Resource resource) { + String owner = super.getOwnerEmailForResource(resource); + if (owner == null) { + // try to find the owner from Janitor Metadata tag set by the volume tagging monkey. + Map janitorTag = VolumeTaggingMonkey.parseJanitorMetaTag(resource.getTag( + JanitorMonkey.JANITOR_META_TAG)); + owner = janitorTag.get(JanitorMonkey.OWNER_TAG_KEY); + } + return owner; + } + + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/InstanceJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/InstanceJanitorCrawler.java new file mode 100644 index 00000000..706e39e0 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/InstanceJanitorCrawler.java @@ -0,0 +1,124 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.autoscaling.model.AutoScalingInstanceDetails; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Tag; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +/** + * The crawler to crawl AWS instances for janitor monkey. + */ +public class InstanceJanitorCrawler extends AbstractAWSJanitorCrawler { + + /** The name representing the additional field name of ASG's name. */ + public static final String INSTANCE_FIELD_ASG_NAME = "ASG_NAME"; + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceJanitorCrawler.class); + + /** + * Instantiates a new basic instance crawler. + * @param awsClient + * the aws client + */ + public InstanceJanitorCrawler(AWSClient awsClient) { + super(awsClient); + } + + @Override + public EnumSet resourceTypes() { + return EnumSet.of(AWSResourceType.INSTANCE); + } + + @Override + public List resources(Enum resourceType) { + if ("INSTANCE".equals(resourceType.name())) { + return getInstanceResources(); + } + return Collections.emptyList(); + } + + @Override + public List resources(String... resourceIds) { + return getInstanceResources(resourceIds); + } + + private List getInstanceResources(String... instanceIds) { + List resources = new LinkedList(); + + AWSClient awsClient = getAWSClient(); + Map idToASGInstance = new HashMap(); + for (AutoScalingInstanceDetails instanceDetails : awsClient.describeAutoScalingInstances(instanceIds)) { + idToASGInstance.put(instanceDetails.getInstanceId(), instanceDetails); + } + + for (Instance instance : awsClient.describeInstances(instanceIds)) { + Resource instanceResource = new AWSResource().withId(instance.getInstanceId()) + .withRegion(getAWSClient().region()).withResourceType(AWSResourceType.INSTANCE) + .withLaunchTime(instance.getLaunchTime()); + for (Tag tag : instance.getTags()) { + instanceResource.setTag(tag.getKey(), tag.getValue()); + } + String description = String.format("type=%s; host=%s", instance.getInstanceType(), + instance.getPublicDnsName() == null ? "" : instance.getPublicDnsName()); + instanceResource.setDescription(description); + instanceResource.setOwnerEmail(getOwnerEmailForResource(instanceResource)); + + String asgName = getAsgName(instanceResource, idToASGInstance); + if (asgName != null) { + instanceResource.setAdditionalField(INSTANCE_FIELD_ASG_NAME, asgName); + LOGGER.info(String.format("instance %s has a ASG tag name %s.", instanceResource.getId(), asgName)); + } + if (instance.getState() != null) { + ((AWSResource) instanceResource).setAWSResourceState(instance.getState().getName()); + } + resources.add(instanceResource); + } + return resources; + } + + private String getAsgName(Resource instanceResource, Map idToASGInstance) { + String asgName = instanceResource.getTag("aws:autoscaling:groupName"); + if (asgName == null) { + // At most times the aws:autoscaling:groupName tag has the ASG name, but there are cases + // that the instance is not correctly tagged and we can find the ASG name from AutoScaling + // service. + AutoScalingInstanceDetails instanceDetails = idToASGInstance.get(instanceResource.getId()); + if (instanceDetails != null) { + asgName = instanceDetails.getAutoScalingGroupName(); + } + } + return asgName; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/ASGInstanceValidator.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/ASGInstanceValidator.java new file mode 100644 index 00000000..2a948d41 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/ASGInstanceValidator.java @@ -0,0 +1,110 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.rule.asg; + +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.appinfo.InstanceInfo; +import com.netflix.appinfo.InstanceInfo.InstanceStatus; +import com.netflix.discovery.DiscoveryClient; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; + +/** + * The class is for checking whether an ASG has any active instance. If Discovery/Eureka is enabled, + * it uses its service to check if the instances in the ASG are registered and up there. + */ +public class ASGInstanceValidator { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ASGInstanceValidator.class); + private final DiscoveryClient discoveryClient; + + /** + * Constructor. + * @param discoveryClient + * the client to access the Discovery/Eureka service for checking the status of instances. + * If Discovery/Eureka is not enabled, the client is null. + */ + public ASGInstanceValidator(DiscoveryClient discoveryClient) { + this.discoveryClient = discoveryClient; + } + + /** + * Checks whether an ASG resource contains any active instances. + * @param resource the ASG resource + * @return true if the ASG contains any active instances, false otherwise. + */ + public boolean hasActiveInstance(Resource resource) { + String instanceIds = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_INSTANCES); + String maxSizeStr = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE); + if (StringUtils.isBlank(instanceIds)) { + if (maxSizeStr != null && Integer.parseInt(maxSizeStr) == 0) { + // The ASG is empty when it has not instance and the max size of the ASG is 0. + // If the max size is not 0, the ASG could probably be in the process of starting new instances. + LOGGER.info(String.format("ASG %s is empty.", resource.getId())); + return false; + } else { + LOGGER.info(String.format("ASG %s does not have instances but the max size is %s", + resource.getId(), maxSizeStr)); + return true; + } + } + String[] instances = StringUtils.split(instanceIds, ","); + LOGGER.debug(String.format("Checking if the %d instances in ASG %s are active.", + instances.length, resource.getId())); + for (String instanceId : instances) { + if (isActiveInstance(instanceId)) { + LOGGER.info(String.format("ASG %s has active instance.", resource.getId())); + return true; + } + } + LOGGER.info(String.format("ASG %s has no active instance.", resource.getId())); + return false; + } + + /** + * Returns true if the instance is registered in Eureka/Discovery. + * @param instanceId the instance id + * @return true if the instance is active, false otherwise + */ + private boolean isActiveInstance(String instanceId) { + Validate.notNull(instanceId); + LOGGER.debug(String.format("Checking if instance %s is active", instanceId)); + if (discoveryClient == null) { + // When Discovery is not enabled, any existing instance is considered active. + LOGGER.debug(String.format("Instance %s is active since Discovery is not enabled.", instanceId)); + return true; + } else { + List instanceInfos = discoveryClient.getInstancesById(instanceId); + for (InstanceInfo info : instanceInfos) { + InstanceStatus status = info.getStatus(); + if (status == InstanceStatus.UP || status == InstanceStatus.STARTING) { + LOGGER.debug(String.format("Instance %s is active in Discovery.", instanceId)); + return true; + } + } + } + LOGGER.debug(String.format("Instance %s is not active in Discovery.", instanceId)); + return false; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/OldEmptyASGRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/OldEmptyASGRule.java new file mode 100644 index 00000000..a0708321 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/OldEmptyASGRule.java @@ -0,0 +1,129 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.asg; + +import java.util.Date; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.discovery.DiscoveryClient; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule for detecting the ASGs that 1) have old launch configurations and + * 2) do not have any instances or all instances are inactive in Eureka. + * 3) are not fronted with any ELBs. + */ +public class OldEmptyASGRule implements Rule { + + private final MonkeyCalendar calendar; + private final int retentionDays; + private final int launchConfigAgeThreshold; + private final ASGInstanceValidator instanceValidator; + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(OldEmptyASGRule.class); + + /** + * Constructor. + * + * @param calendar + * The calendar used to calculate the termination time + * @param retentionDays + * The number of days that the marked ASG is retained before being terminated + * @param launchConfigAgeThreshold + * The number of days that the launch configuration for the ASG has been created that makes the ASG be + * considered obsolete + * @param discoveryClient + * The Discovery client used to check if an instance is registered + */ + public OldEmptyASGRule(MonkeyCalendar calendar, int launchConfigAgeThreshold, int retentionDays, + DiscoveryClient discoveryClient) { + Validate.notNull(calendar); + Validate.isTrue(retentionDays >= 0); + Validate.isTrue(launchConfigAgeThreshold >= 0); + this.calendar = calendar; + this.retentionDays = retentionDays; + this.launchConfigAgeThreshold = launchConfigAgeThreshold; + this.instanceValidator = new ASGInstanceValidator(discoveryClient); + } + + /** {@inheritDoc} */ + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + if (!"ASG".equals(resource.getResourceType().name())) { + return true; + } + + if (StringUtils.isNotEmpty(resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_ELBS))) { + LOGGER.info(String.format("ASG %s has ELBs.", resource.getId())); + return true; + } + + if (instanceValidator.hasActiveInstance(resource)) { + LOGGER.info(String.format("ASG %s has active instance.", resource.getId())); + return true; + } + + String lcName = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (StringUtils.isEmpty(lcName)) { + LOGGER.error(String.format("Failed to find launch configuration for ASG %s", resource.getId())); + markResource(resource, now); + return false; + } + + String lcCreationTime = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME); + if (StringUtils.isEmpty(lcCreationTime)) { + LOGGER.error(String.format("Failed to find creation time for launch configuration %s", lcName)); + return true; + } + + DateTime createTime = new DateTime(Long.parseLong(lcCreationTime)); + if (now.isBefore(createTime.plusDays(launchConfigAgeThreshold))) { + LOGGER.info(String.format("The launch configuation %s has not been created for more than %d days", + lcName, launchConfigAgeThreshold)); + return true; + } + LOGGER.info(String.format("The launch configuation %s has been created for more than %d days", + lcName, launchConfigAgeThreshold)); + markResource(resource, now); + return false; + } + + private void markResource(Resource resource, DateTime now) { + if (resource.getExpectedTerminationTime() == null) { + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(String.format( + "Launch config older than %d days. Not in Discovery. No ELB.", + launchConfigAgeThreshold + retentionDays)); + } else { + LOGGER.info(String.format("Resource %s is already marked as cleanup candidate.", resource.getId())); + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/SuspendedASGRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/SuspendedASGRule.java new file mode 100644 index 00000000..3f13f091 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/asg/SuspendedASGRule.java @@ -0,0 +1,111 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.asg; + +import java.util.Date; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.discovery.DiscoveryClient; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule for detecting the ASGs that 1) have old launch configurations and + * 2) do not have any instances or all instances are inactive in Eureka. + * 3) are not fronted with any ELBs. + */ +public class SuspendedASGRule implements Rule { + + private final MonkeyCalendar calendar; + private final int retentionDays; + private final int suspensionAgeThreshold; + private final ASGInstanceValidator instanceValidator; + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(SuspendedASGRule.class); + + /** + * Constructor. + * + * @param calendar + * The calendar used to calculate the termination time + * @param retentionDays + * The number of days that the marked ASG is retained before being terminated after + * being marked + * @param suspensionAgeThreshold + * The number of days that the ASG has been suspended from ELB that makes the ASG be + * considered a cleanup candidate + * @param discoveryClient + * The Discovery client used to check if an instance is registered + */ + public SuspendedASGRule(MonkeyCalendar calendar, int suspensionAgeThreshold, int retentionDays, + DiscoveryClient discoveryClient) { + Validate.notNull(calendar); + Validate.isTrue(retentionDays >= 0); + Validate.isTrue(suspensionAgeThreshold >= 0); + this.calendar = calendar; + this.retentionDays = retentionDays; + this.suspensionAgeThreshold = suspensionAgeThreshold; + this.instanceValidator = new ASGInstanceValidator(discoveryClient); + } + + /** {@inheritDoc} */ + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + if (!"ASG".equals(resource.getResourceType().name())) { + return true; + } + + if (instanceValidator.hasActiveInstance(resource)) { + return true; + } + + String suspensionTimeStr = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME); + if (!StringUtils.isEmpty(suspensionTimeStr)) { + DateTime createTime = ASGJanitorCrawler.SUSPENSION_TIME_FORMATTER.parseDateTime(suspensionTimeStr); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (now.isBefore(createTime.plusDays(suspensionAgeThreshold))) { + LOGGER.info(String.format("The ASG %s has not been suspended for more than %d days", + resource.getId(), suspensionAgeThreshold)); + return true; + } + LOGGER.info(String.format("The ASG %s has been suspended for more than %d days", + resource.getId(), suspensionAgeThreshold)); + if (resource.getExpectedTerminationTime() == null) { + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(String.format( + "User suspended age more than %d days and all instances are out of service in Discovery", + suspensionAgeThreshold + retentionDays)); + } + return false; + } else { + LOGGER.info(String.format("ASG %s is not suspended from ELB.", resource.getId())); + return true; + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/instance/OrphanedInstanceRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/instance/OrphanedInstanceRule.java new file mode 100644 index 00000000..fe43165b --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/instance/OrphanedInstanceRule.java @@ -0,0 +1,123 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.instance; + +import java.util.Date; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.janitor.crawler.InstanceJanitorCrawler; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule for checking the orphaned instances that do not belong to any ASGs and + * launched for certain days. + */ +public class OrphanedInstanceRule implements Rule { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(OrphanedInstanceRule.class); + + private static final String TERMINATION_REASON = "No ASG is associated with this instance"; + + private final MonkeyCalendar calendar; + + private final int instanceAgeThreshold; + + private final int retentionDaysWithOwner; + + private final int retentionDaysWithoutOwner; + + /** + * Constructor for OrphanedInstanceRule. + * + * @param calendar + * The calendar used to calculate the termination time + * @param instanceAgeThreshold + * The number of days that an instance is considered as orphaned since it is launched + * @param retentionDaysWithOwner + * The number of days that the orphaned instance is retained before being terminated + * when the instance has an owner specified + * @param retentionDaysWithoutOwner + * The number of days that the orphaned instance is retained before being terminated + * when the instance has no owner specified + */ + public OrphanedInstanceRule(MonkeyCalendar calendar, + int instanceAgeThreshold, int retentionDaysWithOwner, int retentionDaysWithoutOwner) { + Validate.notNull(calendar); + Validate.isTrue(instanceAgeThreshold >= 0); + Validate.isTrue(retentionDaysWithOwner >= 0); + Validate.isTrue(retentionDaysWithoutOwner >= 0); + this.calendar = calendar; + this.instanceAgeThreshold = instanceAgeThreshold; + this.retentionDaysWithOwner = retentionDaysWithOwner; + this.retentionDaysWithoutOwner = retentionDaysWithoutOwner; + } + + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + if (!resource.getResourceType().name().equals("INSTANCE")) { + // The rule is supposed to only work on AWS instances. If a non-instance resource + // is passed to the rule, the rule simply ignores it and considers it as a valid + // resource not for cleanup. + return true; + } + String awsStatus = ((AWSResource) resource).getAWSResourceState(); + if (!"running".equals(awsStatus) || "pending".equals(awsStatus)) { + return true; + } + AWSResource instanceResource = (AWSResource) resource; + String asgName = instanceResource.getAdditionalField(InstanceJanitorCrawler.INSTANCE_FIELD_ASG_NAME); + if (StringUtils.isEmpty(asgName)) { + if (resource.getLaunchTime() == null) { + LOGGER.error(String.format("The instance %s has no launch time.", resource.getId())); + return true; + } else { + DateTime launchTime = new DateTime(resource.getLaunchTime().getTime()); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (now.isBefore(launchTime.plusDays(instanceAgeThreshold))) { + LOGGER.info(String.format("The orphaned instance %s has not launched for more than %d days", + resource.getId(), instanceAgeThreshold)); + return true; + } + LOGGER.info(String.format("The orphaned instance %s has launched for more than %d days", + resource.getId(), instanceAgeThreshold)); + if (resource.getExpectedTerminationTime() == null) { + int retentionDays = retentionDaysWithoutOwner; + if (resource.getOwnerEmail() != null) { + retentionDays = retentionDaysWithOwner; + } + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(TERMINATION_REASON); + } + return false; + } + } + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/NoGeneratedAMIRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/NoGeneratedAMIRule.java new file mode 100644 index 00000000..9c949134 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/NoGeneratedAMIRule.java @@ -0,0 +1,142 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.rule.snapshot; + +import java.util.Date; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.janitor.crawler.EBSSnapshotJanitorCrawler; +import com.netflix.simianarmy.janitor.JanitorMonkey; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule is for checking whether an EBS snapshot has any AMIs generated from it. + * If there are no AMIs generated using the snapshot and the snapshot is created + * for certain days, it is marked as a cleanup candidate by this rule. + */ +public class NoGeneratedAMIRule implements Rule { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(NoGeneratedAMIRule.class); + + private static final String TERMINATION_REASON = "No AMI is generated for this snapshot"; + + private final MonkeyCalendar calendar; + + private final int ageThreshold; + + private final int retentionDays; + + /** The date format used to print or parse the user specified termination date. **/ + public static final DateTimeFormatter TERMINATION_DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd"); + + /** + * Constructor. + * + * @param calendar + * The calendar used to calculate the termination time + * @param ageThreshold + * The number of days that a snapshot is considered as cleanup candidate since it is created + * @param retentionDays + * The number of days that the volume is retained before being terminated after being marked + * as cleanup candidate + */ + public NoGeneratedAMIRule(MonkeyCalendar calendar, int ageThreshold, int retentionDays) { + Validate.notNull(calendar); + Validate.isTrue(ageThreshold >= 0); + Validate.isTrue(retentionDays >= 0); + this.calendar = calendar; + this.ageThreshold = ageThreshold; + this.retentionDays = retentionDays; + } + + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + if (!resource.getResourceType().name().equals("EBS_SNAPSHOT")) { + return true; + } + if (!"completed".equals(((AWSResource) resource).getAWSResourceState())) { + return true; + } + String janitorTag = resource.getTag(JanitorMonkey.JANITOR_TAG); + if (janitorTag != null) { + if ("donotmark".equals(janitorTag)) { + LOGGER.info(String.format("The snapshot %s is tagged as not handled by Janitor", + resource.getId())); + return true; + } + try { + // Owners can tag the volume with a termination date in the "janitor" tag. + Date userSpecifiedDate = new Date(TERMINATION_DATE_FORMATTER.parseDateTime(janitorTag).getMillis()); + resource.setExpectedTerminationTime(userSpecifiedDate); + resource.setTerminationReason(String.format("User specified termination date %s", janitorTag)); + return false; + } catch (Exception e) { + LOGGER.error(String.format("The janitor tag is not a user specified date: %s", janitorTag)); + } + } + + if (hasGeneratedImage(resource)) { + return true; + } + + if (resource.getLaunchTime() == null) { + LOGGER.error(String.format("Snapshot %s does not have a creation time.", resource.getId())); + return true; + } + DateTime launchTime = new DateTime(resource.getLaunchTime().getTime()); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (launchTime.plusDays(ageThreshold).isBefore(now)) { + if (resource.getExpectedTerminationTime() == null) { + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(TERMINATION_REASON); + LOGGER.info(String.format( + "Snapshot %s is marked to be cleaned at %s as there is no AMI generated using it", + resource.getId(), resource.getExpectedTerminationTime())); + } else { + LOGGER.info(String.format("Resource %s is already marked.", resource.getId())); + } + return false; + } + return true; + + } + + /** + * Gets the AMI created using the snapshot. This method can be overriden by subclasses + * if they use a different way to check this. + * @param resource the snapshot resource + * @return true if there are AMIs that are created using the snapshot, false otherwise + */ + protected boolean hasGeneratedImage(Resource resource) { + return StringUtils.isNotEmpty(resource.getAdditionalField(EBSSnapshotJanitorCrawler.SNAPSHOT_FIELD_AMIS)); + } + +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/volume/OldDetachedVolumeRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/volume/OldDetachedVolumeRule.java new file mode 100644 index 00000000..7c5ea23b --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/volume/OldDetachedVolumeRule.java @@ -0,0 +1,142 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.rule.volume; + +import java.util.Date; +import java.util.Map; + +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.janitor.VolumeTaggingMonkey; +import com.netflix.simianarmy.janitor.JanitorMonkey; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule is for checking whether an EBS volume is detached for more than + * certain days. The rule mostly relies on tags on the volume to decide if + * the volume should be marked. + */ +public class OldDetachedVolumeRule implements Rule { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(OldDetachedVolumeRule.class); + + private final MonkeyCalendar calendar; + + private final int detachDaysThreshold; + + private final int retentionDays; + + /** The date format used to print or parse the user specified termination date. **/ + public static final DateTimeFormatter TERMINATION_DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd"); + + + /** + * Constructor. + * + * @param calendar + * The calendar used to calculate the termination time + * @param detachDaysThreshold + * The number of days that a volume is considered as cleanup candidate since it is detached + * @param retentionDays + * The number of days that the volume is retained before being terminated after being marked + * as cleanup candidate + */ + public OldDetachedVolumeRule(MonkeyCalendar calendar, int detachDaysThreshold, int retentionDays) { + Validate.notNull(calendar); + Validate.isTrue(detachDaysThreshold >= 0); + Validate.isTrue(retentionDays >= 0); + this.calendar = calendar; + this.detachDaysThreshold = detachDaysThreshold; + this.retentionDays = retentionDays; + } + + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + if (!resource.getResourceType().name().equals("EBS_VOLUME")) { + return true; + } + if (!"available".equals(((AWSResource) resource).getAWSResourceState())) { + return true; + } + String janitorTag = resource.getTag(JanitorMonkey.JANITOR_TAG); + if (janitorTag != null) { + if ("donotmark".equals(janitorTag)) { + LOGGER.info(String.format("The volume %s is tagged as not handled by Janitor", + resource.getId())); + return true; + } + try { + // Owners can tag the volume with a termination date in the "janitor" tag. + Date userSpecifiedDate = new Date( + TERMINATION_DATE_FORMATTER.parseDateTime(janitorTag).getMillis()); + resource.setExpectedTerminationTime(userSpecifiedDate); + resource.setTerminationReason(String.format("User specified termination date %s", janitorTag)); + return false; + } catch (Exception e) { + LOGGER.error(String.format("The janitor tag is not a user specified date: %s", janitorTag)); + } + } + + String janitorMetaTag = resource.getTag(JanitorMonkey.JANITOR_META_TAG); + if (janitorMetaTag == null) { + LOGGER.info(String.format("Volume %s is not tagged with the Janitor meta information, ignore.", + resource.getId())); + return true; + } + + Map metadata = VolumeTaggingMonkey.parseJanitorMetaTag(janitorMetaTag); + String detachTimeTag = metadata.get(JanitorMonkey.DETACH_TIME_TAG_KEY); + if (detachTimeTag == null) { + return true; + } + DateTime detachTime = null; + try { + detachTime = AWSResource.DATE_FORMATTER.parseDateTime(detachTimeTag); + } catch (Exception e) { + LOGGER.error(String.format("Detach time in the JANITOR_META tag of %s is not in the valid format: %s", + resource.getId(), detachTime)); + return true; + } + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (detachTime != null && detachTime.plusDays(detachDaysThreshold).isBefore(now)) { + if (resource.getExpectedTerminationTime() == null) { + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(String.format("Volume not attached for %d days", + detachDaysThreshold + retentionDays)); + LOGGER.info(String.format( + "Volume %s is marked to be cleaned at %s as it is detached for more than %d days", + resource.getId(), resource.getExpectedTerminationTime(), detachDaysThreshold)); + } else { + LOGGER.info(String.format("Resource %s is already marked.", resource.getId())); + } + return false; + } + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicCalendar.java b/src/main/java/com/netflix/simianarmy/basic/BasicCalendar.java index 9e309b38..053a5340 100644 --- a/src/main/java/com/netflix/simianarmy/basic/BasicCalendar.java +++ b/src/main/java/com/netflix/simianarmy/basic/BasicCalendar.java @@ -18,10 +18,13 @@ package com.netflix.simianarmy.basic; import java.util.Calendar; +import java.util.Date; import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; +import org.apache.commons.lang.Validate; + import com.netflix.simianarmy.Monkey; import com.netflix.simianarmy.MonkeyCalendar; import com.netflix.simianarmy.MonkeyConfiguration; @@ -145,44 +148,44 @@ protected void loadHolidays(int year) { // not be in the office to respond to rampaging monkeys // new years, or closest work day - holidays.add(workDayInYear(Calendar.JANUARY, 1)); + holidays.add(workDayInYear(year, Calendar.JANUARY, 1)); // 3rd monday == MLK Day - holidays.add(dayOfYear(Calendar.JANUARY, Calendar.MONDAY, 3)); + holidays.add(dayOfYear(year, Calendar.JANUARY, Calendar.MONDAY, 3)); // 3rd monday == Presidents Day - holidays.add(dayOfYear(Calendar.FEBRUARY, Calendar.MONDAY, 3)); + holidays.add(dayOfYear(year, Calendar.FEBRUARY, Calendar.MONDAY, 3)); // last monday == Memorial Day - holidays.add(dayOfYear(Calendar.MAY, Calendar.MONDAY, -1)); + holidays.add(dayOfYear(year, Calendar.MAY, Calendar.MONDAY, -1)); // 4th of July, or closest work day - holidays.add(workDayInYear(Calendar.JULY, 4)); + holidays.add(workDayInYear(year, Calendar.JULY, 4)); // first monday == Labor Day - holidays.add(dayOfYear(Calendar.SEPTEMBER, Calendar.MONDAY, 1)); + holidays.add(dayOfYear(year, Calendar.SEPTEMBER, Calendar.MONDAY, 1)); // second monday == Columbus Day - holidays.add(dayOfYear(Calendar.OCTOBER, Calendar.MONDAY, 2)); + holidays.add(dayOfYear(year, Calendar.OCTOBER, Calendar.MONDAY, 2)); // veterans day, Nov 11th, or closest work day - holidays.add(workDayInYear(Calendar.NOVEMBER, 11)); + holidays.add(workDayInYear(year, Calendar.NOVEMBER, 11)); // 4th thursday == Thanksgiving - holidays.add(dayOfYear(Calendar.NOVEMBER, Calendar.THURSDAY, 4)); + holidays.add(dayOfYear(year, Calendar.NOVEMBER, Calendar.THURSDAY, 4)); // 4th friday == "black friday", monkey goes shopping! - holidays.add(dayOfYear(Calendar.NOVEMBER, Calendar.FRIDAY, 4)); + holidays.add(dayOfYear(year, Calendar.NOVEMBER, Calendar.FRIDAY, 4)); // christmas eve - holidays.add(dayOfYear(Calendar.DECEMBER, 24)); + holidays.add(dayOfYear(year, Calendar.DECEMBER, 24)); // christmas day - holidays.add(dayOfYear(Calendar.DECEMBER, 25)); + holidays.add(dayOfYear(year, Calendar.DECEMBER, 25)); // day after christmas - holidays.add(dayOfYear(Calendar.DECEMBER, 26)); + holidays.add(dayOfYear(year, Calendar.DECEMBER, 26)); // new years eve - holidays.add(dayOfYear(Calendar.DECEMBER, 31)); + holidays.add(dayOfYear(year, Calendar.DECEMBER, 31)); // mark the holiday set with the year, so on Jan 1 it will automatically // recalculate the holidays for next year @@ -192,14 +195,17 @@ protected void loadHolidays(int year) { /** * Day of year. * + * @param year + * the year * @param month * the month * @param day * the day * @return the day of the year */ - private int dayOfYear(int month, int day) { + private int dayOfYear(int year, int month, int day) { Calendar holiday = now(); + holiday.set(Calendar.YEAR, year); holiday.set(Calendar.MONTH, month); holiday.set(Calendar.DAY_OF_MONTH, day); return holiday.get(Calendar.DAY_OF_YEAR); @@ -208,6 +214,8 @@ private int dayOfYear(int month, int day) { /** * Day of year. * + * @param year + * the year * @param month * the month * @param dayOfWeek @@ -216,8 +224,9 @@ private int dayOfYear(int month, int day) { * the week in month * @return the day of the year */ - private int dayOfYear(int month, int dayOfWeek, int weekInMonth) { + private int dayOfYear(int year, int month, int dayOfWeek, int weekInMonth) { Calendar holiday = now(); + holiday.set(Calendar.YEAR, year); holiday.set(Calendar.MONTH, month); holiday.set(Calendar.DAY_OF_WEEK, dayOfWeek); holiday.set(Calendar.DAY_OF_WEEK_IN_MONTH, weekInMonth); @@ -227,14 +236,17 @@ private int dayOfYear(int month, int dayOfWeek, int weekInMonth) { /** * Work day in year. * + * @param year + * the year * @param month * the month * @param day * the day * @return the day of the year adjusted to the closest workday */ - private int workDayInYear(int month, int day) { + private int workDayInYear(int year, int month, int day) { Calendar holiday = now(); + holiday.set(Calendar.YEAR, year); holiday.set(Calendar.MONTH, month); holiday.set(Calendar.DAY_OF_MONTH, day); int doy = holiday.get(Calendar.DAY_OF_YEAR); @@ -251,4 +263,21 @@ private int workDayInYear(int month, int day) { return doy; } + @Override + public Date getBusinessDay(Date date, int n) { + Validate.isTrue(n >= 0); + Calendar calendar = now(); + calendar.setTime(date); + while (isHoliday(calendar) || isWeekend(calendar) || n-- > 0) { + calendar.add(Calendar.DATE, 1); + } + return calendar.getTime(); + } + + private boolean isWeekend(Calendar calendar) { + int dow = calendar.get(Calendar.DAY_OF_WEEK); + return dow == Calendar.SATURDAY + || dow == Calendar.SUNDAY; + } + } diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicChaosMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/BasicChaosMonkeyContext.java new file mode 100644 index 00000000..ff55c1b7 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/BasicChaosMonkeyContext.java @@ -0,0 +1,98 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic; + +import com.netflix.simianarmy.basic.chaos.BasicChaosInstanceSelector; +import com.netflix.simianarmy.chaos.ChaosCrawler; +import com.netflix.simianarmy.chaos.ChaosEmailNotifier; +import com.netflix.simianarmy.chaos.ChaosInstanceSelector; +import com.netflix.simianarmy.chaos.ChaosMonkey; +import com.netflix.simianarmy.client.aws.chaos.ASGChaosCrawler; + +/** + * The Class BasicContext. This provide the basic context needed for the Chaos Monkey to run. It will configure + * the Chaos Monkey based on a simianarmy.properties file and chaos.properties. The properties file can be + * overriden with -Dsimianarmy.properties=/path/to/my.properties + */ +public class BasicChaosMonkeyContext extends BasicSimianArmyContext implements ChaosMonkey.Context { + + /** The crawler. */ + private ChaosCrawler crawler; + + /** The selector. */ + private ChaosInstanceSelector selector; + + /** The chaos email notifier. */ + private ChaosEmailNotifier chaosEmailNotifier; + + /** + * Instantiates a new basic context. + */ + public BasicChaosMonkeyContext() { + super("simianarmy.properties", "client.properties", "chaos.properties"); + setChaosCrawler(new ASGChaosCrawler(awsClient())); + setChaosInstanceSelector(new BasicChaosInstanceSelector()); + } + + /** {@inheritDoc} */ + @Override + public ChaosCrawler chaosCrawler() { + return crawler; + } + + /** + * Sets the chaos crawler. + * + * @param chaosCrawler + * the new chaos crawler + */ + protected void setChaosCrawler(ChaosCrawler chaosCrawler) { + this.crawler = chaosCrawler; + } + + /** {@inheritDoc} */ + @Override + public ChaosInstanceSelector chaosInstanceSelector() { + return selector; + } + + /** + * Sets the chaos instance selector. + * + * @param chaosInstanceSelector + * the new chaos instance selector + */ + protected void setChaosInstanceSelector(ChaosInstanceSelector chaosInstanceSelector) { + this.selector = chaosInstanceSelector; + } + + @Override + public ChaosEmailNotifier chaosEmailNotifier() { + return chaosEmailNotifier; + } + + /** + * Sets the chaos email notifier. + * + * @param notifier + * the chaos email notifier + */ + protected void setChaosEmailNotifier(ChaosEmailNotifier notifier) { + this.chaosEmailNotifier = notifier; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicContext.java b/src/main/java/com/netflix/simianarmy/basic/BasicContext.java deleted file mode 100644 index 7349ca68..00000000 --- a/src/main/java/com/netflix/simianarmy/basic/BasicContext.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * - * Copyright 2012 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.simianarmy.basic; - -import java.io.InputStream; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.netflix.simianarmy.aws.SimpleDBRecorder; -import com.netflix.simianarmy.basic.chaos.BasicChaosInstanceSelector; -import com.netflix.simianarmy.client.aws.AWSClient; -import com.netflix.simianarmy.client.aws.chaos.ASGChaosCrawler; - -/** - * The Class BasicContext. This provide the basic context needed for the monkeys to run. It will configure the monkeys - * based on a simianarmy.properties file. The properties file can be override with - * -Dsimianarmy.properties=/path/to/my.properties - */ -public class BasicContext extends BasicContextShell { - - /** The Constant LOGGER. */ - private static final Logger LOGGER = LoggerFactory.getLogger(BasicContext.class); - - /** The client configuration properties. */ - private Properties props = new Properties(); - - /** The Constant MONKEY_THREADS. */ - private static final int MONKEY_THREADS = 1; - - /** The client used to interact with the target machines. */ - private AWSClient awsClient; - - public AWSClient getAwsClient() { - return awsClient; - } - - public void setAwsClient(AWSClient awsClient) { - this.awsClient = awsClient; - } - - /** loads all relevant configuration files to configure the client. */ - protected void addClientConfigurationProperties() { - loadClientConfigurationIntoProperties("simianarmy.properties"); - loadClientConfigurationIntoProperties("client.properties"); - } - - /** loads the given config ontop of the config read by previous calls. */ - protected void loadClientConfigurationIntoProperties(String propertyFileName) { - String propFile = System.getProperty(propertyFileName, "/" + propertyFileName); - try { - InputStream is = BasicContext.class.getResourceAsStream(propFile); - try { - props.load(is); - } finally { - is.close(); - } - } catch (Exception e) { - String msg = "Unable to load properties file " + propFile + " set System property \"" + propertyFileName - + "\" to valid file"; - LOGGER.error(msg); - throw new RuntimeException(msg, e); - } - } - - /** - * Instantiates a new basic context. - */ - public BasicContext() { - BasicConfiguration config = loadClientConfig(); - - setCalendar(new BasicCalendar(config)); - - createClient(config); - - createScheduler(config); - - setChaosCrawler(new ASGChaosCrawler(this.awsClient)); - setChaosInstanceSelector(new BasicChaosInstanceSelector()); - - createRecorder(config); - } - - private BasicConfiguration loadClientConfig() { - addClientConfigurationProperties(); - BasicConfiguration config = new BasicConfiguration(props); - setConfiguration(config); - return config; - } - - private void createScheduler(BasicConfiguration config) { - int freq = (int) config.getNumOrElse("simianarmy.scheduler.frequency", 1); - TimeUnit freqUnit = TimeUnit.valueOf(config.getStrOrElse("simianarmy.scheduler.frequencyUnit", "HOURS")); - int threads = (int) config.getNumOrElse("simianarmy.scheduler.threads", MONKEY_THREADS); - setScheduler(new BasicScheduler(freq, freqUnit, threads)); - } - - private void createRecorder(BasicConfiguration config) { - String account = config.getStr("simianarmy.client.aws.accountKey"); - String secret = config.getStr("simianarmy.client.aws.secretKey"); - String region = config.getStrOrElse("simianarmy.client.aws.region", "us-east-1"); - String domain = config.getStrOrElse("simianarmy.sdb.domain", "SIMIAN_ARMY"); - setRecorder(new SimpleDBRecorder(account, secret, region, domain)); - } - - /** - * Create the specific client. Override to provide your own client. - */ - protected void createClient(BasicConfiguration config) { - String account = config.getStr("simianarmy.client.aws.accountKey"); - String secret = config.getStr("simianarmy.client.aws.secretKey"); - String region = config.getStrOrElse("simianarmy.client.aws.region", "us-east-1"); - this.awsClient = new AWSClient(account, secret, region); - - setCloudClient(this.awsClient); - } - -} diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicContextShell.java b/src/main/java/com/netflix/simianarmy/basic/BasicContextShell.java deleted file mode 100644 index 884bf513..00000000 --- a/src/main/java/com/netflix/simianarmy/basic/BasicContextShell.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * - * Copyright 2012 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.simianarmy.basic; - -import java.util.LinkedList; - -import com.netflix.simianarmy.CloudClient; -import com.netflix.simianarmy.MonkeyCalendar; -import com.netflix.simianarmy.MonkeyConfiguration; -import com.netflix.simianarmy.MonkeyRecorder; -import com.netflix.simianarmy.MonkeyRecorder.Event; -import com.netflix.simianarmy.MonkeyScheduler; -import com.netflix.simianarmy.chaos.ChaosCrawler; -import com.netflix.simianarmy.chaos.ChaosEmailNotifier; -import com.netflix.simianarmy.chaos.ChaosInstanceSelector; -import com.netflix.simianarmy.chaos.ChaosMonkey; - -/** - * The Class BasicContextShell. - */ -public class BasicContextShell implements ChaosMonkey.Context { - - /** The scheduler. */ - private MonkeyScheduler scheduler; - - /** The calendar. */ - private MonkeyCalendar calendar; - - /** The config. */ - private MonkeyConfiguration config; - - /** The client. */ - private CloudClient client; - - /** The crawler. */ - private ChaosCrawler crawler; - - /** The selector. */ - private ChaosInstanceSelector selector; - - /** The recorder. */ - private MonkeyRecorder recorder; - - /** The chaos email notifier. */ - private ChaosEmailNotifier chaosEmailNotifier; - - /** The reported events. */ - private LinkedList eventReport; - - /** protected constructor as the Shell is meant to be subclassed. */ - protected BasicContextShell() { - eventReport = new LinkedList(); - } - - @Override - public void reportEvent(Event evt) { - this.eventReport.add(evt); - } - - @Override - public void resetEventReport() { - eventReport.clear(); - } - - @Override - public String getEventReport() { - StringBuilder report = new StringBuilder(); - - for (Event event : this.eventReport) { - report.append(event.eventType()).append(" ").append(event.id()).append(" (") - .append(event.field("groupType")).append(", ").append(event.field("groupName")).append(")\n"); - } - return report.toString(); - } - - /** {@inheritDoc} */ - @Override - public MonkeyScheduler scheduler() { - return scheduler; - } - - /** - * Sets the scheduler. - * - * @param scheduler - * the new scheduler - */ - protected void setScheduler(MonkeyScheduler scheduler) { - this.scheduler = scheduler; - } - - /** {@inheritDoc} */ - @Override - public MonkeyCalendar calendar() { - return calendar; - } - - /** - * Sets the calendar. - * - * @param calendar - * the new calendar - */ - protected void setCalendar(MonkeyCalendar calendar) { - this.calendar = calendar; - } - - /** {@inheritDoc} */ - @Override - public MonkeyConfiguration configuration() { - return config; - } - - /** - * Sets the configuration. - * - * @param configuration - * the new configuration - */ - protected void setConfiguration(MonkeyConfiguration configuration) { - this.config = configuration; - } - - /** {@inheritDoc} */ - @Override - public CloudClient cloudClient() { - return client; - } - - /** - * Sets the cloud client. - * - * @param cloudClient - * the new cloud client - */ - protected void setCloudClient(CloudClient cloudClient) { - this.client = cloudClient; - } - - /** {@inheritDoc} */ - @Override - public ChaosCrawler chaosCrawler() { - return crawler; - } - - /** - * Sets the chaos crawler. - * - * @param chaosCrawler - * the new chaos crawler - */ - protected void setChaosCrawler(ChaosCrawler chaosCrawler) { - this.crawler = chaosCrawler; - } - - /** {@inheritDoc} */ - @Override - public ChaosInstanceSelector chaosInstanceSelector() { - return selector; - } - - /** - * Sets the chaos instance selector. - * - * @param chaosInstanceSelector - * the new chaos instance selector - */ - protected void setChaosInstanceSelector(ChaosInstanceSelector chaosInstanceSelector) { - this.selector = chaosInstanceSelector; - } - - /** {@inheritDoc} */ - @Override - public MonkeyRecorder recorder() { - return recorder; - } - - /** - * Sets the recorder. - * - * @param recorder - * the new recorder - */ - protected void setRecorder(MonkeyRecorder recorder) { - this.recorder = recorder; - } - - @Override - public ChaosEmailNotifier chaosEmailNotifier() { - return chaosEmailNotifier; - } - - /** - * Sets the chaos email notifier. - * - * @param notifier - * the chaos email notifier - */ - protected void setChaosEmailNotifier(ChaosEmailNotifier notifier) { - this.chaosEmailNotifier = notifier; - } - -} diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java b/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java index 6ed6cd60..7b0aefe6 100644 --- a/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java +++ b/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java @@ -28,7 +28,11 @@ import org.slf4j.LoggerFactory; import com.netflix.simianarmy.MonkeyRunner; +import com.netflix.simianarmy.aws.janitor.VolumeTaggingMonkey; import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey; +import com.netflix.simianarmy.basic.janitor.BasicJanitorMonkey; +import com.netflix.simianarmy.basic.janitor.BasicJanitorMonkeyContext; +import com.netflix.simianarmy.basic.janitor.BasicVolumeTaggingMonkeyContext; /** * Will periodically run the configured monkeys. @@ -44,17 +48,22 @@ public class BasicMonkeyServer extends HttpServlet { */ @SuppressWarnings("unchecked") public void addMonkeysToRun() { - RUNNER.replaceMonkey(getMonkeyClass(), this.clientContextClass); + LOGGER.info("Adding Chaos Monkey."); + RUNNER.replaceMonkey(getChaosMonkeyClass(), this.chaosContextClass); + LOGGER.info("Adding Volume Tagging Monkey."); + RUNNER.replaceMonkey(VolumeTaggingMonkey.class, BasicVolumeTaggingMonkeyContext.class); + LOGGER.info("Adding Janitor Monkey."); + RUNNER.replaceMonkey(BasicJanitorMonkey.class, BasicJanitorMonkeyContext.class); } /** * make the class of the client object configurable. */ @SuppressWarnings("rawtypes") - private Class clientContextClass = com.netflix.simianarmy.basic.BasicContext.class; + private Class chaosContextClass = com.netflix.simianarmy.basic.BasicChaosMonkeyContext.class; @SuppressWarnings("rawtypes") - protected Class getMonkeyClass() { + protected Class getChaosMonkeyClass() { return BasicChaosMonkey.class; } @@ -84,11 +93,11 @@ private void loadClientContextClass(Properties clientConfig) throws ServletExcep try { String clientContextClassName = clientConfig.getProperty(clientContextClassKey); if (clientContextClassName == null || clientContextClassName.isEmpty()) { - LOGGER.info("using standard client " + this.clientContextClass.getCanonicalName()); + LOGGER.info("using standard client " + this.chaosContextClass.getCanonicalName()); return; } - this.clientContextClass = classLoader.loadClass(clientContextClassName); - LOGGER.info("as " + clientContextClassKey + " loaded " + clientContextClass.getCanonicalName()); + this.chaosContextClass = classLoader.loadClass(clientContextClassName); + LOGGER.info("as " + clientContextClassKey + " loaded " + chaosContextClass.getCanonicalName()); } catch (ClassNotFoundException e) { throw new ServletException("Could not load " + clientContextClassKey, e); } @@ -126,7 +135,12 @@ private Properties loadClientConfigProperties() throws ServletException { @Override public void destroy() { RUNNER.stop(); - RUNNER.removeMonkey(getMonkeyClass()); + LOGGER.info("Stopping Chaos Monkey."); + RUNNER.removeMonkey(getChaosMonkeyClass()); + LOGGER.info("Stopping volume tagging Monkey."); + RUNNER.removeMonkey(VolumeTaggingMonkey.class); + LOGGER.info("Stopping Janitor Monkey."); + RUNNER.removeMonkey(BasicJanitorMonkey.class); super.destroy(); } } diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java b/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java new file mode 100644 index 00000000..5cb84d20 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java @@ -0,0 +1,296 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic; + +import java.io.InputStream; +import java.util.LinkedList; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.CloudClient; +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.MonkeyScheduler; +import com.netflix.simianarmy.aws.SimpleDBRecorder; +import com.netflix.simianarmy.client.aws.AWSClient; + +/** + * The Class BasicSimianArmyContext. + */ +public class BasicSimianArmyContext implements Monkey.Context { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicSimianArmyContext.class); + + /** The configuration properties. */ + private final Properties properties = new Properties(); + + /** The Constant MONKEY_THREADS. */ + private static final int MONKEY_THREADS = 1; + + /** The scheduler. */ + private MonkeyScheduler scheduler; + + /** The calendar. */ + private MonkeyCalendar calendar; + + /** The config. */ + private BasicConfiguration config; + + /** The client. */ + private AWSClient client; + + /** The recorder. */ + private MonkeyRecorder recorder; + + /** The reported events. */ + private final LinkedList eventReport; + + private final String account; + + private final String secret; + + private final String region; + + /** protected constructor as the Shell is meant to be subclassed. */ + protected BasicSimianArmyContext(String... configFiles) { + eventReport = new LinkedList(); + // Load the config files into props following the provided order. + for (String configFile : configFiles) { + loadConfigurationFileIntoProperties(configFile); + } + LOGGER.info("The folowing are properties in the context."); + for (Entry prop : properties.entrySet()) { + LOGGER.info(String.format("%s = %s", prop.getKey(), prop.getValue())); + } + + config = new BasicConfiguration(properties); + calendar = new BasicCalendar(config); + + account = config.getStr("simianarmy.client.aws.accountKey"); + secret = config.getStr("simianarmy.client.aws.secretKey"); + region = config.getStrOrElse("simianarmy.client.aws.region", "us-east-1"); + + createClient(); + + createScheduler(); + + createRecorder(); + + } + + /** loads the given config on top of the config read by previous calls. */ + protected void loadConfigurationFileIntoProperties(String propertyFileName) { + String propFile = System.getProperty(propertyFileName, "/" + propertyFileName); + try { + InputStream is = BasicSimianArmyContext.class.getResourceAsStream(propFile); + try { + properties.load(is); + } finally { + is.close(); + } + } catch (Exception e) { + String msg = "Unable to load properties file " + propFile + " set System property \"" + propertyFileName + + "\" to valid file"; + LOGGER.error(msg); + throw new RuntimeException(msg, e); + } + } + + private void createScheduler() { + int freq = (int) config.getNumOrElse("simianarmy.scheduler.frequency", 1); + TimeUnit freqUnit = TimeUnit.valueOf(config.getStrOrElse("simianarmy.scheduler.frequencyUnit", "HOURS")); + int threads = (int) config.getNumOrElse("simianarmy.scheduler.threads", MONKEY_THREADS); + setScheduler(new BasicScheduler(freq, freqUnit, threads)); + } + + private void createRecorder() { + String domain = config.getStrOrElse("simianarmy.recorder.sdb.domain", "SIMIAN_ARMY"); + if (client != null) { + setRecorder(new SimpleDBRecorder(client, domain)); + } + } + + /** + * Create the specific client. Override to provide your own client. + */ + protected void createClient() { + if (StringUtils.isEmpty(account) || StringUtils.isEmpty(secret)) { + return; + } + this.client = new AWSClient(account, secret, region); + setCloudClient(this.client); + } + + /** + * Gets the AWS client. + * @return the AWS client + */ + public AWSClient awsClient() { + return client; + } + + /** + * Gets the AWS account. + * @return the AWS account + */ + protected String account() { + return account; + } + + /** + * Gets the region. + * @return the region + */ + public String region() { + return region; + } + + /** + * Gets the AWS secret. + * @return the AWS secret + */ + protected String secret() { + return secret; + } + + @Override + public void reportEvent(Event evt) { + this.eventReport.add(evt); + } + + @Override + public void resetEventReport() { + eventReport.clear(); + } + + @Override + public String getEventReport() { + StringBuilder report = new StringBuilder(); + for (Event event : this.eventReport) { + report.append(String.format("%s %s (", event.eventType(), event.id())); + boolean isFirst = true; + for (Entry field : event.fields().entrySet()) { + if (!isFirst) { + report.append(", "); + } else { + isFirst = false; + } + report.append(String.format("%s:%s", field.getKey(), field.getValue())); + } + report.append(")\n"); + } + return report.toString(); + } + + /** {@inheritDoc} */ + @Override + public MonkeyScheduler scheduler() { + return scheduler; + } + + /** + * Sets the scheduler. + * + * @param scheduler + * the new scheduler + */ + protected void setScheduler(MonkeyScheduler scheduler) { + this.scheduler = scheduler; + } + + /** {@inheritDoc} */ + @Override + public MonkeyCalendar calendar() { + return calendar; + } + + /** + * Sets the calendar. + * + * @param calendar + * the new calendar + */ + protected void setCalendar(MonkeyCalendar calendar) { + this.calendar = calendar; + } + + /** {@inheritDoc} */ + @Override + public MonkeyConfiguration configuration() { + return config; + } + + /** + * Sets the configuration. + * + * @param configuration + * the new configuration + */ + protected void setConfiguration(MonkeyConfiguration configuration) { + this.config = (BasicConfiguration) configuration; + } + + /** {@inheritDoc} */ + @Override + public CloudClient cloudClient() { + return client; + } + + /** + * Sets the cloud client. + * + * @param cloudClient + * the new cloud client + */ + protected void setCloudClient(CloudClient cloudClient) { + this.client = (AWSClient) cloudClient; + } + + /** {@inheritDoc} */ + @Override + public MonkeyRecorder recorder() { + return recorder; + } + + /** + * Sets the recorder. + * + * @param recorder + * the new recorder + */ + protected void setRecorder(MonkeyRecorder recorder) { + this.recorder = recorder; + } + + /** + * Gets the configuration properties. + * @return the configuration properties + */ + protected Properties getProperties() { + return this.properties; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorEmailBuilder.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorEmailBuilder.java new file mode 100644 index 00000000..86c95e68 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorEmailBuilder.java @@ -0,0 +1,141 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.janitor; + +import java.util.Collection; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.janitor.JanitorEmailBuilder; + +/** The basic implementation of the email builder for Janitor monkey. */ +public class BasicJanitorEmailBuilder extends JanitorEmailBuilder { + private static final String[] TABLE_COLUMNS = + {"Resource Type", "Resource", "Region", "Description", "Expected Termination Time", + "Termination Reason", "View/Edit"}; + private static final String AHREF_TEMPLATE = "%s"; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("EEE, MMM dd, yyyy"); + + private Map> emailToResources; + + @Override + public void setEmailToResources(Map> emailToResources) { + Validate.notNull(emailToResources); + this.emailToResources = emailToResources; + } + + @Override + protected String getHeader() { + StringBuilder header = new StringBuilder(); + header.append("

Janitor Notifications

"); + header.append( + "The following resource(s) have been marked for cleanup by Janitor monkey " + + "as potential unused resources. This is a non-repeating notification.
"); + return header.toString(); + } + + @Override + protected String getEntryTable(String emailAddress) { + StringBuilder table = new StringBuilder(); + table.append(getHtmlTableHeader(getTableColumns())); + for (Resource resource : emailToResources.get(emailAddress)) { + table.append(getResourceRow(resource)); + } + table.append("
"); + return table.toString(); + } + + @Override + protected String getFooter() { + return "
Janitor Monkey wiki: https://github.com/Netflix/SimianArmy/wiki
"; + } + + /** + * Gets the url to view the details of the resource. + * @param resource the resource + * @return the url to view/edit the resource. + */ + protected String getResourceUrl(Resource resource) { + return null; + } + + /** + * Gets the string when displaying the resource, e.g. the id. + * @param resource the resource to display + * @return the string to represent the resource + */ + protected String getResourceDisplay(Resource resource) { + return resource.getId(); + } + + /** + * Gets the url to edit the Janitor termination of the resource. + * @param resource the resource + * @return the url to edit the Janitor termination the resource. + */ + protected String getJanitorResourceUrl(Resource resource) { + return null; + } + + /** Gets the table columns for the table in the email. + * + * @return the array of column names + */ + protected String[] getTableColumns() { + return TABLE_COLUMNS; + } + + /** + * Gets the row for a resource in the table in the email body. + * @param resource the resource to display + * @return the table row in the email body + */ + protected String getResourceRow(Resource resource) { + StringBuilder message = new StringBuilder(); + message.append(""); + message.append(getHtmlCell(resource.getResourceType().name())); + String resourceUrl = getResourceUrl(resource); + if (!StringUtils.isEmpty(resourceUrl)) { + message.append(getHtmlCell(String.format(AHREF_TEMPLATE, resourceUrl, getResourceDisplay(resource)))); + } else { + message.append(getHtmlCell(getResourceDisplay(resource))); + } + message.append(getHtmlCell(resource.getRegion())); + if (resource.getDescription() == null) { + message.append(getHtmlCell("")); + } else { + message.append(getHtmlCell(resource.getDescription().replace(";", "
").replace(",", "
"))); + } + message.append(getHtmlCell(DATE_FORMATTER.print(resource.getExpectedTerminationTime().getTime()))); + message.append(getHtmlCell(resource.getTerminationReason())); + String janitorUrl = getJanitorResourceUrl(resource); + if (!StringUtils.isEmpty(janitorUrl)) { + message.append(getHtmlCell(String.format(AHREF_TEMPLATE, janitorUrl, "View/Extend"))); + } else { + message.append(getHtmlCell("")); + } + message.append(""); + return message.toString(); + } + +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java new file mode 100644 index 00000000..23337f54 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java @@ -0,0 +1,216 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.janitor; + +import java.util.Collection; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.janitor.AbstractJanitor; +import com.netflix.simianarmy.janitor.JanitorEmailNotifier; +import com.netflix.simianarmy.janitor.JanitorMonkey; +import com.netflix.simianarmy.janitor.JanitorResourceTracker; + +/** The basic implementation of Janitor Monkey. */ +public class BasicJanitorMonkey extends JanitorMonkey { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicJanitorMonkey.class); + + /** The Constant NS. */ + private static final String NS = "simianarmy.janitor."; + + /** The cfg. */ + private final MonkeyConfiguration cfg; + + private final List janitors; + + private final JanitorEmailNotifier emailNotifier; + + private final String region; + + private final JanitorResourceTracker resourceTracker; + + private final MonkeyRecorder recorder; + + private final MonkeyCalendar calendar; + + /** + * Instantiates a new basic janitor monkey. + * + * @param ctx + * the ctx + */ + public BasicJanitorMonkey(Context ctx) { + super(ctx); + this.cfg = ctx.configuration(); + + janitors = ctx.janitors(); + emailNotifier = ctx.emailNotifier(); + region = ctx.region(); + resourceTracker = ctx.resourceTracker(); + recorder = ctx.recorder(); + calendar = ctx.calendar(); + } + + /** {@inheritDoc} */ + @Override + public void doMonkeyBusiness() { + cfg.reload(); + context().resetEventReport(); + + if (!isJanitorMonkeyEnabled()) { + return; + } else { + LOGGER.info(String.format("Marking resources with %d janitors.", janitors.size())); + for (AbstractJanitor janitor : janitors) { + janitor.markResources(); + LOGGER.info(String.format("Marked %d resources of type %s in the last run.", + janitor.getMarkedResources().size(), janitor.getResourceType().name())); + LOGGER.info(String.format("Unmarked %d resources of type %s in the last run.", + janitor.getUnmarkedResources().size(), janitor.getResourceType())); + } + + if (!cfg.getBoolOrElse("simianarmy.janitor.leashed", true)) { + emailNotifier.sendNotifications(); + } else { + LOGGER.info("Janitor Monkey is leashed, no notification is sent."); + } + + LOGGER.info(String.format("Cleaning resources with %d janitors.", janitors.size())); + for (AbstractJanitor janitor : janitors) { + janitor.cleanupResources(); + LOGGER.info(String.format("Cleaned %d resources of type %s in the last run.", + janitor.getCleanedResources().size(), janitor.getResourceType())); + LOGGER.info(String.format("Failed to clean %d resources of type %s in the last run.", + janitor.getFailedToCleanResources().size(), janitor.getResourceType())); + } + sendJanitorSummaryEmail(); + } + } + + @Override + public Event optInResource(String resourceId) { + return optInOrOutResource(resourceId, true); + } + + @Override + public Event optOutResource(String resourceId) { + return optInOrOutResource(resourceId, false); + } + + private Event optInOrOutResource(String resourceId, boolean optIn) { + Resource resource = resourceTracker.getResource(resourceId); + if (resource == null) { + return null; + } + EventTypes eventType = optIn ? EventTypes.OPT_IN_RESOURCE : EventTypes.OPT_OUT_RESOURCE; + long timestamp = calendar.now().getTimeInMillis(); + // The same resource can have multiple events, so we add the timestamp to the id. + Event evt = recorder.newEvent(Type.JANITOR, eventType, region, resourceId + "@" + timestamp); + recorder.recordEvent(evt); + resource.setOptOutOfJanitor(!optIn); + resourceTracker.addOrUpdate(resource); + return evt; + } + + /** + * Send a summary email with about the last run of the janitor monkey. + */ + protected void sendJanitorSummaryEmail() { + String summaryEmailTarget = cfg.getStr(NS + "summaryEmail.to"); + if (!StringUtils.isEmpty(summaryEmailTarget)) { + if (!emailNotifier.isValidEmail(summaryEmailTarget)) { + LOGGER.error(String.format("The email target address '%s' for Janitor summary email is invalid", + summaryEmailTarget)); + return; + } + StringBuilder message = new StringBuilder(); + for (AbstractJanitor janitor : janitors) { + Enum resourceType = janitor.getResourceType(); + appendSummary(message, "markings", resourceType, janitor.getMarkedResources()); + appendSummary(message, "unmarkings", resourceType, janitor.getUnmarkedResources()); + appendSummary(message, "cleanups", resourceType, janitor.getCleanedResources()); + appendSummary(message, "cleanup failures", resourceType, janitor.getFailedToCleanResources()); + } + String subject = getSummaryEmailSubject(); + emailNotifier.sendEmail(summaryEmailTarget, subject, message.toString()); + } + } + + private void appendSummary(StringBuilder message, String summaryName, + Enum resourceType, Collection resources) { + message.append(String.format("Total %s for %s = %d
", + summaryName, resourceType.name(), resources.size())); + message.append(String.format("List: %s
", printResources(resources))); + } + + private String printResources(Collection resources) { + StringBuilder sb = new StringBuilder(); + boolean isFirst = true; + for (Resource r : resources) { + if (!isFirst) { + sb.append(","); + } else { + isFirst = false; + } + sb.append(r.getId()); + } + return sb.toString(); + } + + /** + * Gets the summary email subject for the last run of janitor monkey. + * @return the subject of the summary email + */ + protected String getSummaryEmailSubject() { + return String.format("Janitor monkey execution summary (%s)", region); + } + + /** + * Handle cleanup error. This has been abstracted so subclasses can decide to continue causing chaos if desired. + * + * @param resource + * the instance + * @param e + * the exception + */ + protected void handleCleanupError(Resource resource, Throwable e) { + String msg = String.format("Failed to clean up %s resource %s with error %s", + resource.getResourceType(), resource.getId(), e.getMessage()); + LOGGER.error(msg); + throw new RuntimeException(msg, e); + } + + private boolean isJanitorMonkeyEnabled() { + String prop = NS + "enabled"; + if (cfg.getBoolOrElse(prop, true)) { + return true; + } + LOGGER.info("JanitorMonkey disabled, set {}=true", prop); + return false; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java new file mode 100644 index 00000000..6216c971 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java @@ -0,0 +1,367 @@ +/* + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// CHECKSTYLE IGNORE MagicNumberCheck +package com.netflix.simianarmy.basic.janitor; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.netflix.discovery.DiscoveryClient; +import com.netflix.discovery.DiscoveryManager; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.aws.janitor.ASGJanitor; +import com.netflix.simianarmy.aws.janitor.EBSSnapshotJanitor; +import com.netflix.simianarmy.aws.janitor.EBSVolumeJanitor; +import com.netflix.simianarmy.aws.janitor.InstanceJanitor; +import com.netflix.simianarmy.aws.janitor.SimpleDBJanitorResourceTracker; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.crawler.EBSSnapshotJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.crawler.EBSVolumeJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.crawler.InstanceJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.asg.OldEmptyASGRule; +import com.netflix.simianarmy.aws.janitor.rule.asg.SuspendedASGRule; +import com.netflix.simianarmy.aws.janitor.rule.instance.OrphanedInstanceRule; +import com.netflix.simianarmy.aws.janitor.rule.snapshot.NoGeneratedAMIRule; +import com.netflix.simianarmy.aws.janitor.rule.volume.OldDetachedVolumeRule; +import com.netflix.simianarmy.basic.BasicSimianArmyContext; +import com.netflix.simianarmy.janitor.AbstractJanitor; +import com.netflix.simianarmy.janitor.JanitorCrawler; +import com.netflix.simianarmy.janitor.JanitorEmailBuilder; +import com.netflix.simianarmy.janitor.JanitorEmailNotifier; +import com.netflix.simianarmy.janitor.JanitorMonkey; +import com.netflix.simianarmy.janitor.JanitorResourceTracker; +import com.netflix.simianarmy.janitor.JanitorRuleEngine; + +/** + * The basic implementation of the context class for Janitor monkey. + */ +public class BasicJanitorMonkeyContext extends BasicSimianArmyContext implements JanitorMonkey.Context { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicJanitorMonkeyContext.class); + + /** The email notifier. */ + private final JanitorEmailNotifier emailNotifier; + + private final JanitorResourceTracker janitorResourceTracker; + + /** The janitors. */ + private final List janitors; + + private final String monkeyRegion; + + private final MonkeyCalendar monkeyCalendar; + + private final AmazonSimpleEmailServiceClient sesClient; + + private final JanitorEmailBuilder janitorEmailBuilder; + + private final String defaultEmail; + + private final String[] ccEmails; + + private final String sourceEmail; + + private final int daysBeforeTermination; + + /** + * The constructor. + */ + public BasicJanitorMonkeyContext() { + super("simianarmy.properties", "client.properties", "janitor.properties"); + + monkeyRegion = region(); + monkeyCalendar = calendar(); + + String resourceDomain = configuration().getStrOrElse("simianarmy.janitor.resources.sdb.domain", "SIMIAN_ARMY"); + + Set enabledResourceSet = getEnabledResourceSet(); + + janitorResourceTracker = new SimpleDBJanitorResourceTracker(awsClient(), resourceDomain); + + janitorEmailBuilder = new BasicJanitorEmailBuilder(); + sesClient = new AmazonSimpleEmailServiceClient( + new BasicAWSCredentials(account(), secret())); + defaultEmail = configuration().getStrOrElse("simianarmy.janitor.notification.defaultEmail", ""); + ccEmails = StringUtils.split( + configuration().getStrOrElse("simianarmy.janitor.notification.ccEmails", ""), ","); + sourceEmail = configuration().getStrOrElse("simianarmy.janitor.notification.sourceEmail", ""); + daysBeforeTermination = + (int) configuration().getNumOrElse("simianarmy.janitor.notification.daysBeforeTermination", 3); + + emailNotifier = new JanitorEmailNotifier(getJanitorEmailNotifierContext()); + + janitors = new ArrayList(); + if (enabledResourceSet.contains("ASG")) { + janitors.add(getASGJanitor()); + } + + if (enabledResourceSet.contains("INSTANCE")) { + janitors.add(getInstanceJanitor()); + } + + if (enabledResourceSet.contains("EBS_VOLUME")) { + janitors.add(getEBSVolumeJanitor()); + } + + if (enabledResourceSet.contains("EBS_SNAPSHOT")) { + janitors.add(getEBSSnapshotJanitor()); + } + } + + private ASGJanitor getASGJanitor() { + JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine(); + boolean discoveryEnabled = configuration().getBoolOrElse("simianarmy.janitor.Eureka.enabled", false); + DiscoveryClient discoveryClient; + if (discoveryEnabled) { + LOGGER.info("Initializing Discovery client."); + discoveryClient = DiscoveryManager.getInstance().getDiscoveryClient(); + } else { + LOGGER.info("Discovery/Eureka is not enabled."); + discoveryClient = null; + } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.oldEmptyASGRule.enabled", false)) { + ruleEngine.addRule(new OldEmptyASGRule(monkeyCalendar, + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.oldEmptyASGRule.launchConfigAgeThreshold", 50), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.oldEmptyASGRule.retentionDays", 10), + discoveryClient + )); + } + + if (configuration().getBoolOrElse("simianarmy.janitor.rule.suspendedASGRule.enabled", false)) { + ruleEngine.addRule(new SuspendedASGRule(monkeyCalendar, + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.suspendedASGRule.suspensionAgeThreshold", 2), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.suspendedASGRule.retentionDays", 5), + discoveryClient + )); + } + JanitorCrawler asgCrawler = new ASGJanitorCrawler(awsClient()); + BasicJanitorContext asgJanitorCtx = new BasicJanitorContext( + monkeyRegion, ruleEngine, asgCrawler, janitorResourceTracker, + monkeyCalendar, configuration(), recorder()); + return new ASGJanitor(awsClient(), asgJanitorCtx); + } + + private InstanceJanitor getInstanceJanitor() { + JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine(); + if (configuration().getBoolOrElse("simianarmy.janitor.rule.orphanedInstanceRule.enabled", false)) { + ruleEngine.addRule(new OrphanedInstanceRule(monkeyCalendar, + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.orphanedInstanceRule.instanceAgeThreshold", 2), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler instanceCrawler = new InstanceJanitorCrawler(awsClient()); + BasicJanitorContext instanceJanitorCtx = new BasicJanitorContext( + monkeyRegion, ruleEngine, instanceCrawler, janitorResourceTracker, + monkeyCalendar, configuration(), recorder()); + return new InstanceJanitor(awsClient(), instanceJanitorCtx); + } + + private EBSVolumeJanitor getEBSVolumeJanitor() { + JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine(); + if (configuration().getBoolOrElse("simianarmy.janitor.rule.oldDetachedVolumeRule.enabled", false)) { + ruleEngine.addRule(new OldDetachedVolumeRule(monkeyCalendar, + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.oldDetachedVolumeRule.detachDaysThreshold", 30), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.oldDetachedVolumeRule.retentionDays", 7))); + } + JanitorCrawler volumeCrawler = new EBSVolumeJanitorCrawler(awsClient()); + BasicJanitorContext volumeJanitorCtx = new BasicJanitorContext( + monkeyRegion, ruleEngine, volumeCrawler, janitorResourceTracker, + monkeyCalendar, configuration(), recorder()); + return new EBSVolumeJanitor(awsClient(), volumeJanitorCtx); + } + + private EBSSnapshotJanitor getEBSSnapshotJanitor() { + JanitorRuleEngine ruleEngine = new BasicJanitorRuleEngine(); + if (configuration().getBoolOrElse("simianarmy.janitor.rule.noGeneratedAMIRule.enabled", false)) { + ruleEngine.addRule(new NoGeneratedAMIRule(monkeyCalendar, + (int) configuration().getNumOrElse("simianarmy.janitor.rule.noGeneratedAMIRule.ageThreshold", 30), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.noGeneratedAMIRule.retentionDays", 7))); + } + JanitorCrawler snapshotCrawler = new EBSSnapshotJanitorCrawler(awsClient()); + BasicJanitorContext snapshotJanitorCtx = new BasicJanitorContext( + monkeyRegion, ruleEngine, snapshotCrawler, janitorResourceTracker, + monkeyCalendar, configuration(), recorder()); + return new EBSSnapshotJanitor(awsClient(), snapshotJanitorCtx); + } + + private Set getEnabledResourceSet() { + Set enabledResourceSet = new HashSet(); + String enabledResources = configuration().getStr("simianarmy.janitor.enabledResources"); + if (StringUtils.isNotBlank(enabledResources)) { + for (String resourceType : enabledResources.split(",")) { + enabledResourceSet.add(resourceType.trim().toUpperCase()); + } + } + return enabledResourceSet; + } + + public JanitorEmailNotifier.Context getJanitorEmailNotifierContext() { + return new JanitorEmailNotifier.Context() { + @Override + public AmazonSimpleEmailServiceClient sesClient() { + return sesClient; + } + + @Override + public String defaultEmail() { + return defaultEmail; + } + + @Override + public int daysBeforeTermination() { + return daysBeforeTermination; + } + + @Override + public String region() { + return monkeyRegion; + } + + @Override + public JanitorResourceTracker resourceTracker() { + return janitorResourceTracker; + } + + @Override + public JanitorEmailBuilder emailBuilder() { + return janitorEmailBuilder; + } + + @Override + public MonkeyCalendar calendar() { + return monkeyCalendar; + } + + @Override + public String[] ccEmails() { + return ccEmails; + } + + @Override + public String sourceEmail() { + return sourceEmail; + } + }; + } + + /** {@inheritDoc} */ + @Override + public List janitors() { + return janitors; + } + + /** {@inheritDoc} */ + @Override + public JanitorEmailNotifier emailNotifier() { + return emailNotifier; + } + + @Override + public JanitorResourceTracker resourceTracker() { + return janitorResourceTracker; + } + + /** The Context class for Janitor. + */ + public static class BasicJanitorContext implements AbstractJanitor.Context { + private final String region; + private final JanitorRuleEngine ruleEngine; + private final JanitorCrawler crawler; + private final JanitorResourceTracker resourceTracker; + private final MonkeyCalendar calendar; + private final MonkeyConfiguration config; + private final MonkeyRecorder recorder; + + /** + * Constructor. + * @param region the region of the janitor + * @param ruleEngine the rule engine used by the janitor + * @param crawler the crawler used by the janitor + * @param resourceTracker the resource tracker used by the janitor + * @param calendar the calendar used by the janitor + * @param config the monkey configuration used by the janitor + */ + public BasicJanitorContext(String region, JanitorRuleEngine ruleEngine, JanitorCrawler crawler, + JanitorResourceTracker resourceTracker, MonkeyCalendar calendar, MonkeyConfiguration config, + MonkeyRecorder recorder) { + this.region = region; + this.resourceTracker = resourceTracker; + this.ruleEngine = ruleEngine; + this.crawler = crawler; + this.calendar = calendar; + this.config = config; + this.recorder = recorder; + } + + @Override + public String region() { + return region; + } + + @Override + public MonkeyConfiguration configuration() { + return config; + } + + @Override + public MonkeyCalendar calendar() { + return calendar; + } + + @Override + public JanitorRuleEngine janitorRuleEngine() { + return ruleEngine; + } + + @Override + public JanitorCrawler janitorCrawler() { + return crawler; + } + + @Override + public JanitorResourceTracker janitorResourceTracker() { + return resourceTracker; + } + + @Override + public MonkeyRecorder recorder() { + return recorder; + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorRuleEngine.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorRuleEngine.java new file mode 100644 index 00000000..1e8a49b6 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorRuleEngine.java @@ -0,0 +1,98 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.basic.janitor; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.janitor.JanitorRuleEngine; +import com.netflix.simianarmy.janitor.Rule; + +/** + * Basic implementation of janitor rule engine that runs all containing rules to decide if a resource should be + * a candidate of cleanup. + */ +public class BasicJanitorRuleEngine implements JanitorRuleEngine { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicJanitorRuleEngine.class); + + /** The rules to decide if a resource should be a candidate for cleanup. **/ + private final List rules; + + /** + * The constructor of JanitorRuleEngine. + */ + public BasicJanitorRuleEngine() { + rules = new ArrayList(); + } + + /** + * Decides whether the resource should be a candidate of cleanup based on the underlying rules. If any rule in the + * rule set thinks the resource should be a candidate of cleanup, the method returns false which indicates that the + * resource should be marked for cleanup. If multiple rules think the resource should be cleaned up, the rule with + * the nearest expected termination time fills the termination reason and expected termination time. + * + * @param resource + * The resource + * @return true if the resource is valid and should not be a candidate of cleanup based on the underlying rules, + * false otherwise. + */ + @Override + public boolean isValid(Resource resource) { + LOGGER.debug(String.format("Checking if resource %s of type %s is a cleanup candidate against %d rules.", + resource.getId(), resource.getResourceType(), rules.size())); + // We create a clone of the resource each time when we try the rule. In the first iteration of the rules + // we identify the rule with the nearest termination date if there is any rule considers the resource + // as a cleanup candidate. Then the rule is applied to the original resource. + Rule nearestRule = null; + if (rules.size() == 1) { + nearestRule = rules.get(0); + } else { + Date nearestTerminationTime = null; + for (Rule rule : rules) { + Resource clone = resource.cloneResource(); + if (!rule.isValid(clone) && (nearestTerminationTime == null + || nearestTerminationTime.after(clone.getExpectedTerminationTime()))) { + nearestRule = rule; + nearestTerminationTime = clone.getExpectedTerminationTime(); + } + } + } + if (nearestRule != null && !nearestRule.isValid(resource)) { + LOGGER.info(String.format("Resource %s is marked as a cleanup candidate.", resource.getId())); + return false; + } else { + LOGGER.info(String.format("Resource %s is not marked as a cleanup candidate.", resource.getId())); + return true; + } + } + + /** {@inheritDoc} */ + @Override + public BasicJanitorRuleEngine addRule(Rule rule) { + rules.add(rule); + return this; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicVolumeTaggingMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicVolumeTaggingMonkeyContext.java new file mode 100644 index 00000000..c3c6d9db --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicVolumeTaggingMonkeyContext.java @@ -0,0 +1,33 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.janitor; + +import com.netflix.simianarmy.aws.janitor.VolumeTaggingMonkey; +import com.netflix.simianarmy.basic.BasicSimianArmyContext; + +/** The basic context for the monkey that tags volumes with Janitor meta data. + */ +public class BasicVolumeTaggingMonkeyContext extends BasicSimianArmyContext implements VolumeTaggingMonkey.Context { + + /** + * The constructor. + */ + public BasicVolumeTaggingMonkeyContext() { + super("simianarmy.properties", "client.properties", "volumeTagging.properties"); + } +} diff --git a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java index ef8161cc..0cf0df4e 100644 --- a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java +++ b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java @@ -17,9 +17,15 @@ */ package com.netflix.simianarmy.client.aws; +import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.amazonaws.AmazonServiceException; import com.amazonaws.auth.AWSCredentials; @@ -27,11 +33,37 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.autoscaling.AmazonAutoScalingClient; import com.amazonaws.services.autoscaling.model.AutoScalingGroup; +import com.amazonaws.services.autoscaling.model.AutoScalingInstanceDetails; +import com.amazonaws.services.autoscaling.model.DeleteAutoScalingGroupRequest; import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest; import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult; +import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesRequest; +import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesResult; +import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsRequest; +import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsResult; +import com.amazonaws.services.autoscaling.model.LaunchConfiguration; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.model.CreateTagsRequest; +import com.amazonaws.services.ec2.model.DeleteSnapshotRequest; +import com.amazonaws.services.ec2.model.DeleteVolumeRequest; +import com.amazonaws.services.ec2.model.DescribeImagesRequest; +import com.amazonaws.services.ec2.model.DescribeImagesResult; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesResult; +import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest; +import com.amazonaws.services.ec2.model.DescribeSnapshotsResult; +import com.amazonaws.services.ec2.model.DescribeVolumesRequest; +import com.amazonaws.services.ec2.model.DescribeVolumesResult; +import com.amazonaws.services.ec2.model.Image; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.Reservation; +import com.amazonaws.services.ec2.model.Snapshot; +import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; +import com.amazonaws.services.ec2.model.Volume; +import com.amazonaws.services.simpledb.AmazonSimpleDB; +import com.amazonaws.services.simpledb.AmazonSimpleDBClient; import com.netflix.simianarmy.CloudClient; import com.netflix.simianarmy.NotFoundException; @@ -40,6 +72,9 @@ */ public class AWSClient implements CloudClient { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(AWSClient.class); + /** The credential. */ private final AWSCredentials cred; @@ -96,7 +131,7 @@ public String region() { /** * Amazon EC2 client. Abstracted to aid testing. * - * @return the amazon ec2 client + * @return the Amazon EC2 client */ protected AmazonEC2 ec2Client() { AmazonEC2 client = new AmazonEC2Client(cred); @@ -105,9 +140,9 @@ protected AmazonEC2 ec2Client() { } /** - * Amazon ASG client. Abstradted to aid testing. + * Amazon ASG client. Abstracted to aid testing. * - * @return the amazon auto scaling client + * @return the Amazon Auto Scaling client */ protected AmazonAutoScalingClient asgClient() { AmazonAutoScalingClient client = new AmazonAutoScalingClient(cred); @@ -115,6 +150,23 @@ protected AmazonAutoScalingClient asgClient() { return client; } + /** + * Amazon SimpleDB client. + * + * @return the Amazon SimpleDB client + */ + public AmazonSimpleDB sdbClient() { + AmazonSimpleDB client = new AmazonSimpleDBClient(cred); + // us-east-1 has special naming + // http://docs.amazonwebservices.com/general/latest/gr/rande.html#sdb_region + if (region == null || region.equals("us-east-1")) { + client.setEndpoint("sdb.amazonaws.com"); + } else { + client.setEndpoint("sdb." + region + ".amazonaws.com"); + } + return client; + } + /** * Describe auto scaling groups. * @@ -127,28 +179,160 @@ public List describeAutoScalingGroups() { /** * Describe a set of specific auto scaling groups. * + * @param names the ASG names * @return the auto scaling groups */ public List describeAutoScalingGroups(String... names) { + if (names == null || names.length == 0) { + LOGGER.info("Getting all auto-scaling groups."); + } else { + LOGGER.info(String.format("Getting auto-scaling groups for %d names.", names.length)); + } + List asgs = new LinkedList(); AmazonAutoScalingClient asgClient = asgClient(); DescribeAutoScalingGroupsRequest request = new DescribeAutoScalingGroupsRequest(); if (names != null) { - request.withAutoScalingGroupNames(Arrays.asList(names)); + request.setAutoScalingGroupNames(Arrays.asList(names)); } DescribeAutoScalingGroupsResult result = asgClient.describeAutoScalingGroups(request); asgs.addAll(result.getAutoScalingGroups()); while (result.getNextToken() != null) { - request = request.withNextToken(result.getNextToken()); + request.setNextToken(result.getNextToken()); result = asgClient.describeAutoScalingGroups(request); asgs.addAll(result.getAutoScalingGroups()); } + LOGGER.info(String.format("Got %d auto-scaling groups.", asgs.size())); return asgs; } + /** + * Describe a set of specific auto-scaling instances. + * + * @param instanceIds the instance ids + * @return the instances + */ + public List describeAutoScalingInstances(String... instanceIds) { + if (instanceIds == null || instanceIds.length == 0) { + LOGGER.info("Getting all auto-scaling instances."); + } else { + LOGGER.info(String.format("Getting auto-scaling instances for %d ids.", instanceIds.length)); + } + + List instances = new LinkedList(); + + AmazonAutoScalingClient asgClient = asgClient(); + DescribeAutoScalingInstancesRequest request = new DescribeAutoScalingInstancesRequest(); + if (instanceIds != null) { + request.setInstanceIds(Arrays.asList(instanceIds)); + } + DescribeAutoScalingInstancesResult result = asgClient.describeAutoScalingInstances(request); + + instances.addAll(result.getAutoScalingInstances()); + while (result.getNextToken() != null) { + request = request.withNextToken(result.getNextToken()); + result = asgClient.describeAutoScalingInstances(request); + instances.addAll(result.getAutoScalingInstances()); + } + + LOGGER.info(String.format("Got %d auto-scaling instances.", instances.size())); + return instances; + } + + /** + * Describe a set of specific instances. + * + * @param instanceIds the instance ids + * @return the instances + */ + public List describeInstances(String... instanceIds) { + if (instanceIds == null || instanceIds.length == 0) { + LOGGER.info("Getting all EC2 instances."); + } else { + LOGGER.info(String.format("Getting EC2 instances for %d ids.", instanceIds.length)); + } + + List instances = new LinkedList(); + + AmazonEC2 ec2Client = ec2Client(); + DescribeInstancesRequest request = new DescribeInstancesRequest(); + if (instanceIds != null) { + request.withInstanceIds(Arrays.asList(instanceIds)); + } + DescribeInstancesResult result = ec2Client.describeInstances(request); + for (Reservation reservation : result.getReservations()) { + instances.addAll(reservation.getInstances()); + } + + LOGGER.info(String.format("Got %d EC2 instances.", instances.size())); + return instances; + } + + /** + * Describe a set of specific launch configurations. + * + * @param names the launch configuration names + * @return the launch configurations + */ + public List describeLaunchConfigurations(String... names) { + if (names == null || names.length == 0) { + LOGGER.info("Getting all launch configurations."); + } else { + LOGGER.info(String.format("Getting launch configurations for %d names.", names.length)); + } + + List lcs = new LinkedList(); + + AmazonAutoScalingClient asgClient = asgClient(); + DescribeLaunchConfigurationsRequest request = new DescribeLaunchConfigurationsRequest() + .withLaunchConfigurationNames(names); + DescribeLaunchConfigurationsResult result = asgClient.describeLaunchConfigurations(request); + + lcs.addAll(result.getLaunchConfigurations()); + while (result.getNextToken() != null) { + request.setNextToken(result.getNextToken()); + result = asgClient.describeLaunchConfigurations(request); + lcs.addAll(result.getLaunchConfigurations()); + } + + LOGGER.info(String.format("Got %d launch configurations.", lcs.size())); + return lcs; + } + + /** {@inheritDoc} */ + @Override + public void deleteAutoScalingGroup(String asgName) { + Validate.notEmpty(asgName); + LOGGER.info(String.format("Deleting auto-scaling group with name %s.", asgName)); + AmazonAutoScalingClient asgClient = asgClient(); + DeleteAutoScalingGroupRequest request = new DeleteAutoScalingGroupRequest() + .withAutoScalingGroupName(asgName); + asgClient.deleteAutoScalingGroup(request); + } + + /** {@inheritDoc} */ + @Override + public void deleteVolume(String volumeId) { + Validate.notEmpty(volumeId); + LOGGER.info(String.format("Deleting volume %s.", volumeId)); + AmazonEC2 ec2Client = ec2Client(); + DeleteVolumeRequest request = new DeleteVolumeRequest().withVolumeId(volumeId); + ec2Client.deleteVolume(request); + } + + /** {@inheritDoc} */ + @Override + public void deleteSnapshot(String snapshotId) { + Validate.notEmpty(snapshotId); + LOGGER.info(String.format("Deleting snapshot %s.", snapshotId)); + AmazonEC2 ec2Client = ec2Client(); + DeleteSnapshotRequest request = new DeleteSnapshotRequest().withSnapshotId(snapshotId); + ec2Client.deleteSnapshot(request); + } + /** {@inheritDoc} */ @Override public void terminateInstance(String instanceId) { @@ -161,4 +345,94 @@ public void terminateInstance(String instanceId) { throw e; } } + + /** + * Describe a set of specific EBS volumes. + * + * @param volumeIds the volume ids + * @return the volumes + */ + public List describeVolumes(String... volumeIds) { + if (volumeIds == null || volumeIds.length == 0) { + LOGGER.info("Getting all EBS volumes."); + } else { + LOGGER.info(String.format("Getting EBS volumes for %d ids.", volumeIds.length)); + } + + AmazonEC2 ec2Client = ec2Client(); + DescribeVolumesRequest request = new DescribeVolumesRequest(); + if (volumeIds != null) { + request.setVolumeIds(Arrays.asList(volumeIds)); + } + DescribeVolumesResult result = ec2Client.describeVolumes(request); + List volumes = result.getVolumes(); + + LOGGER.info(String.format("Got %d EBS volumes.", volumes.size())); + return volumes; + } + + /** + * Describe a set of specific EBS snapshots. + * + * @param snapshotIds the snapshot ids + * @return the snapshots + */ + public List describeSnapshots(String... snapshotIds) { + if (snapshotIds == null || snapshotIds.length == 0) { + LOGGER.info("Getting all EBS volumes."); + } else { + LOGGER.info(String.format("Getting EBS snapshotIds for %d ids.", snapshotIds.length)); + } + + AmazonEC2 ec2Client = ec2Client(); + DescribeSnapshotsRequest request = new DescribeSnapshotsRequest(); + if (snapshotIds != null) { + request.setSnapshotIds(Arrays.asList(snapshotIds)); + } + DescribeSnapshotsResult result = ec2Client.describeSnapshots(request); + List snapshots = result.getSnapshots(); + + LOGGER.info(String.format("Got %d EBS snapshots.", snapshots.size())); + return snapshots; + } + + @Override + public void createTagsForResources(Map keyValueMap, String... resourceIds) { + Validate.notNull(keyValueMap); + Validate.notEmpty(keyValueMap); + Validate.notNull(resourceIds); + Validate.notEmpty(resourceIds); + AmazonEC2 ec2Client = ec2Client(); + List tags = new ArrayList(); + for (Map.Entry entry : keyValueMap.entrySet()) { + tags.add(new Tag(entry.getKey(), entry.getValue())); + } + CreateTagsRequest req = new CreateTagsRequest(Arrays.asList(resourceIds), tags); + ec2Client.createTags(req); + } + + /** + * Describe a set of specific images. + * + * @param imageIds the image ids + * @return the images + */ + public List describeImages(String... imageIds) { + if (imageIds == null || imageIds.length == 0) { + LOGGER.info("Getting all AMIs."); + } else { + LOGGER.info(String.format("Getting AMIs for %d ids.", imageIds.length)); + } + + AmazonEC2 ec2Client = ec2Client(); + DescribeImagesRequest request = new DescribeImagesRequest(); + if (imageIds != null) { + request.setImageIds(Arrays.asList(imageIds)); + } + DescribeImagesResult result = ec2Client.describeImages(request); + List images = result.getImages(); + + LOGGER.info(String.format("Got %d AMIs.", images.size())); + return images; + } } diff --git a/src/main/java/com/netflix/simianarmy/client/vsphere/PropertyBasedTerminationStrategy.java b/src/main/java/com/netflix/simianarmy/client/vsphere/PropertyBasedTerminationStrategy.java index 54c156d2..a13c109d 100644 --- a/src/main/java/com/netflix/simianarmy/client/vsphere/PropertyBasedTerminationStrategy.java +++ b/src/main/java/com/netflix/simianarmy/client/vsphere/PropertyBasedTerminationStrategy.java @@ -17,7 +17,7 @@ import java.rmi.RemoteException; -import com.netflix.simianarmy.basic.BasicConfiguration; +import com.netflix.simianarmy.MonkeyConfiguration; import com.vmware.vim25.mo.VirtualMachine; /** @@ -30,19 +30,19 @@ * @author ingmar.krusch@immobilienscout24.de */ public class PropertyBasedTerminationStrategy implements TerminationStrategy { - private String propertyName; - private String propertyValue; + private final String propertyName; + private final String propertyValue; /** * Reads property name simianarmy.client.vsphere.terminationStrategy.property.name * (default: Force Boot) and value simianarmy.client.vsphere.terminationStrategy.property.value * (default: server) from config. */ - public PropertyBasedTerminationStrategy(BasicConfiguration config) { - this.propertyName - = config.getStrOrElse("simianarmy.client.vsphere.terminationStrategy.property.name", "Force Boot"); - this.propertyValue - = config.getStrOrElse("simianarmy.client.vsphere.terminationStrategy.property.value", "server"); + public PropertyBasedTerminationStrategy(MonkeyConfiguration config) { + this.propertyName = config.getStrOrElse( + "simianarmy.client.vsphere.terminationStrategy.property.name", "Force Boot"); + this.propertyValue = config.getStrOrElse( + "simianarmy.client.vsphere.terminationStrategy.property.value", "server"); } @Override diff --git a/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereContext.java b/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereContext.java index 534bfb35..985f6e63 100644 --- a/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereContext.java +++ b/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereContext.java @@ -15,21 +15,21 @@ */ package com.netflix.simianarmy.client.vsphere; -import com.netflix.simianarmy.basic.BasicConfiguration; -import com.netflix.simianarmy.basic.BasicContext; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.basic.BasicChaosMonkeyContext; /** * This Context extends the BasicContext in order to provide a different client: the VSphereClient. * * @author ingmar.krusch@immobilienscout24.de */ -public class VSphereContext extends BasicContext { +public class VSphereContext extends BasicChaosMonkeyContext { @Override - protected void createClient(BasicConfiguration config) { + protected void createClient() { + MonkeyConfiguration config = configuration(); final PropertyBasedTerminationStrategy terminationStrategy = new PropertyBasedTerminationStrategy(config); final VSphereServiceConnection connection = new VSphereServiceConnection(config); final VSphereClient client = new VSphereClient(terminationStrategy, connection); - setAwsClient(client); setCloudClient(client); } } diff --git a/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereServiceConnection.java b/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereServiceConnection.java index 7310cc19..34e0cda7 100644 --- a/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereServiceConnection.java +++ b/src/main/java/com/netflix/simianarmy/client/vsphere/VSphereServiceConnection.java @@ -21,7 +21,7 @@ import java.util.Arrays; import com.amazonaws.AmazonServiceException; -import com.netflix.simianarmy.basic.BasicConfiguration; +import com.netflix.simianarmy.MonkeyConfiguration; import com.vmware.vim25.InvalidProperty; import com.vmware.vim25.RuntimeFault; import com.vmware.vim25.mo.InventoryNavigator; @@ -40,7 +40,7 @@ * @author ingmar.krusch@immobilienscout24.de */ public class VSphereServiceConnection { - /** The type of managedEntity we operate on are virtual machines. */ + /** The type of managedEntity we operate on are virtual machines. */ public static final String VIRTUAL_MACHINE_TYPE_NAME = "VirtualMachine"; /** The username that is used to connect to VSpehere Center. */ @@ -58,7 +58,7 @@ public class VSphereServiceConnection { /** * Constructor. */ - public VSphereServiceConnection(BasicConfiguration config) { + public VSphereServiceConnection(MonkeyConfiguration config) { this.url = config.getStr("simianarmy.client.vsphere.url"); this.username = config.getStr("simianarmy.client.vsphere.username"); this.password = config.getStr("simianarmy.client.vsphere.password"); @@ -116,9 +116,9 @@ public VirtualMachine[] describeVirtualMachines() throws AmazonServiceException if (mes == null || mes.length == 0) { throw new AmazonServiceException( - "vsphere returned zero entities of type \"" - + VIRTUAL_MACHINE_TYPE_NAME + "\"" - ); + "vsphere returned zero entities of type \"" + + VIRTUAL_MACHINE_TYPE_NAME + "\"" + ); } else { return Arrays.copyOf(mes, mes.length, VirtualMachine[].class); } diff --git a/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java new file mode 100644 index 00000000..c4ef1ed6 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java @@ -0,0 +1,384 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.janitor.JanitorMonkey.EventTypes; +import com.netflix.simianarmy.janitor.JanitorMonkey.Type; + +/** + * An abstract implementation of Janitor. It marks resources that the rule engine considers + * invalid as cleanup candidate and sets the expected termination date. It also removes the + * cleanup candidate flag from resources that no longer exist or the rule engine no longer + * considers invalid due to change of conditions. For resources marked as cleanup candidates + * and the expected termination date is passed, the janitor removes the resources from the + * cloud. + */ +public abstract class AbstractJanitor implements Janitor { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJanitor.class); + + /** The region the janitor is running in. */ + private final String region; + + /** + * The rule engine used to decide if a resource should be a cleanup + * candidate. + */ + private final JanitorRuleEngine ruleEngine; + + /** The janitor crawler to get resources from the cloud. */ + private final JanitorCrawler crawler; + + /** The resource type that the janitor is responsible for to clean up. **/ + private final Enum resourceType; + + /** The janitor resource tracker that is responsible for keeping track of + * resource status. + */ + private final JanitorResourceTracker resourceTracker; + + private final Collection markedResources = new ArrayList(); + + private final Collection cleanedResources = new ArrayList(); + + private final Collection unmarkedResources = new ArrayList(); + + private final Collection failedToCleanResources = new ArrayList(); + + private final MonkeyCalendar calendar; + + private final MonkeyConfiguration config; + + /** Flag to indicate whether the Janitor is leashed. */ + private boolean leashed; + + private final MonkeyRecorder recorder; + + /** + * Sets the flag to indicate if the janitor is leashed. + * + * @param isLeashed true if the the janitor is leased, false otherwise. + */ + protected void setLeashed(boolean isLeashed) { + this.leashed = isLeashed; + } + + /** + * The Interface Context. + */ + public interface Context { + + /** Region. + * + * @return the region + */ + String region(); + + /** + * Configuration. + * + * @return the monkey configuration + */ + MonkeyConfiguration configuration(); + + /** + * Calendar. + * + * @return the monkey calendar + */ + MonkeyCalendar calendar(); + + /** + * Janitor rule engine. + * @return the janitor rule engine + */ + JanitorRuleEngine janitorRuleEngine(); + + /** + * Janitor crawler. + * + * @return the chaos crawler + */ + JanitorCrawler janitorCrawler(); + + /** + * Janitor resource tracker. + * + * @return the janitor resource tracker + */ + JanitorResourceTracker janitorResourceTracker(); + + /** + * Recorder. + * + * @return the recorder to record events + */ + MonkeyRecorder recorder(); + } + + /** + * Constructor. + * @param ctx the context + * @param resourceType the resource type the janitor is taking care + */ + public AbstractJanitor(Context ctx, Enum resourceType) { + Validate.notNull(ctx); + Validate.notNull(resourceType); + this.region = ctx.region(); + Validate.notNull(region); + this.ruleEngine = ctx.janitorRuleEngine(); + Validate.notNull(ruleEngine); + this.crawler = ctx.janitorCrawler(); + Validate.notNull(crawler); + this.resourceTracker = ctx.janitorResourceTracker(); + Validate.notNull(resourceTracker); + this.calendar = ctx.calendar(); + Validate.notNull(calendar); + this.config = ctx.configuration(); + Validate.notNull(config); + // By default the janitor is leashed. + this.leashed = config.getBoolOrElse("simianarmy.janitor.leashed", true); + this.resourceType = resourceType; + Validate.notNull(resourceType); + // recorder could be null and no events are recorded when it is. + this.recorder = ctx.recorder(); + } + + @Override + public Enum getResourceType() { + return resourceType; + } + + /** + * Marks all resources obtained from the crawler as cleanup candidate if + * the janitor rule engine thinks so. + */ + @Override + public void markResources() { + markedResources.clear(); + unmarkedResources.clear(); + Map trackedMarkedResources = new HashMap(); + for (Resource resource : resourceTracker.getResources(resourceType, Resource.CleanupState.MARKED, region)) { + trackedMarkedResources.put(resource.getId(), resource); + } + + List crawledResources = crawler.resources(resourceType); + LOGGER.info(String.format("Looking for cleanup candidate in %d crawled resources.", + crawledResources.size())); + Date now = calendar.now().getTime(); + for (Resource resource : crawledResources) { + Resource trackedResource = trackedMarkedResources.get(resource.getId()); + if (!ruleEngine.isValid(resource)) { + // If the resource is already marked, ignore it + if (trackedResource != null) { + LOGGER.debug(String.format("Resource %s is already marked.", resource.getId())); + continue; + } + LOGGER.info(String.format("Marking resource %s of type %s with expected termination time as %s", + resource.getId(), resource.getResourceType(), resource.getExpectedTerminationTime())); + resource.setState(CleanupState.MARKED); + resource.setMarkTime(now); + if (!leashed) { + if (recorder != null) { + Event evt = recorder.newEvent(Type.JANITOR, EventTypes.MARK_RESOURCE, region, resource.getId()); + recorder.recordEvent(evt); + } + resourceTracker.addOrUpdate(resource); + postMark(resource); + } else { + LOGGER.info(String.format( + "The janitor is leashed, no data change is made for marking the resource %s.", + resource.getId())); + } + markedResources.add(resource); + } else if (trackedResource != null) { + // The resource was marked and now the rule engine does not consider it as a cleanup candidate. + // So the janitor needs to unmark the resource. + LOGGER.info(String.format("Unmarking resource %s", resource.getId())); + resource.setState(CleanupState.UNMARKED); + if (!leashed) { + if (recorder != null) { + Event evt = recorder.newEvent( + Type.JANITOR, EventTypes.UNMARK_RESOURCE, region, resource.getId()); + recorder.recordEvent(evt); + } + resourceTracker.addOrUpdate(resource); + } else { + LOGGER.info(String.format( + "The janitor is leashed, no data change is made for unmarking the resource %s.", + resource.getId())); + } + unmarkedResources.add(resource); + } + } + + // Unmark the resources that are terminated by user so not returned by the crawler. + unmarkUserTerminatedResources(crawledResources, trackedMarkedResources); + } + + /** + * Cleans up all cleanup candidates that are OK to remove. + */ + @Override + public void cleanupResources() { + cleanedResources.clear(); + failedToCleanResources.clear(); + List trackedMarkedResources = resourceTracker.getResources( + resourceType, Resource.CleanupState.MARKED, region); + LOGGER.info(String.format("Checking %d marked resources for cleanup.", trackedMarkedResources.size())); + + Date now = calendar.now().getTime(); + for (Resource markedResource : trackedMarkedResources) { + if (canClean(markedResource, now)) { + LOGGER.info(String.format("Cleaning up resource %s of type %s", + markedResource.getId(), markedResource.getResourceType().name())); + if (!leashed) { + try { + if (recorder != null) { + Event evt = recorder.newEvent(Type.JANITOR, EventTypes.CLEANUP_RESOURCE, region, + markedResource.getId()); + recorder.recordEvent(evt); + } + cleanup(markedResource); + markedResource.setActualTerminationTime(now); + markedResource.setState(Resource.CleanupState.JANITOR_TERMINATED); + resourceTracker.addOrUpdate(markedResource); + } catch (Exception e) { + LOGGER.error(String.format("Failed to clean up the resource %s.", + markedResource.getId()), e); + failedToCleanResources.add(markedResource); + continue; + } + postCleanup(markedResource); + } else { + LOGGER.info(String.format( + "The janitor is leashed, no data change is made for cleaning up the resource %s.", + markedResource.getId())); + } + cleanedResources.add(markedResource); + } + } + } + + /** Determines if the input resource can be cleaned. The Janitor calls this method + * before cleaning up a resource and only cleans the resource when the method returns + * true. A resource is considered to be OK to clean if + * 1) it is marked as cleanup candidates + * 2) the expected termination time is already passed + * 3) the owner has already been notified about the cleanup + * 4) the resource is not opted out of Janitor monkey + * The method can be overriden in subclasses. + * @param resource the resource the Janitor considers to clean + * @param now the time that represents the current time + * @return true if the resource is OK to clean, false otherwise + */ + protected boolean canClean(Resource resource, Date now) { + return resource.getState() == Resource.CleanupState.MARKED + && !resource.isOptOutOfJanitor() + && resource.getExpectedTerminationTime() != null + && resource.getExpectedTerminationTime().before(now) + && resource.getNotificationTime() != null + && resource.getNotificationTime().before(now); + } + + /** + * Implements required operations after a resource is marked. + * @param resource The resource that is marked + */ + protected abstract void postMark(Resource resource); + + /** + * Cleans a resource up, e.g. deleting the resource from the cloud. + * @param resource The resource that is cleaned up. + */ + protected abstract void cleanup(Resource resource); + + /** + * Implements required operations after a resource is cleaned. + * @param resource The resource that is cleaned up. + */ + protected abstract void postCleanup(Resource resource); + + /** gets the resources marked in the last run of the Janitor. */ + public Collection getMarkedResources() { + return Collections.unmodifiableCollection(markedResources); + } + + /** gets the resources unmarked in the last run of the Janitor. */ + public Collection getUnmarkedResources() { + return Collections.unmodifiableCollection(unmarkedResources); + } + + /** gets the resources cleaned in the last run of the Janitor. */ + public Collection getCleanedResources() { + return Collections.unmodifiableCollection(cleanedResources); + } + + /** gets the resources that failed to be cleaned in the last run of the Janitor. */ + public Collection getFailedToCleanResources() { + return Collections.unmodifiableCollection(failedToCleanResources); + } + + private void unmarkUserTerminatedResources( + List crawledResources, Map trackedMarkedResources) { + Set crawledResourceIds = new HashSet(); + for (Resource crawledResource : crawledResources) { + crawledResourceIds.add(crawledResource.getId()); + } + for (Resource markedResource : trackedMarkedResources.values()) { + if (!crawledResourceIds.contains(markedResource.getId())) { + // The resource does not exist anymore. + LOGGER.info(String.format( + "Resource %s is not returned by the crawler. It should already be terminated.", + markedResource.getId())); + if (!leashed) { + markedResource.setState(Resource.CleanupState.USER_TERMINATED); + resourceTracker.addOrUpdate(markedResource); + } else { + LOGGER.info(String.format( + "The janitor is leashed, no data change is made for unmarking " + + "the user terminated resource %s.", + markedResource.getId())); + } + unmarkedResources.add(markedResource); + } + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/Janitor.java b/src/main/java/com/netflix/simianarmy/janitor/Janitor.java new file mode 100644 index 00000000..157b3363 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/Janitor.java @@ -0,0 +1,43 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +/** + * The interface for a janitor that performs the mark and cleanup operations for + * cloud resources of a resource type. + */ +public interface Janitor { + + /** + * Gets the resource type the janitor is cleaning up. + * @return the resource type the janitor is cleaning up. + */ + Enum getResourceType(); + + /** + * Mark cloud resources as cleanup candidates and remove the marks for resources + * that no longer exist or should not be cleanup candidates anymore. + */ + void markResources(); + + /** + * Clean the resources up that are marked as cleanup candidates when appropriate. + */ + void cleanupResources(); +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorCrawler.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorCrawler.java new file mode 100644 index 00000000..9b33eedf --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorCrawler.java @@ -0,0 +1,62 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +import java.util.EnumSet; +import java.util.List; + +import com.netflix.simianarmy.Resource; + +/** + * The crawler for janitor monkey. + */ +public interface JanitorCrawler { + + /** + * Resource types. + * + * @return the type of resources this crawler crawls + */ + EnumSet resourceTypes(); + + /** + * Resources crawled by this crawler for a specific resource type. + * + * @param resourceType the resource type + * @return the list + */ + List resources(Enum resourceType); + + /** + * Gets the up to date information for a collection of resource ids. When the input argument is null + * or empty, the method returns all resources. + * + * @param resourceIds + * the resource ids + * @return the list of resources + */ + List resources(String... resourceIds); + + /** + * Gets the owner email for a resource to set the ownerEmail field when crawl. + * @param resource the resource + * @return the owner email of the resource + */ + String getOwnerEmailForResource(Resource resource); +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailBuilder.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailBuilder.java new file mode 100644 index 00000000..c3433712 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailBuilder.java @@ -0,0 +1,35 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.janitor; + +import java.util.Collection; +import java.util.Map; + +import com.netflix.simianarmy.AbstractEmailBuilder; +import com.netflix.simianarmy.Resource; + +/** The abstract class for building Janitor monkey email notifications. */ +public abstract class JanitorEmailBuilder extends AbstractEmailBuilder { + + /** + * Sets the map from an owner email to the resources that belong to the owner + * and need to send notifications for. + * @param emailToResources the map from owner email to the owned resource + */ + public abstract void setEmailToResources(Map> emailToResources); +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java new file mode 100644 index 00000000..edaa28b9 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java @@ -0,0 +1,290 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.janitor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.aws.AWSEmailNotifier; + +/** The email notifier implemented for Janitor Monkey. */ +public class JanitorEmailNotifier extends AWSEmailNotifier { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(JanitorEmailNotifier.class); + private static final String EMAIL_PATTERN = + "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@" + + "[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$"; + private static final String UNKNOWN_EMAIL = "UNKNOWN"; + + private final String region; + private final String defaultEmail; + private final List ccEmails; + private final Pattern emailPattern; + private final JanitorResourceTracker resourceTracker; + private final JanitorEmailBuilder emailBuilder; + private final MonkeyCalendar calendar; + private final int daysBeforeTermination; + private final String sourceEmail; + private final Map> invalidEmailToResources = + new HashMap>(); + + /** + * The Interface Context. + */ + public interface Context { + /** + * Gets the Amazon Simple Email Service client. + * @return the Amazon Simple Email Service client + */ + AmazonSimpleEmailServiceClient sesClient(); + + /** + * Gets the source email the notifier uses to send email. + * @return the source email + */ + String sourceEmail(); + + /** + * Gets the default email the notifier sends to when there is no owner specified for a resource. + * @return the default email + */ + String defaultEmail(); + + /** + * Gets the number of days a notification is sent before the expected termination date.. + * @return the number of days a notification is sent before the expected termination date. + */ + int daysBeforeTermination(); + + /** + * Gets the region the notifier is running in. + * @return the region the notifier is running in. + */ + String region(); + + /** Gets the janitor resource tracker. + * @return the janitor resource tracker + */ + JanitorResourceTracker resourceTracker(); + + /** Gets the janitor email builder. + * @return the janitor email builder + */ + JanitorEmailBuilder emailBuilder(); + + /** Gets the calendar. + * @return the calendar + */ + MonkeyCalendar calendar(); + + /** Gets the cc email addresses. + * @return the cc email addresses + */ + String[] ccEmails(); + } + + /** + * Constructor. + * @param ctx the context. + */ + public JanitorEmailNotifier(Context ctx) { + super(ctx.sesClient()); + this.region = ctx.region(); + this.emailPattern = Pattern.compile(EMAIL_PATTERN); + this.defaultEmail = ctx.defaultEmail(); + this.daysBeforeTermination = ctx.daysBeforeTermination(); + this.resourceTracker = ctx.resourceTracker(); + this.emailBuilder = ctx.emailBuilder(); + this.calendar = ctx.calendar(); + this.ccEmails = new ArrayList(); + String[] ctxCCs = ctx.ccEmails(); + if (ctxCCs != null) { + for (String ccEmail : ctxCCs) { + this.ccEmails.add(ccEmail); + } + } + this.sourceEmail = ctx.sourceEmail(); + } + + /** + * Gets all the resources that are marked and no notifications have been sent. Send email notifications + * for these resources. If there is a valid email address in the ownerEmail field of the resource, send + * to that address. Otherwise send to the default email address. + */ + public void sendNotifications() { + validateEmails(); + List markedResources = resourceTracker.getResources(null, CleanupState.MARKED, region); + Map> emailToResources = new HashMap>(); + invalidEmailToResources.clear(); + for (Resource r : markedResources) { + if (r.isOptOutOfJanitor()) { + LOGGER.info(String.format("Resource %s is opted out of Janitor Monkey so no notification is sent.", + r.getId())); + continue; + } + if (canNotify(r)) { + String email = r.getOwnerEmail(); + if (!isValidEmail(email)) { + if (defaultEmail != null) { + email = defaultEmail; + LOGGER.info(String.format("Email %s is not valid, send to the default email address %s", + email, defaultEmail)); + } else { + if (email == null) { + email = UNKNOWN_EMAIL; + } + LOGGER.info(String.format("Email %s is not valid and default email is not set for resource %s", + email, r.getId())); + putEmailAndResource(invalidEmailToResources, email, r); + } + } else { + putEmailAndResource(emailToResources, email, r); + } + } else { + LOGGER.debug(String.format("Not the time to send notification for resource %s", r.getId())); + } + } + emailBuilder.setEmailToResources(emailToResources); + Date now = calendar.now().getTime(); + for (Map.Entry> entry : emailToResources.entrySet()) { + String email = entry.getKey(); + String emailBody = emailBuilder.buildEmailBody(email); + String subject = buildEmailSubject(email); + sendEmail(email, subject, emailBody); + for (Resource r : entry.getValue()) { + LOGGER.debug(String.format("Notification is sent for resource %s", r.getId())); + r.setNotificationTime(now); + resourceTracker.addOrUpdate(r); + } + LOGGER.info(String.format("Email notification has been sent to %s for %d resources.", + email, entry.getValue().size())); + } + } + + private void validateEmails() { + if (defaultEmail != null) { + Validate.isTrue(isValidEmail(defaultEmail), String.format("Default email %s is invalid", defaultEmail)); + } + if (ccEmails != null) { + for (String ccEmail : ccEmails) { + Validate.isTrue(isValidEmail(ccEmail), String.format("CC email %s is invalid", ccEmail)); + } + } + } + + @Override + public boolean isValidEmail(String email) { + if (email == null) { + return false; + } + if (emailPattern.matcher(email).matches()) { + return true; + } else { + LOGGER.error(String.format("Invalid email address: %s", email)); + return false; + } + } + + @Override + public String buildEmailSubject(String email) { + return String.format("Janitor Monkey Notification for %s", email); + } + + /** + * Decides if it is time for sending notification for the resource. This method can be + * overriden in subclasses so notifications can be send earlier or later. + * @param resource the resource + * @return true if it is OK to send notification now, otherwise false. + */ + protected boolean canNotify(Resource resource) { + Validate.notNull(resource); + if (resource.getState() != CleanupState.MARKED || resource.isOptOutOfJanitor()) { + return false; + } + + Date notificationTime = resource.getNotificationTime(); + // We don't want to send notification too early (since things may change) or too late (we need + // to give owners enough time to take actions. + Date windowStart = calendar.getBusinessDay(calendar.now().getTime(), daysBeforeTermination); + Date windowEnd = calendar.getBusinessDay(calendar.now().getTime(), daysBeforeTermination + 1); + Date terminationDate = resource.getExpectedTerminationTime(); + if (notificationTime == null + || resource.getMarkTime().after(notificationTime)) { // remarked after a notification + if (!terminationDate.before(windowStart) && !terminationDate.after(windowEnd)) { + // The expected termination time is close enough for sending notification + return true; + } else if (terminationDate.before(windowStart)) { + // The expected termination date is too close. To give the owner time to take possible actions, + // we extend the expected termination time here. + LOGGER.info(String.format("It is less than %d days before the expected termination date," + + " of resource %s, extending the termination time to %s.", + daysBeforeTermination, resource.getId(), windowStart)); + resource.setExpectedTerminationTime(windowStart); + resourceTracker.addOrUpdate(resource); + return true; + } else { + return false; + } + } + return false; + } + + /** + * Gets the map from invalid email address to the resources that were supposed to be sent to the address. + * + * @return the map from invalid address to resources that failed to be delivered + */ + public Map> getInvalidEmailToResources() { + return Collections.unmodifiableMap(invalidEmailToResources); + } + + @Override + public String[] getCcAddresses(String to) { + return ccEmails.toArray(new String[ccEmails.size()]); + } + + @Override + public String getSourceAddress(String to) { + return sourceEmail; + } + + private void putEmailAndResource( + Map> map, String email, Resource resource) { + Collection resources = map.get(email); + if (resources == null) { + resources = new ArrayList(); + map.put(email, resources); + } + resources.add(resource); + } +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java new file mode 100644 index 00000000..118f79fd --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java @@ -0,0 +1,147 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.janitor; + +import java.util.List; + +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder.Event; + +/** + * The abstract class for a Janitor Monkey. + */ +public abstract class JanitorMonkey extends Monkey { + + /** The key name of the Janitor tag used to tag resources. */ + public static final String JANITOR_TAG = "janitor"; + /** The key name of the Janitor meta tag used to tag resources. */ + public static final String JANITOR_META_TAG = "JANITOR_META"; + /** The key name of the tag owner used to tag resources. */ + public static final String OWNER_TAG_KEY = "owner"; + /** The key name of the tag instance used to tag resources. */ + public static final String INSTANCE_TAG_KEY = "instance"; + /** The key name of the tag detach time used to tag resources. */ + public static final String DETACH_TIME_TAG_KEY = "detachTime"; + + /** + * The Interface Context. + */ + public interface Context extends Monkey.Context { + + /** + * Configuration. + * + * @return the monkey configuration + */ + MonkeyConfiguration configuration(); + + /** + * Janitors run by this monkey. + * @return the janitors + */ + List janitors(); + + /** + * Email notifier used to send notifications by the janitor monkey. + * @return the email notifier + */ + JanitorEmailNotifier emailNotifier(); + + /** + * The region the monkey is running in. + * @return the region the monkey is running in. + */ + String region(); + + /** + * The Janitor resource tracker. + * @return the Janitor resource tracker. + */ + JanitorResourceTracker resourceTracker(); + } + + /** The context. */ + private final Context ctx; + + /** + * Instantiates a new janitor monkey. + * + * @param ctx + * the context. + */ + public JanitorMonkey(Context ctx) { + super(ctx); + this.ctx = ctx; + } + + /** + * The monkey Type. + */ + public enum Type { + /** janitor monkey. */ + JANITOR + } + + /** + * The event types that this monkey causes. + */ + public enum EventTypes { + /** Marking a resource as a cleanup candidate. */ + MARK_RESOURCE, + /** Un-Marking a resource. */ + UNMARK_RESOURCE, + /** Clean up a resource. */ + CLEANUP_RESOURCE, + /** Opt in a resource. */ + OPT_IN_RESOURCE, + /** Opt out a resource. */ + OPT_OUT_RESOURCE + } + + /** {@inheritDoc} */ + @Override + public final Enum type() { + return Type.JANITOR; + } + + /** {@inheritDoc} */ + @Override + public Context context() { + return ctx; + } + + /** {@inheritDoc} */ + @Override + public abstract void doMonkeyBusiness(); + + /** + * Opt in a resource for Janitor Monkey. + * @param resourceId the resource id + * @return the opt-in event + */ + public abstract Event optInResource(String resourceId); + + /** + * Opt out a resource for Janitor Monkey. + * @param resourceId the resource id + * @return the opt-out event + */ + public abstract Event optOutResource(String resourceId); + +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java new file mode 100644 index 00000000..1a3a526b --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java @@ -0,0 +1,54 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.janitor; + +import java.util.List; + +import com.netflix.simianarmy.Resource; + +/** + * The interface to track the resources marked/cleaned by the Janitor Monkey. + * + */ +public interface JanitorResourceTracker { + + /** + * Adds a resource to the tracker. If the resource with the same id already exists + * in the tracker, the method updates the record with the resource parameter. + * @param resource the resource to add or update + */ + void addOrUpdate(Resource resource); + + /** Gets the list of resources of a specific resource type and cleanup state in a region. + * + * @param resourceType the resource type + * @param state the cleanup state of the resources + * @param region the region of the resources, when the parameter is null, the method returns + * resources from all regions + * @return list of resources that match the resource type, state and region + */ + List getResources(Enum resourceType, Resource.CleanupState state, String region); + + /** Gets the resource of a specific id. + * + * @param resourceId the resource id + * @return list of resources that match the resource id + */ + Resource getResource(String resourceId); + +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorRuleEngine.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorRuleEngine.java new file mode 100644 index 00000000..0d15bf9a --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorRuleEngine.java @@ -0,0 +1,47 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +import com.netflix.simianarmy.Resource; + +/** + * The interface for janitor rule engine that can decide if a resource should be a candidate of cleanup + * based on a collection of rules. + */ +public interface JanitorRuleEngine { + + /** + * Decides whether the resource should be a candidate of cleanup based on the underlying rules. + * + * @param resource + * The resource + * @return true if the resource is valid and should not be a candidate of cleanup based on the underlying rules, + * false otherwise. + */ + boolean isValid(Resource resource); + + /** + * Add a rule to decide if a resource should be a candidate for cleanup. + * + * @param rule + * The rule to decide if a resource should be a candidate for cleanup. + * @return The JanitorRuleEngine object. + */ + JanitorRuleEngine addRule(Rule rule); +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/Rule.java b/src/main/java/com/netflix/simianarmy/janitor/Rule.java new file mode 100644 index 00000000..8275e598 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/janitor/Rule.java @@ -0,0 +1,37 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +import com.netflix.simianarmy.Resource; + +/** + * The rule implementing a logic to decide if a resource should be considered as a candidate of cleanup. + */ +public interface Rule { + /** + * Decides whether the resource should be a candidate of cleanup based on the underlying rule. When + * the rule considers the resource as a candidate of cleanup, it sets the expected termination time + * and termination reason of the resource. + * + * @param resource + * The resource + * @return true if the resource is valid and is not for cleanup, false otherwise + */ + boolean isValid(Resource resource); +} diff --git a/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java b/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java index 035c4563..fe134d2c 100644 --- a/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java +++ b/src/main/java/com/netflix/simianarmy/resources/chaos/ChaosMonkeyResource.java @@ -146,6 +146,7 @@ public Response getChaosEvents(@Context UriInfo uriInfo) throws IOException { @POST public Response addEvent(String content) throws IOException { ObjectMapper mapper = new ObjectMapper(); + LOGGER.info(String.format("JSON content: '%s'", content)); JsonNode input = mapper.readTree(content); String eventType = getStringField(input, "eventType"); @@ -173,6 +174,7 @@ public Response addEvent(String content) throws IOException { } gen.writeEndObject(); gen.close(); + LOGGER.info("entity content is '{}'", baos.toString("UTF-8")); return Response.status(responseStatus).entity(baos.toString("UTF-8")).build(); } diff --git a/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java b/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java new file mode 100644 index 00000000..c9cf05d8 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java @@ -0,0 +1,153 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.resources.janitor; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; + +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Response; + +import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.JsonEncoding; +import org.codehaus.jackson.JsonGenerator; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.MappingJsonFactory; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.MonkeyRunner; +import com.netflix.simianarmy.janitor.JanitorMonkey; + +/** + * The Class JanitorMonkeyResource for json REST apis. + */ +@Path("/v1/janitor") +public class JanitorMonkeyResource { + + /** The Constant JSON_FACTORY. */ + private static final MappingJsonFactory JSON_FACTORY = new MappingJsonFactory(); + + /** The monkey. */ + private final JanitorMonkey monkey; + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(JanitorMonkeyResource.class); + + /** + * Instantiates a janitor monkey resource with a specific janitor monkey. + * + * @param monkey + * the janitor monkey + */ + public JanitorMonkeyResource(JanitorMonkey monkey) { + this.monkey = monkey; + } + + /** + * Instantiates a janitor monkey resource using a registered janitor monkey from factory. + */ + public JanitorMonkeyResource() { + this.monkey = MonkeyRunner.getInstance().factory(JanitorMonkey.class); + } + + /** + * POST /api/v1/janitor will try a add a new event with the information in the url context. + * + * @param content + * the Json content passed to the http POST request + * @return the response + * @throws IOException + */ + @POST + public Response addEvent(String content) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + LOGGER.info(String.format("JSON content: '%s'", content)); + JsonNode input = mapper.readTree(content); + + String eventType = getStringField(input, "eventType"); + String resourceId = getStringField(input, "resourceId"); + + Response.Status responseStatus; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator gen = JSON_FACTORY.createJsonGenerator(baos, JsonEncoding.UTF8); + gen.writeStartObject(); + gen.writeStringField("eventType", eventType); + gen.writeStringField("resourceId", resourceId); + + if (StringUtils.isEmpty(eventType) || StringUtils.isEmpty(resourceId)) { + responseStatus = Response.Status.BAD_REQUEST; + gen.writeStringField("message", "eventType and resourceId parameters are all required"); + } else { + if (eventType.equals("OPTIN")) { + responseStatus = optInResource(resourceId, true, gen); + } else if (eventType.equals("OPTOUT")) { + responseStatus = optInResource(resourceId, false, gen); + } else { + responseStatus = Response.Status.BAD_REQUEST; + gen.writeStringField("message", String.format("Unrecognized event type: %s", eventType)); + } + } + gen.writeEndObject(); + gen.close(); + LOGGER.info("entity content is '{}'", baos.toString("UTF-8")); + return Response.status(responseStatus).entity(baos.toString("UTF-8")).build(); + } + + private Response.Status optInResource(String resourceId, boolean optIn, JsonGenerator gen) + throws IOException { + String op = optIn ? "in" : "out"; + LOGGER.info(String.format("Opt %s resource %s for Janitor Monkey.", op, resourceId)); + Response.Status responseStatus; + Event evt; + if (optIn) { + evt = monkey.optInResource(resourceId); + } else { + evt = monkey.optOutResource(resourceId); + } + if (evt != null) { + responseStatus = Response.Status.OK; + gen.writeStringField("monkeyType", evt.monkeyType().name()); + gen.writeStringField("eventId", evt.id()); + gen.writeNumberField("eventTime", evt.eventTime().getTime()); + gen.writeStringField("region", evt.region()); + for (Map.Entry pair : evt.fields().entrySet()) { + gen.writeStringField(pair.getKey(), pair.getValue()); + } + } else { + responseStatus = Response.Status.INTERNAL_SERVER_ERROR; + gen.writeStringField("message", + String.format("Failed to opt %s resource %s", op, resourceId)); + } + LOGGER.info(String.format("Opt %s operaction completed.", op)); + return responseStatus; + } + + private String getStringField(JsonNode input, String field) { + JsonNode node = input.get(field); + if (node == null) { + return null; + } + return node.getTextValue(); + } + +} diff --git a/src/main/resources/chaos.properties b/src/main/resources/chaos.properties new file mode 100644 index 00000000..ae9bcb6a --- /dev/null +++ b/src/main/resources/chaos.properties @@ -0,0 +1,44 @@ +# The file contains the properties for Chaos Monkey. +# see documentation at: +# https://github.com/Netflix/SimianArmy/wiki/Configuration + +# let chaos run +simianarmy.chaos.enabled = true + +# don't allow chaos to kill (ie dryrun mode) +simianarmy.chaos.leashed = true + +# set to "false" for Opt-In behavior, "true" for Opt-Out behavior +simianarmy.chaos.ASG.enabled = false + +# default probability for all ASGs +simianarmy.chaos.ASG.probability = 1.0 + +# increase or decrease the termination limit +simianarmy.chaos.ASG.maxTerminationsPerDay = 1.0 + +# enable a specific ASG +# simianarmy.chaos.ASG..enabled = true +# simianarmy.chaos.ASG..probability = 1.0 + +# increase or decrease the termination limit for a specific ASG +# simianarmy.chaos.ASG..maxTerminationsPerDay = 1.0 + +# Enroll in mandatory terminations. If a group has not had a +# termination within the windowInDays range then it will terminate +# one instance in the group with a 0.5 probability (at some point in +# the next 2 days an instance should be terminated), then +# do nothing again for windowInDays. This forces "enabled" groups +# that have a probability of 0.0 to have terminations periodically. +simianarmy.chaos.mandatoryTermination.enabled = false +simianarmy.chaos.mandatoryTermination.windowInDays = 32 +simianarmy.chaos.mandatoryTermination.defaultProbability = 0.5 + +# Enable notification for Chaos termination for a specific instance group +# simianarmy.chaos...notification.enabled = true + +# Set the destination email the termination notification sent to for a specific instance group +# simianarmy.chaos...ownerEmail = foo@bar.com + +# Set the source email that sends the termination notification +# simianarmy.chaos.notification.sourceEmail = foo@bar.com diff --git a/src/main/resources/client.properties b/src/main/resources/client.properties index 98752d75..b1a27e3e 100644 --- a/src/main/resources/client.properties +++ b/src/main/resources/client.properties @@ -20,7 +20,7 @@ simianarmy.client.aws.accountKey = fakeAccount simianarmy.client.aws.secretKey = fakeSecret -simianarmy.client.aws.region = eu-west-1 +simianarmy.client.aws.region = us-east-1 ### The VSpehere client uses a TerminationStrategy for killing VirtualMachines ### You can configure which property and value for it to set prior to resetting the VirtualMachine diff --git a/src/main/resources/janitor.properties b/src/main/resources/janitor.properties new file mode 100644 index 00000000..2cabbac5 --- /dev/null +++ b/src/main/resources/janitor.properties @@ -0,0 +1,97 @@ +# see documentation at: +# https://github.com/Netflix/SimianArmy/wiki/Configuration + +# By default Janitor Monkey wakes up every hour +simianarmy.scheduler.frequency = 1 +simianarmy.scheduler.frequencyUnit = HOURS +simianarmy.scheduler.threads = 1 +# Janitor Monkey runs every day at 11am. +simianarmy.calendar.openHour = 11 +simianarmy.calendar.closeHour = 11 +simianarmy.calendar.timezone = America/Los_Angeles + +# Let Janitor Monkey run +simianarmy.janitor.enabled = true + +# Don't allow Janitor Monkey to change resources (dryrun mode) +simianarmy.janitor.leashed = true + +# The SDB domain for storing the resources managed by the Janitor Monkey. +simianarmy.janitor.resources.sdb.domain = SIMIAN_ARMY + +# override to force monkey time, useful for debugging off hours +#simianarmy.calendar.isMonkeyTime = true + +# Currently Janitor Monkey can clean up the following resources +simianarmy.janitor.enabledResources = Instance, ASG, EBS_Volume, EBS_Snapshot + +# The property below needs to be a valid email address to send notifications for Janitor Monkey +simianarmy.janitor.notification.sourceEmail = foo@bar.com + +# The property below needs to be a valid email address to receive the summary email of Janitor Monkey +# after each run +simianarmy.janitor.summaryEmail.to = foo@bar.com + +# The property below needs to be a valid email address to receive the notifications of Janitor Monkey +# for resouces that do not have a valid owner email specified +simianarmy.janitor.notification.defaultEmail = foo@bar.com + +# The property below specifies the number of business days that a notification is sent before the +# expected termination time. For example, if a resource is scheduled to be cleaned up by Janitor +# Monkey on 12/13/2012, Thursday and the property is set to 2, the owner will receive notification +# about the cleanup on 12/11/2012, Tuesday, which is 2 business days before the termination date. +simianarmy.janitor.notification.daysBeforeTermination = 2 + +# The following properties are used by the Janitor rule for cleaning up orphaned instances, +# i.e. instances that are not in an auto-scaling group. +simianarmy.janitor.rule.orphanedInstanceRule.enabled = true +# An orphaned instance is marked as cleanup candidate if it has launched for more than the number +# of days specified in the property below. +simianarmy.janitor.rule.orphanedInstanceRule.instanceAgeThreshold = 2 +# The number of business days the instance is kept after a notification is sent for the termination +# when the instance has an owner. +simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithOwner = 3 +# The number of business days the instance is kept after a notification is sent for the termination +# when the instance has no owner. +simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithoutOwner = 8 + +# The following properties are used by the Janitor rule for cleaning up volumes that have been +# detached from instances for certain days. +simianarmy.janitor.rule.oldDetachedVolumeRule.enabled = true +# A volume is considered a cleanup candidate after being detached for the number of days specified +# in the property below. +simianarmy.janitor.rule.oldDetachedVolumeRule.detachDaysThreshold = 30 +# The number of business days the volume is kept after a notification is sent for the termination. +simianarmy.janitor.rule.oldDetachedVolumeRule.retentionDays = 7 + +# The following properties are used by the Janitor rule for cleaning up snapshots that have no existing +# images generated from them and launched for certain days. +simianarmy.janitor.rule.noGeneratedAMIRule.enabled = true +# A snapshot without an image is considered a cleanup candidate after launching for the number of +# days specified in the property below. +simianarmy.janitor.rule.noGeneratedAMIRule.ageThreshold = 30 +# The number of business days the snapshot is kept after a notification is sent for the termination. +simianarmy.janitor.rule.noGeneratedAMIRule.retentionDays = 7 + +# The following properties are used by the Janitor rule for cleaning up auto-scaling groups that have +# no active instances and the launch configuration is older than certain days. +simianarmy.janitor.rule.oldEmptyASGRule.enabled = true +# An an auto-scaling group without active instances is considered a cleanup candidate when its launch +# configuration is older than the number of days specified in the property below. +simianarmy.janitor.rule.oldEmptyASGRule.launchConfigAgeThreshold = 50 +# The number of business days the auto-scaling group is kept after a notification is sent for the termination. +simianarmy.janitor.rule.oldEmptyASGRule.retentionDays = 10 + +# The following properties are used by the Janitor rule for cleaning up auto-scaling groups that have +# no active instances and have been suspended from the associated ELB traffic for certain days. +simianarmy.janitor.rule.suspendedASGRule.enabled = true +# An auto-scaling group without active instances is considered a cleanup candidate when it has been +# suspended from the associated ELB traffic for the number of days specified in the property below. +simianarmy.janitor.rule.suspendedASGRule.suspensionAgeThreshold = 2 +# The number of business days the auto-scaling group is kept after a notification is sent for the termination. +simianarmy.janitor.rule.suspendedASGRule.retentionDays = 5 + +# The property below specifies whether or not Eureka/Discovery is available for Janitor monkey to use. +# Discovery/Eureka is used in the rules for cleaning up auto-scaling groups to decide if an auto-scaling group +# has an 'active' instance, i.e. an instance that is registered and up in Discovery/Eureka. +simianarmy.janitor.Eureka.enabled = false diff --git a/src/main/resources/simianarmy.properties b/src/main/resources/simianarmy.properties index 9ff56495..0e6a14c6 100644 --- a/src/main/resources/simianarmy.properties +++ b/src/main/resources/simianarmy.properties @@ -1,7 +1,8 @@ # see documentation at: # https://github.com/Netflix/SimianArmy/wiki/Configuration -simianarmy.sdb.domain = SIMIAN_ARMY +simianarmy.recorder.sdb.domain = SIMIAN_ARMY + simianarmy.scheduler.frequency = 1 simianarmy.scheduler.frequencyUnit = HOURS simianarmy.scheduler.threads = 1 @@ -9,45 +10,4 @@ simianarmy.calendar.openHour = 9 simianarmy.calendar.closeHour = 15 simianarmy.calendar.timezone = America/Los_Angeles # override to force monkey time, useful for debugging off hours -# simianarmy.calendar.isMonkeyTime = true - -# let chaos run -simianarmy.chaos.enabled = true - -# don't allow chaos to kill (ie dryrun mode) -simianarmy.chaos.leashed = true - -# set to "false" for Opt-In behavior, "true" for Opt-Out behavior -simianarmy.chaos.ASG.enabled = false - -# default probability for all ASGs -simianarmy.chaos.ASG.probability = 1.0 - -# increase or decrease the termination limit -simianarmy.chaos.ASG.maxTerminationsPerDay = 1.0 - -# enable a specific ASG -# simianarmy.chaos.ASG..enabled = true -# simianarmy.chaos.ASG..probability = 1.0 - -# increase or decrease the termination limit for a specific ASG -# simianarmy.chaos.ASG..maxTerminationsPerDay = 1.0 - -# Enroll in mandatory terminations. If a group has not had a -# termination within the windowInDays range then it will terminate -# one instance in the group with a 0.5 probability (at some point in -# the next 2 days an instance should be terminated), then -# do nothing again for windowInDays. This forces "enabled" groups -# that have a probability of 0.0 to have terminations periodically. -simianarmy.chaos.mandatoryTermination.enabled = false -simianarmy.chaos.mandatoryTermination.windowInDays = 32 -simianarmy.chaos.mandatoryTermination.defaultProbability = 0.5 - -# Enable notification for Chaos termination for a specific instance group -# simianarmy.chaos...notification.enabled = true - -# Set the destination email the termination notification sent to for a specific instance group -# simianarmy.chaos...ownerEmail = foo@bar.com - -# Set the source email that sends the termination notification -# simianarmy.chaos.notification.sourceEmail = foo@bar.com +#simianarmy.calendar.isMonkeyTime = true diff --git a/src/main/resources/volumeTagging.properties b/src/main/resources/volumeTagging.properties new file mode 100644 index 00000000..01ada879 --- /dev/null +++ b/src/main/resources/volumeTagging.properties @@ -0,0 +1,18 @@ +# see documentation at: +# https://github.com/Netflix/SimianArmy/wiki/Configuration + +# The properties in this file are used by the monkey that tags volumes with information that +# Janitor Monkey will need for cleaning up volumes. + +# Let the monkey run. +simianarmy.volumeTagging.enabled = true +# Running in the dryrun mode, no tagging is really done. +simianarmy.volumeTagging.leashed = true + +# Set the property below if you need the owner alias to be converted to a valid email address +#simianarmy.volumeTagging.ownerEmailDomain = foo.com + +# The volume tagging monkey always runs. The tagging process is needed by the Janitor Monkey to +# clean up volumes. We can keep the volume tagging monkey running so we don't miss any change +# of volumes. +simianarmy.calendar.isMonkeyTime = true diff --git a/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java b/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java index 5ffc413a..5acd832f 100644 --- a/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/TestMonkeyContext.java @@ -23,90 +23,135 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.Assert; import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.basic.BasicConfiguration; import com.netflix.simianarmy.basic.BasicRecorderEvent; public class TestMonkeyContext implements Monkey.Context { - private Enum monkeyType; - private LinkedList eventReport = new LinkedList(); + private final Enum monkeyType; + private final LinkedList eventReport = new LinkedList(); public TestMonkeyContext(Enum monkeyType) { this.monkeyType = monkeyType; } + @Override + public MonkeyConfiguration configuration() { + return new BasicConfiguration(new Properties()); + } + + @Override public MonkeyScheduler scheduler() { return new MonkeyScheduler() { + @Override public int frequency() { return 1; } + @Override public TimeUnit frequencyUnit() { return TimeUnit.HOURS; } + @Override public void start(Monkey monkey, Runnable run) { Assert.assertEquals(monkey.type().name(), monkeyType.name(), "starting monkey"); run.run(); } + @Override public void stop(Monkey monkey) { Assert.assertEquals(monkey.type().name(), monkeyType.name(), "stopping monkey"); } }; } + @Override public MonkeyCalendar calendar() { // CHECKSTYLE IGNORE MagicNumberCheck return new MonkeyCalendar() { + @Override public boolean isMonkeyTime(Monkey monkey) { return true; } + @Override public int openHour() { return 10; } + @Override public int closeHour() { return 11; } + @Override public Calendar now() { return Calendar.getInstance(); } + + @Override + public Date getBusinessDay(Date date, int n) { + throw new RuntimeException("Not implemented."); + } }; } + @Override public CloudClient cloudClient() { return new CloudClient() { + @Override public void terminateInstance(String instanceId) { } + + @Override + public void createTagsForResources(Map keyValueMap, String... resourceIds) { + } + + @Override + public void deleteAutoScalingGroup(String asgName) { + } + + @Override + public void deleteVolume(String volumeId) { + } + + @Override + public void deleteSnapshot(String snapshotId) { + } }; } - private MonkeyRecorder recorder = new MonkeyRecorder() { - private List events = new LinkedList(); + private final MonkeyRecorder recorder = new MonkeyRecorder() { + private final List events = new LinkedList(); + @Override public Event newEvent(Enum mkType, Enum eventType, String region, String id) { return new BasicRecorderEvent(mkType, eventType, region, id); } + @Override public void recordEvent(Event evt) { events.add(evt); } + @Override public List findEvents(Map query, Date after) { return events; } + @Override public List findEvents(Enum mkeyType, Map query, Date after) { // used from BasicScheduler return events; } + @Override public List findEvents(Enum mkeyType, Enum eventType, Map query, Date after) { // used from ChaosMonkey List evts = new LinkedList(); @@ -120,6 +165,7 @@ public List findEvents(Enum mkeyType, Enum eventType, Map } }; + @Override public MonkeyRecorder recorder() { return recorder; } @@ -144,4 +190,4 @@ public String getEventReport() { } return report.toString(); } - } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/TestSimpleDBRecorder.java b/src/test/java/com/netflix/simianarmy/aws/TestSimpleDBRecorder.java index 35940945..6d77a847 100644 --- a/src/test/java/com/netflix/simianarmy/aws/TestSimpleDBRecorder.java +++ b/src/test/java/com/netflix/simianarmy/aws/TestSimpleDBRecorder.java @@ -18,49 +18,52 @@ */ package com.netflix.simianarmy.aws; -import java.util.Map; -import java.util.List; -import java.util.LinkedList; -import java.util.LinkedHashMap; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; import java.util.Date; import java.util.HashMap; -import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.AWSCredentials; +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.Test; import com.amazonaws.services.simpledb.AmazonSimpleDB; +import com.amazonaws.services.simpledb.model.Attribute; +import com.amazonaws.services.simpledb.model.Item; import com.amazonaws.services.simpledb.model.PutAttributesRequest; import com.amazonaws.services.simpledb.model.ReplaceableAttribute; import com.amazonaws.services.simpledb.model.SelectRequest; import com.amazonaws.services.simpledb.model.SelectResult; -import com.amazonaws.services.simpledb.model.Item; -import com.amazonaws.services.simpledb.model.Attribute; - -import com.netflix.simianarmy.MonkeyRecorder.Event; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import org.mockito.ArgumentCaptor; -import static org.mockito.Matchers.any; - -import org.testng.annotations.Test; -import org.testng.Assert; +import com.netflix.simianarmy.client.aws.AWSClient; // CHECKSTYLE IGNORE MagicNumberCheck public class TestSimpleDBRecorder extends SimpleDBRecorder { - public TestSimpleDBRecorder() { - super("accessKey", "secretKey", "region", "DOMAIN"); + + private static AWSClient makeMockAWSClient() { + AmazonSimpleDB sdbMock = mock(AmazonSimpleDB.class); + AWSClient awsClient = mock(AWSClient.class); + when(awsClient.sdbClient()).thenReturn(sdbMock); + when(awsClient.region()).thenReturn("region"); + return awsClient; } - public TestSimpleDBRecorder(AWSCredentials cred) { - super(cred, "region", "DOMAIN"); + public TestSimpleDBRecorder() { + super(makeMockAWSClient(), "DOMAIN"); + sdbMock = super.sdbClient(); } - private AmazonSimpleDB sdbMock = mock(AmazonSimpleDB.class); + private final AmazonSimpleDB sdbMock; + @Override protected AmazonSimpleDB sdbClient() { return sdbMock; } @@ -74,7 +77,7 @@ public void testClients() { TestSimpleDBRecorder recorder1 = new TestSimpleDBRecorder(); Assert.assertNotNull(recorder1.superSdbClient(), "non null super sdbClient"); - TestSimpleDBRecorder recorder2 = new TestSimpleDBRecorder(new BasicAWSCredentials("accessKey", "secretKey")); + TestSimpleDBRecorder recorder2 = new TestSimpleDBRecorder(); Assert.assertNotNull(recorder2.superSdbClient(), "non null super sdbClient"); } @@ -98,7 +101,7 @@ public void testRecordEvent() { PutAttributesRequest req = arg.getValue(); Assert.assertEquals(req.getDomainName(), "DOMAIN"); - Assert.assertEquals(req.getItemName(), "MONKEY-testId-region"); + Assert.assertEquals(req.getItemName(), "MONKEY-testId-region-" + evt.eventTime().getTime()); Map map = new HashMap(); for (ReplaceableAttribute attr : req.getAttributes()) { map.put(attr.getName(), attr.getValue()); diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/TestAWSResource.java b/src/test/java/com/netflix/simianarmy/aws/janitor/TestAWSResource.java new file mode 100644 index 00000000..f4cdfe5d --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/TestAWSResource.java @@ -0,0 +1,155 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import java.lang.reflect.Field; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; + +public class TestAWSResource { + /** Make sure the getFieldToValue returns the right field and values. + * @throws Exception **/ + @Test + public void testFieldToValueMapWithoutNullForInstance() throws Exception { + Date now = new Date(); + Resource resource = getTestingResource(now); + + Map resourceFieldValueMap = resource.getFieldToValueMap(); + + verifyMapsAreEqual(resourceFieldValueMap, + getTestingFieldValueMap(now, getTestingFields())); + } + + /** + * When all fields are null, the map returned is empty. + */ + @Test + public void testFieldToValueMapWithNull() { + Resource resource = new AWSResource(); + Map resourceFieldValueMap = resource.getFieldToValueMap(); + // The only value in the map is the boolean of opt out + Assert.assertEquals(resourceFieldValueMap.size(), 1); + } + + @Test + public void testParseFieldToValueMap() throws Exception { + Date now = new Date(); + Map map = getTestingFieldValueMap(now, getTestingFields()); + AWSResource resource = AWSResource.parseFieldtoValueMap(map); + + Map resourceFieldValueMap = resource.getFieldToValueMap(); + + verifyMapsAreEqual(resourceFieldValueMap, map); + } + + @Test + public void testClone() { + Date now = new Date(); + Resource resource = getTestingResource(now); + Resource clone = resource.cloneResource(); + verifyMapsAreEqual(clone.getFieldToValueMap(), resource.getFieldToValueMap()); + verifyTagsAreEqual(clone, resource); + } + + private void verifyMapsAreEqual(Map map1, Map map2) { + Assert.assertFalse(map1 == null ^ map2 == null); + Assert.assertEquals(map1.size(), map2.size()); + for (Map.Entry entry : map1.entrySet()) { + Assert.assertEquals(entry.getValue(), map2.get(entry.getKey())); + } + } + + private void verifyTagsAreEqual(Resource r1, Resource r2) { + Collection keys1 = r1.getAllTagKeys(); + Collection keys2 = r2.getAllTagKeys(); + Assert.assertEquals(keys1.size(), keys2.size()); + + for (String key : keys1) { + Assert.assertEquals(r1.getTag(key), r2.getTag(key)); + } + } + + private Map getTestingFieldValueMap(Date defaultDate, Map additionalFields) + throws Exception { + Field[] fields = AWSResource.class.getFields(); + Map fieldToValue = new HashMap(); + + String dateString = AWSResource.DATE_FORMATTER.print(defaultDate.getTime()); + for (Field field : fields) { + if (field.getName().startsWith("FIELD_")) { + String value; + String key = (String) (field.get(null)); + if (field.getName().endsWith("TIME")) { + value = dateString; + } else if (field.getName().equals("FIELD_STATE")) { + value = "MARKED"; + } else if (field.getName().equals("FIELD_RESOURCE_TYPE")) { + value = "INSTANCE"; + } else if (field.getName().equals("FIELD_OPT_OUT_OF_JANITOR")) { + value = "false"; + } else { + value = (String) (field.get(null)); + } + fieldToValue.put(key, value); + } + } + if (additionalFields != null) { + fieldToValue.putAll(additionalFields); + } + return fieldToValue; + } + + private Resource getTestingResource(Date now) { + String id = "resourceId"; + Resource resource = new AWSResource().withId(id).withRegion("region").withResourceType(AWSResourceType.INSTANCE) + .withState(Resource.CleanupState.MARKED).withDescription("description") + .withExpectedTerminationTime(now).withActualTerminationTime(now) + .withLaunchTime(now).withMarkTime(now).withNnotificationTime(now).withOwnerEmail("ownerEmail") + .withTerminationReason("terminationReason").withOptOutOfJanitor(false); + ((AWSResource) resource).setAWSResourceState("awsResourceState"); + + for (Map.Entry field : getTestingFields().entrySet()) { + resource.setAdditionalField(field.getKey(), field.getValue()); + } + + for (int i = 1; i < 10; i++) { + resource.setTag("tagKey_" + i, "tagValue_" + i); + } + + return resource; + } + + private Map getTestingFields() { + Map map = new HashMap(); + for (int i = 0; i < 10; i++) { + map.put("name" + i, "value" + i); + } + return map; + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/TestSimpleDBJanitorResourceTracker.java b/src/test/java/com/netflix/simianarmy/aws/janitor/TestSimpleDBJanitorResourceTracker.java new file mode 100644 index 00000000..58af3f37 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/TestSimpleDBJanitorResourceTracker.java @@ -0,0 +1,220 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +// CHECKSTYLE IGNORE ParameterNumber +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.joda.time.DateTime; +import org.mockito.ArgumentCaptor; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.amazonaws.services.simpledb.AmazonSimpleDB; +import com.amazonaws.services.simpledb.model.Attribute; +import com.amazonaws.services.simpledb.model.Item; +import com.amazonaws.services.simpledb.model.PutAttributesRequest; +import com.amazonaws.services.simpledb.model.ReplaceableAttribute; +import com.amazonaws.services.simpledb.model.SelectRequest; +import com.amazonaws.services.simpledb.model.SelectResult; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +public class TestSimpleDBJanitorResourceTracker extends SimpleDBJanitorResourceTracker { + + private static AWSClient makeMockAWSClient() { + AmazonSimpleDB sdbMock = mock(AmazonSimpleDB.class); + AWSClient awsClient = mock(AWSClient.class); + when(awsClient.sdbClient()).thenReturn(sdbMock); + return awsClient; + } + + public TestSimpleDBJanitorResourceTracker() { + super(makeMockAWSClient(), "DOMAIN"); + sdbMock = super.getSimpleDBClient(); + } + + private final AmazonSimpleDB sdbMock; + + @Test + public void testAddResource() { + String id = "i-1234567"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + Resource resource = new AWSResource().withId(id).withResourceType(resourceType) + .withDescription(description).withOwnerEmail(ownerEmail).withRegion(region) + .withState(state).withTerminationReason(terminationReason) + .withExpectedTerminationTime(expectedTerminationTime) + .withMarkTime(markTime).withOptOutOfJanitor(false) + .setAdditionalField(fieldName, fieldValue); + ArgumentCaptor arg = ArgumentCaptor.forClass(PutAttributesRequest.class); + + TestSimpleDBJanitorResourceTracker tracker = new TestSimpleDBJanitorResourceTracker(); + + tracker.addOrUpdate(resource); + verify(tracker.sdbMock).putAttributes(arg.capture()); + PutAttributesRequest req = arg.getValue(); + + Assert.assertEquals(req.getDomainName(), "DOMAIN"); + Assert.assertEquals(req.getItemName(), getSimpleDBItemName(resource)); + Map map = new HashMap(); + for (ReplaceableAttribute attr : req.getAttributes()) { + map.put(attr.getName(), attr.getValue()); + } + + Assert.assertEquals(map.remove(AWSResource.FIELD_RESOURCE_ID), id); + Assert.assertEquals(map.remove(AWSResource.FIELD_DESCRIPTION), description); + Assert.assertEquals(map.remove(AWSResource.FIELD_EXPECTED_TERMINATION_TIME), + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + Assert.assertEquals(map.remove(AWSResource.FIELD_MARK_TIME), + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + Assert.assertEquals(map.remove(AWSResource.FIELD_REGION), region); + Assert.assertEquals(map.remove(AWSResource.FIELD_OWNER_EMAIL), ownerEmail); + Assert.assertEquals(map.remove(AWSResource.FIELD_RESOURCE_TYPE), resourceType.name()); + Assert.assertEquals(map.remove(AWSResource.FIELD_STATE), state.name()); + Assert.assertEquals(map.remove(AWSResource.FIELD_TERMINATION_REASON), terminationReason); + Assert.assertEquals(map.remove(AWSResource.FIELD_OPT_OUT_OF_JANITOR), "false"); + Assert.assertEquals(map.remove(fieldName), fieldValue); + Assert.assertEquals(map.size(), 0); + } + + + @Test + public void testGetResources() { + String id1 = "id-1"; + String id2 = "id-2"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + + SelectResult result1 = mkSelectResult(id1, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, false, fieldName, fieldValue); + result1.setNextToken("nextToken"); + SelectResult result2 = mkSelectResult(id2, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, true, fieldName, fieldValue); + + ArgumentCaptor arg = ArgumentCaptor.forClass(SelectRequest.class); + + TestSimpleDBJanitorResourceTracker tracker = new TestSimpleDBJanitorResourceTracker(); + + when(tracker.sdbMock.select(any(SelectRequest.class))).thenReturn(result1).thenReturn(result2); + + verifyResources(tracker.getResources(resourceType, state, region), + id1, id2, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, fieldName, fieldValue); + + verify(tracker.sdbMock, times(2)).select(arg.capture()); + } + + private void verifyResources(List resources, String id1, String id2, AWSResourceType resourceType, + Resource.CleanupState state, String description, String ownerEmail, String region, + String terminationReason, Date expectedTerminationTime, Date markTime, String fieldName, + String fieldValue) { + Assert.assertEquals(resources.size(), 2); + + Assert.assertEquals(resources.get(0).getId(), id1); + Assert.assertEquals(resources.get(0).getResourceType(), resourceType); + Assert.assertEquals(resources.get(0).getState(), state); + Assert.assertEquals(resources.get(0).getDescription(), description); + Assert.assertEquals(resources.get(0).getOwnerEmail(), ownerEmail); + Assert.assertEquals(resources.get(0).getRegion(), region); + Assert.assertEquals(resources.get(0).getTerminationReason(), terminationReason); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(0).getExpectedTerminationTime().getTime()), + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(0).getMarkTime().getTime()), + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + Assert.assertEquals(resources.get(0).getAdditionalField(fieldName), fieldValue); + Assert.assertEquals(resources.get(0).isOptOutOfJanitor(), false); + + Assert.assertEquals(resources.get(1).getId(), id2); + Assert.assertEquals(resources.get(1).getResourceType(), resourceType); + Assert.assertEquals(resources.get(1).getState(), state); + Assert.assertEquals(resources.get(1).getDescription(), description); + Assert.assertEquals(resources.get(1).getOwnerEmail(), ownerEmail); + Assert.assertEquals(resources.get(1).getRegion(), region); + Assert.assertEquals(resources.get(1).getTerminationReason(), terminationReason); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(1).getExpectedTerminationTime().getTime()), + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(1).getMarkTime().getTime()), + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + Assert.assertEquals(resources.get(1).isOptOutOfJanitor(), true); + Assert.assertEquals(resources.get(1).getAdditionalField(fieldName), fieldValue); + } + + private SelectResult mkSelectResult(String id, AWSResourceType resourceType, Resource.CleanupState state, + String description, String ownerEmail, String region, String terminationReason, + Date expectedTerminationTime, Date markTime, boolean optOut, String fieldName, String fieldValue) { + Item item = new Item(); + List attrs = new LinkedList(); + attrs.add(new Attribute(AWSResource.FIELD_RESOURCE_ID, id)); + attrs.add(new Attribute(AWSResource.FIELD_RESOURCE_TYPE, resourceType.name())); + attrs.add(new Attribute(AWSResource.FIELD_DESCRIPTION, description)); + attrs.add(new Attribute(AWSResource.FIELD_REGION, region)); + attrs.add(new Attribute(AWSResource.FIELD_STATE, state.name())); + attrs.add(new Attribute(AWSResource.FIELD_OWNER_EMAIL, ownerEmail)); + attrs.add(new Attribute(AWSResource.FIELD_TERMINATION_REASON, terminationReason)); + attrs.add(new Attribute(AWSResource.FIELD_EXPECTED_TERMINATION_TIME, + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime()))); + attrs.add(new Attribute(AWSResource.FIELD_MARK_TIME, + AWSResource.DATE_FORMATTER.print(markTime.getTime()))); + attrs.add(new Attribute(AWSResource.FIELD_OPT_OUT_OF_JANITOR, String.valueOf(optOut))); + attrs.add(new Attribute(fieldName, fieldValue)); + + item.setAttributes(attrs); + item.setName(String.format("%s-%s-%s", resourceType.name(), id, region)); + SelectResult result = new SelectResult(); + result.setItems(Arrays.asList(item)); + return result; + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestASGJanitorCrawler.java b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestASGJanitorCrawler.java new file mode 100644 index 00000000..892e95e7 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestASGJanitorCrawler.java @@ -0,0 +1,124 @@ +// CHECKSTYLE IGNORE Javadoc +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.amazonaws.services.autoscaling.model.AutoScalingGroup; +import com.amazonaws.services.autoscaling.model.SuspendedProcess; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +public class TestASGJanitorCrawler { + + @Test + public void testResourceTypes() { + ASGJanitorCrawler crawler = new ASGJanitorCrawler(createMockAWSClient(createASGList())); + EnumSet types = crawler.resourceTypes(); + Assert.assertEquals(types.size(), 1); + Assert.assertEquals(types.iterator().next().name(), "ASG"); + } + + @Test + public void testInstancesWithNullNames() { + List asgList = createASGList(); + AWSClient awsMock = createMockAWSClient(asgList); + ASGJanitorCrawler crawler = new ASGJanitorCrawler(awsMock); + List resources = crawler.resources(); + verifyASGList(resources, asgList); + } + + @Test + public void testInstancesWithNames() { + List asgList = createASGList(); + String[] asgNames = {"asg1", "asg2"}; + AWSClient awsMock = createMockAWSClient(asgList, asgNames); + ASGJanitorCrawler crawler = new ASGJanitorCrawler(awsMock); + List resources = crawler.resources(asgNames); + verifyASGList(resources, asgList); + } + + @Test + public void testInstancesWithResourceType() { + List asgList = createASGList(); + AWSClient awsMock = createMockAWSClient(asgList); + ASGJanitorCrawler crawler = new ASGJanitorCrawler(awsMock); + for (AWSResourceType resourceType : AWSResourceType.values()) { + List resources = crawler.resources(resourceType); + if (resourceType == AWSResourceType.ASG) { + verifyASGList(resources, asgList); + } else { + Assert.assertTrue(resources.isEmpty()); + } + } + } + + private void verifyASGList(List resources, List asgList) { + Assert.assertEquals(resources.size(), asgList.size()); + for (int i = 0; i < resources.size(); i++) { + AutoScalingGroup asg = asgList.get(i); + verifyASG(resources.get(i), asg.getAutoScalingGroupName()); + } + } + + private void verifyASG(Resource asg, String asgName) { + Assert.assertEquals(asg.getResourceType(), AWSResourceType.ASG); + Assert.assertEquals(asg.getId(), asgName); + Assert.assertEquals(asg.getRegion(), "us-east-1"); + Assert.assertEquals(asg.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME), + "2012-12-03T23:00:03"); + } + + private AWSClient createMockAWSClient(List asgList, String... asgNames) { + AWSClient awsMock = mock(AWSClient.class); + when(awsMock.describeAutoScalingGroups(asgNames)).thenReturn(asgList); + when(awsMock.region()).thenReturn("us-east-1"); + return awsMock; + } + + private List createASGList() { + List asgList = new LinkedList(); + asgList.add(mkASG("asg1")); + asgList.add(mkASG("asg2")); + return asgList; + } + + private AutoScalingGroup mkASG(String asgName) { + AutoScalingGroup asg = new AutoScalingGroup().withAutoScalingGroupName(asgName); + // set the suspended processes + List sps = new ArrayList(); + sps.add(new SuspendedProcess().withProcessName("Launch") + .withSuspensionReason("User suspended at 2012-12-02T23:00:03")); + sps.add(new SuspendedProcess().withProcessName("AddToLoadBalancer") + .withSuspensionReason("User suspended at 2012-12-03T23:00:03")); + asg.setSuspendedProcesses(sps); + return asg; + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSSnapshotJanitorCrawler.java b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSSnapshotJanitorCrawler.java new file mode 100644 index 00000000..073e9125 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSSnapshotJanitorCrawler.java @@ -0,0 +1,119 @@ +//CHECKSTYLE IGNORE Javadoc +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Date; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.amazonaws.services.ec2.model.Snapshot; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +public class TestEBSSnapshotJanitorCrawler { + + @Test + public void testResourceTypes() { + Date startTime = new Date(); + List snapshotList = createSnapshotList(startTime); + EBSSnapshotJanitorCrawler crawler = new EBSSnapshotJanitorCrawler(createMockAWSClient(snapshotList)); + EnumSet types = crawler.resourceTypes(); + Assert.assertEquals(types.size(), 1); + Assert.assertEquals(types.iterator().next().name(), "EBS_SNAPSHOT"); + } + + @Test + public void testSnapshotsWithNullIds() { + Date startTime = new Date(); + List snapshotList = createSnapshotList(startTime); + EBSSnapshotJanitorCrawler crawler = new EBSSnapshotJanitorCrawler(createMockAWSClient(snapshotList)); + List resources = crawler.resources(); + verifySnapshotList(resources, snapshotList, startTime); + } + + @Test + public void testSnapshotsWithIds() { + Date startTime = new Date(); + List snapshotList = createSnapshotList(startTime); + String[] ids = {"snap-123456780", "snap-123456781"}; + EBSSnapshotJanitorCrawler crawler = new EBSSnapshotJanitorCrawler(createMockAWSClient(snapshotList, ids)); + List resources = crawler.resources(ids); + verifySnapshotList(resources, snapshotList, startTime); + } + + @Test + public void testSnapshotsWithResourceType() { + Date startTime = new Date(); + List snapshotList = createSnapshotList(startTime); + EBSSnapshotJanitorCrawler crawler = new EBSSnapshotJanitorCrawler(createMockAWSClient(snapshotList)); + for (AWSResourceType resourceType : AWSResourceType.values()) { + List resources = crawler.resources(resourceType); + if (resourceType == AWSResourceType.EBS_SNAPSHOT) { + verifySnapshotList(resources, snapshotList, startTime); + } else { + Assert.assertTrue(resources.isEmpty()); + } + } + } + + private void verifySnapshotList(List resources, List snapshotList, Date startTime) { + Assert.assertEquals(resources.size(), snapshotList.size()); + for (int i = 0; i < resources.size(); i++) { + Snapshot snapshot = snapshotList.get(i); + verifySnapshot(resources.get(i), snapshot.getSnapshotId(), startTime); + } + } + + private void verifySnapshot(Resource snapshot, String snapshotId, Date startTime) { + Assert.assertEquals(snapshot.getResourceType(), AWSResourceType.EBS_SNAPSHOT); + Assert.assertEquals(snapshot.getId(), snapshotId); + Assert.assertEquals(snapshot.getRegion(), "us-east-1"); + Assert.assertEquals(((AWSResource) snapshot).getAWSResourceState(), "completed"); + Assert.assertEquals(snapshot.getLaunchTime(), startTime); + } + + private AWSClient createMockAWSClient(List snapshotList, String... ids) { + AWSClient awsMock = mock(AWSClient.class); + when(awsMock.describeSnapshots(ids)).thenReturn(snapshotList); + when(awsMock.region()).thenReturn("us-east-1"); + return awsMock; + } + + private List createSnapshotList(Date startTime) { + List snapshotList = new LinkedList(); + snapshotList.add(mkSnapshot("snap-123456780", startTime)); + snapshotList.add(mkSnapshot("snap-123456781", startTime)); + return snapshotList; + } + + private Snapshot mkSnapshot(String snapshotId, Date startTime) { + return new Snapshot().withSnapshotId(snapshotId).withState("completed").withStartTime(startTime); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSVolumeJanitorCrawler.java b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSVolumeJanitorCrawler.java new file mode 100644 index 00000000..2ee102f8 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestEBSVolumeJanitorCrawler.java @@ -0,0 +1,119 @@ +//CHECKSTYLE IGNORE Javadoc +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Date; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.amazonaws.services.ec2.model.Volume; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +public class TestEBSVolumeJanitorCrawler { + + @Test + public void testResourceTypes() { + Date createTime = new Date(); + List volumeList = createVolumeList(createTime); + EBSVolumeJanitorCrawler crawler = new EBSVolumeJanitorCrawler(createMockAWSClient(volumeList)); + EnumSet types = crawler.resourceTypes(); + Assert.assertEquals(types.size(), 1); + Assert.assertEquals(types.iterator().next().name(), "EBS_VOLUME"); + } + + @Test + public void testVolumesWithNullIds() { + Date createTime = new Date(); + List volumeList = createVolumeList(createTime); + EBSVolumeJanitorCrawler crawler = new EBSVolumeJanitorCrawler(createMockAWSClient(volumeList)); + List resources = crawler.resources(); + verifyVolumeList(resources, volumeList, createTime); + } + + @Test + public void testVolumesWithIds() { + Date createTime = new Date(); + List volumeList = createVolumeList(createTime); + String[] ids = {"vol-123456780", "vol-123456781"}; + EBSVolumeJanitorCrawler crawler = new EBSVolumeJanitorCrawler(createMockAWSClient(volumeList, ids)); + List resources = crawler.resources(ids); + verifyVolumeList(resources, volumeList, createTime); + } + + @Test + public void testVolumesWithResourceType() { + Date createTime = new Date(); + List volumeList = createVolumeList(createTime); + EBSVolumeJanitorCrawler crawler = new EBSVolumeJanitorCrawler(createMockAWSClient(volumeList)); + for (AWSResourceType resourceType : AWSResourceType.values()) { + List resources = crawler.resources(resourceType); + if (resourceType == AWSResourceType.EBS_VOLUME) { + verifyVolumeList(resources, volumeList, createTime); + } else { + Assert.assertTrue(resources.isEmpty()); + } + } + } + + private void verifyVolumeList(List resources, List volumeList, Date createTime) { + Assert.assertEquals(resources.size(), volumeList.size()); + for (int i = 0; i < resources.size(); i++) { + Volume volume = volumeList.get(i); + verifyVolume(resources.get(i), volume.getVolumeId(), createTime); + } + } + + private void verifyVolume(Resource volume, String volumeId, Date createTime) { + Assert.assertEquals(volume.getResourceType(), AWSResourceType.EBS_VOLUME); + Assert.assertEquals(volume.getId(), volumeId); + Assert.assertEquals(volume.getRegion(), "us-east-1"); + Assert.assertEquals(((AWSResource) volume).getAWSResourceState(), "available"); + Assert.assertEquals(volume.getLaunchTime(), createTime); + } + + private AWSClient createMockAWSClient(List volumeList, String... ids) { + AWSClient awsMock = mock(AWSClient.class); + when(awsMock.describeVolumes(ids)).thenReturn(volumeList); + when(awsMock.region()).thenReturn("us-east-1"); + return awsMock; + } + + private List createVolumeList(Date createTime) { + List volumeList = new LinkedList(); + volumeList.add(mkVolume("vol-123456780", createTime)); + volumeList.add(mkVolume("vol-123456781", createTime)); + return volumeList; + } + + private Volume mkVolume(String volumeId, Date createTime) { + return new Volume().withVolumeId(volumeId).withState("available").withCreateTime(createTime); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestInstanceJanitorCrawler.java b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestInstanceJanitorCrawler.java new file mode 100644 index 00000000..32caa47d --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/crawler/TestInstanceJanitorCrawler.java @@ -0,0 +1,149 @@ +// CHECKSTYLE IGNORE Javadoc +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.crawler; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.amazonaws.services.autoscaling.model.AutoScalingInstanceDetails; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceState; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.client.aws.AWSClient; + +public class TestInstanceJanitorCrawler { + + @Test + public void testResourceTypes() { + List instanceDetailsList = createInstanceDetailsList(); + List instanceList = createInstanceList(); + InstanceJanitorCrawler crawler = new InstanceJanitorCrawler(createMockAWSClient( + instanceDetailsList, instanceList)); + EnumSet types = crawler.resourceTypes(); + Assert.assertEquals(types.size(), 1); + Assert.assertEquals(types.iterator().next().name(), "INSTANCE"); + } + + @Test + public void testInstancesWithNullIds() { + List instanceDetailsList = createInstanceDetailsList(); + List instanceList = createInstanceList(); + AWSClient awsMock = createMockAWSClient(instanceDetailsList, instanceList); + InstanceJanitorCrawler crawler = new InstanceJanitorCrawler(awsMock); + List resources = crawler.resources(); + verifyInstanceList(resources, instanceDetailsList); + } + + @Test + public void testInstancesWithIds() { + List instanceDetailsList = createInstanceDetailsList(); + List instanceList = createInstanceList(); + String[] ids = {"i-123456780", "i-123456780"}; + AWSClient awsMock = createMockAWSClient(instanceDetailsList, instanceList, ids); + InstanceJanitorCrawler crawler = new InstanceJanitorCrawler(awsMock); + List resources = crawler.resources(ids); + verifyInstanceList(resources, instanceDetailsList); + } + + @Test + public void testInstancesWithResourceType() { + List instanceDetailsList = createInstanceDetailsList(); + List instanceList = createInstanceList(); + AWSClient awsMock = createMockAWSClient(instanceDetailsList, instanceList); + InstanceJanitorCrawler crawler = new InstanceJanitorCrawler(awsMock); + for (AWSResourceType resourceType : AWSResourceType.values()) { + List resources = crawler.resources(resourceType); + if (resourceType == AWSResourceType.INSTANCE) { + verifyInstanceList(resources, instanceDetailsList); + } else { + Assert.assertTrue(resources.isEmpty()); + } + } + } + + @Test + public void testInstancesNotExistingInASG() { + List instanceDetailsList = Collections.emptyList(); + List instanceList = createInstanceList(); + AWSClient awsMock = createMockAWSClient(instanceDetailsList, instanceList); + InstanceJanitorCrawler crawler = new InstanceJanitorCrawler(awsMock); + List resources = crawler.resources(); + Assert.assertEquals(resources.size(), instanceList.size()); + } + + private void verifyInstanceList(List resources, List instanceList) { + Assert.assertEquals(resources.size(), instanceList.size()); + for (int i = 0; i < resources.size(); i++) { + AutoScalingInstanceDetails instance = instanceList.get(i); + verifyInstance(resources.get(i), instance.getInstanceId(), instance.getAutoScalingGroupName()); + } + } + + private void verifyInstance(Resource instance, String instanceId, String asgName) { + Assert.assertEquals(instance.getResourceType(), AWSResourceType.INSTANCE); + Assert.assertEquals(instance.getId(), instanceId); + Assert.assertEquals(instance.getRegion(), "us-east-1"); + Assert.assertEquals(instance.getAdditionalField(InstanceJanitorCrawler.INSTANCE_FIELD_ASG_NAME), asgName); + Assert.assertEquals(((AWSResource) instance).getAWSResourceState(), "running"); + } + + private AWSClient createMockAWSClient(List instanceDetailsList, + List instanceList, String... ids) { + AWSClient awsMock = mock(AWSClient.class); + when(awsMock.describeAutoScalingInstances(ids)).thenReturn(instanceDetailsList); + when(awsMock.describeInstances(ids)).thenReturn(instanceList); + when(awsMock.region()).thenReturn("us-east-1"); + return awsMock; + } + + private List createInstanceDetailsList() { + List instanceList = new LinkedList(); + instanceList.add(mkInstanceDetails("i-123456780", "asg1")); + instanceList.add(mkInstanceDetails("i-123456781", "asg2")); + return instanceList; + } + + private AutoScalingInstanceDetails mkInstanceDetails(String instanceId, String asgName) { + return new AutoScalingInstanceDetails().withInstanceId(instanceId).withAutoScalingGroupName(asgName); + } + + private List createInstanceList() { + List instanceList = new LinkedList(); + instanceList.add(mkInstance("i-123456780")); + instanceList.add(mkInstance("i-123456781")); + return instanceList; + } + + private Instance mkInstance(String instanceId) { + return new Instance().withInstanceId(instanceId) + .withState(new InstanceState().withName("running")); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/TestMonkeyCalendar.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/TestMonkeyCalendar.java new file mode 100644 index 00000000..f78f0b3f --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/TestMonkeyCalendar.java @@ -0,0 +1,61 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor.rule; + +import java.util.Calendar; +import java.util.Date; + +import org.joda.time.DateTime; + +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.MonkeyCalendar; + +/** + * The class is an implementation of MonkeyCalendar that can always run and + * considers calendar days only when calculating the termination date. + * + */ +public class TestMonkeyCalendar implements MonkeyCalendar { + @Override + public boolean isMonkeyTime(Monkey monkey) { + return true; + } + + @Override + public int openHour() { + return 0; + } + + @Override + public int closeHour() { + return 24; + } + + @Override + public Calendar now() { + return Calendar.getInstance(); + } + + @Override + public Date getBusinessDay(Date date, int n) { + DateTime target = new DateTime(date.getTime()).plusDays(n); + return new Date(target.getMillis()); + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestOldEmptyASGRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestOldEmptyASGRule.java new file mode 100644 index 00000000..45554f1a --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestOldEmptyASGRule.java @@ -0,0 +1,181 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.asg; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; + + +public class TestOldEmptyASGRule { + @Test + public void testEmptyASGWithObsoleteLaunchConfig() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME, + String.valueOf(now.minusDays(launchConfiguAgeThreshold + 1).getMillis())); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDays, now); + } + + @Test + public void testEmptyASGWithValidLaunchConfig() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME, + String.valueOf(now.minusDays(launchConfiguAgeThreshold - 1).getMillis())); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testASGWithInstances() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "2"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_INSTANCES, "i-1,i-2"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME, + String.valueOf(now.minusDays(launchConfiguAgeThreshold + 1).getMillis())); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testASGWithoutInstanceAndNonZeroSize() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "2"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME, + String.valueOf(now.minusDays(launchConfiguAgeThreshold + 1).getMillis())); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testEmptyASGWithoutLaunchConfig() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDays, now); + } + + @Test + public void testEmptyASGWithLaunchConfigWithoutCreateTime() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int launchConfiguAgeThreshold = 60; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + int retentionDays = 3; + OldEmptyASGRule rule = new OldEmptyASGRule(calendar, launchConfiguAgeThreshold, retentionDays, null); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + resource.setExpectedTerminationTime(oldTermDate); + resource.setTerminationReason(oldTermReason); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullResource() { + OldEmptyASGRule rule = new OldEmptyASGRule(new TestMonkeyCalendar(), 3, 60, null); + rule.isValid(null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDays() { + new OldEmptyASGRule(new TestMonkeyCalendar(), -1, 60, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeLaunchConfigAgeThreshold() { + new OldEmptyASGRule(new TestMonkeyCalendar(), 3, -1, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullCalendar() { + new OldEmptyASGRule(null, 3, 60, null); + } + + @Test + public void testNonASGResource() { + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE); + OldEmptyASGRule rule = new OldEmptyASGRule(new TestMonkeyCalendar(), 3, 60, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + /** Verify that the termination date is roughly rentionDays from now **/ + private void verifyTerminationTime(Resource resource, int retentionDays, DateTime now) { + long days = (resource.getExpectedTerminationTime().getTime() - now.getMillis()) / (24 * 60 * 60 * 1000); + Assert.assertEquals(days, retentionDays); + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestSuspendedASGRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestSuspendedASGRule.java new file mode 100644 index 00000000..577c31d2 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/asg/TestSuspendedASGRule.java @@ -0,0 +1,182 @@ +//CHECKSTYLE IGNORE Javadoc +//CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.asg; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; + + +public class TestSuspendedASGRule { + @Test + public void testEmptyASGSuspendedMoreThanThreshold() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + int suspensionAgeThreshold = 2; + DateTime suspensionTime = now.minusDays(suspensionAgeThreshold + 1); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME, + ASGJanitorCrawler.SUSPENSION_TIME_FORMATTER.print(suspensionTime)); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDays, now); + } + + @Test + public void testEmptyASGSuspendedLessThanThreshold() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME, "launchConfig"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int suspensionAgeThreshold = 2; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME, + String.valueOf(now.minusDays(suspensionAgeThreshold + 1).getMillis())); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testASGWithInstances() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "2"); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_INSTANCES, "i-1,i-2"); + int suspensionAgeThreshold = 2; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + DateTime suspensionTime = now.minusDays(suspensionAgeThreshold + 1); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME, + ASGJanitorCrawler.SUSPENSION_TIME_FORMATTER.print(suspensionTime)); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testASGWithoutInstanceAndNonZeroSize() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "2"); + int suspensionAgeThreshold = 2; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + DateTime suspensionTime = now.minusDays(suspensionAgeThreshold + 1); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME, + ASGJanitorCrawler.SUSPENSION_TIME_FORMATTER.print(suspensionTime)); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testEmptyASGNotSuspended() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + int suspensionAgeThreshold = 2; + MonkeyCalendar calendar = new TestMonkeyCalendar(); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + MonkeyCalendar calendar = new TestMonkeyCalendar(); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + int suspensionAgeThreshold = 2; + DateTime suspensionTime = now.minusDays(suspensionAgeThreshold + 1); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME, + ASGJanitorCrawler.SUSPENSION_TIME_FORMATTER.print(suspensionTime)); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + resource.setExpectedTerminationTime(oldTermDate); + resource.setTerminationReason(oldTermReason); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullResource() { + SuspendedASGRule rule = new SuspendedASGRule(new TestMonkeyCalendar(), 3, 2, null); + rule.isValid(null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDays() { + new SuspendedASGRule(new TestMonkeyCalendar(), -1, 2, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeLaunchConfigAgeThreshold() { + new SuspendedASGRule(new TestMonkeyCalendar(), 3, -1, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullCalendar() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_MAX_SIZE, "0"); + MonkeyCalendar calendar = new TestMonkeyCalendar(); + int suspensionAgeThreshold = 2; + resource.setAdditionalField(ASGJanitorCrawler.ASG_FIELD_SUSPENSION_TIME, "foo"); + int retentionDays = 3; + SuspendedASGRule rule = new SuspendedASGRule(calendar, suspensionAgeThreshold, retentionDays, null); + Assert.assertFalse(rule.isValid(resource)); + } + + @Test + public void testNonASGResource() { + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE); + SuspendedASGRule rule = new SuspendedASGRule(new TestMonkeyCalendar(), 3, 2, null); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testSuspensionTimeIncorrectFormat() { + new SuspendedASGRule(null, 3, 2, null); + } + + /** Verify that the termination date is roughly rentionDays from now **/ + private void verifyTerminationTime(Resource resource, int retentionDays, DateTime now) { + long days = (resource.getExpectedTerminationTime().getTime() - now.getMillis()) / (24 * 60 * 60 * 1000); + Assert.assertEquals(days, retentionDays); + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/instance/TestOrphanedInstanceRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/instance/TestOrphanedInstanceRule.java new file mode 100644 index 00000000..3015765c --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/instance/TestOrphanedInstanceRule.java @@ -0,0 +1,179 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.instance; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.crawler.InstanceJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; + +public class TestOrphanedInstanceRule { + + @Test + public void testOrphanedInstancesWithOwner() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())) + .withOwnerEmail("owner@foo.com"); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDaysWithOwner, now); + } + + @Test + public void testOrphanedInstancesWithoutOwner() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDaysWithoutOwner, now); + } + + @Test + public void testOrphanedInstancesWithoutLaunchTime() { + int ageThreshold = 5; + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testOrphanedInstancesWithLaunchTimeNotExpires() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withLaunchTime(new Date(now.minusDays(ageThreshold - 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testNonOrphanedInstances() { + int ageThreshold = 5; + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .setAdditionalField(InstanceJanitorCrawler.INSTANCE_FIELD_ASG_NAME, "asg1"); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + DateTime now = DateTime.now(); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + int ageThreshold = 5; + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())) + .withExpectedTerminationTime(oldTermDate) + .withTerminationReason(oldTermReason); + ((AWSResource) resource).setAWSResourceState("running"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), + ageThreshold, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullResource() { + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), 5, 4, 8); + rule.isValid(null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeAgeThreshold() { + new OrphanedInstanceRule(new TestMonkeyCalendar(), -1, 4, 8); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDaysWithOwner() { + new OrphanedInstanceRule(new TestMonkeyCalendar(), 5, -4, 8); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDaysWithoutOwner() { + new OrphanedInstanceRule(new TestMonkeyCalendar(), 5, 4, -8); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullCalendar() { + new OrphanedInstanceRule(null, 5, 4, 8); + } + + @Test + public void testNonInstanceResource() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + ((AWSResource) resource).setAWSResourceState("running"); + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), 0, 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testNonRunningInstance() { + Resource resource = new AWSResource().withId("i-123").withResourceType(AWSResourceType.INSTANCE); + ((AWSResource) resource).setAWSResourceState("stopping"); + OrphanedInstanceRule rule = new OrphanedInstanceRule(new TestMonkeyCalendar(), 0, 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + /** Verify that the termination date is roughly rentionDays from now **/ + private void verifyTerminationTime(Resource resource, int retentionDays, DateTime now) { + long days = (resource.getExpectedTerminationTime().getTime() - now.getMillis()) / (24 * 60 * 60 * 1000); + Assert.assertEquals(days, retentionDays); + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/TestNoGeneratedAMIRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/TestNoGeneratedAMIRule.java new file mode 100644 index 00000000..d8dbeac2 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/snapshot/TestNoGeneratedAMIRule.java @@ -0,0 +1,190 @@ +//CHECKSTYLE IGNORE Javadoc +//CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.snapshot; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.crawler.EBSSnapshotJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; +import com.netflix.simianarmy.janitor.JanitorMonkey; + + +public class TestNoGeneratedAMIRule { + + @Test + public void testNonSnapshotResource() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + ((AWSResource) resource).setAWSResourceState("completed"); + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testUncompletedVolume() { + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT); + ((AWSResource) resource).setAWSResourceState("stopped"); + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testTaggedAsNotMark() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + resource.setTag(JanitorMonkey.JANITOR_TAG, "donotmark"); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testUserSpecifiedTerminationDate() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + int retentionDays = 4; + DateTime userDate = new DateTime(now.plusDays(3).toDateMidnight()); + resource.setTag(JanitorMonkey.JANITOR_TAG, + NoGeneratedAMIRule.TERMINATION_DATE_FORMATTER.print(userDate)); + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(resource.getExpectedTerminationTime().getTime(), userDate.getMillis()); + } + + @Test + public void testOldSnapshotWithoutAMI() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDays, now); + } + + @Test + public void testSnapshotWithoutAMINotOld() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold - 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testWithAMIs() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + resource.setAdditionalField(EBSSnapshotJanitorCrawler.SNAPSHOT_FIELD_AMIS, "ami-123"); + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testSnapshotsWithoutLauchTime() { + int ageThreshold = 5; + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT); + ((AWSResource) resource).setAWSResourceState("completed"); + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("snap123").withResourceType(AWSResourceType.EBS_SNAPSHOT) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("completed"); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + int retentionDays = 4; + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + resource.setExpectedTerminationTime(oldTermDate); + resource.setTerminationReason(oldTermReason); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullResource() { + NoGeneratedAMIRule rule = new NoGeneratedAMIRule(new TestMonkeyCalendar(), 5, 4); + rule.isValid(null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeAgeThreshold() { + new NoGeneratedAMIRule(new TestMonkeyCalendar(), -1, 4); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDaysWithOwner() { + new NoGeneratedAMIRule(new TestMonkeyCalendar(), 5, -4); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullCalendar() { + new NoGeneratedAMIRule(null, 5, 4); + } + + /** Verify that the termination date is roughly rentionDays from now **/ + private void verifyTerminationTime(Resource resource, int retentionDays, DateTime now) { + long days = (resource.getExpectedTerminationTime().getTime() - now.getMillis()) / (24 * 60 * 60 * 1000); + Assert.assertEquals(days, retentionDays); + } +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/volume/TestOldDetachedVolumeRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/volume/TestOldDetachedVolumeRule.java new file mode 100644 index 00000000..e1130b07 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/volume/TestOldDetachedVolumeRule.java @@ -0,0 +1,206 @@ +//CHECKSTYLE IGNORE Javadoc +//CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.volume; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.VolumeTaggingMonkey; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; +import com.netflix.simianarmy.janitor.JanitorMonkey; + + +public class TestOldDetachedVolumeRule { + + @Test + public void testNonVolumeResource() { + Resource resource = new AWSResource().withId("asg1").withResourceType(AWSResourceType.ASG); + ((AWSResource) resource).setAWSResourceState("available"); + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testUnavailableVolume() { + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME); + ((AWSResource) resource).setAWSResourceState("stopped"); + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), 0, 0); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testTaggedAsNotMark() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + Date lastDetachTime = new Date(now.minusDays(ageThreshold + 1).getMillis()); + String metaTag = VolumeTaggingMonkey.makeMetaTag(null, null, lastDetachTime); + resource.setTag(JanitorMonkey.JANITOR_META_TAG, metaTag); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + resource.setTag(JanitorMonkey.JANITOR_TAG, "donotmark"); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testNoMetaTag() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + resource.setTag(JanitorMonkey.JANITOR_TAG, "donotmark"); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testUserSpecifiedTerminationDate() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + int retentionDays = 4; + DateTime userDate = new DateTime(now.plusDays(3).toDateMidnight()); + resource.setTag(JanitorMonkey.JANITOR_TAG, + OldDetachedVolumeRule.TERMINATION_DATE_FORMATTER.print(userDate)); + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(resource.getExpectedTerminationTime().getTime(), userDate.getMillis()); + } + + @Test + public void testOldDetachedVolume() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + Date lastDetachTime = new Date(now.minusDays(ageThreshold + 1).getMillis()); + String metaTag = VolumeTaggingMonkey.makeMetaTag(null, null, lastDetachTime); + resource.setTag(JanitorMonkey.JANITOR_META_TAG, metaTag); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertFalse(rule.isValid(resource)); + verifyTerminationTime(resource, retentionDays, now); + } + + @Test + public void testDetachedVolumeNotOld() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + Date lastDetachTime = new Date(now.minusDays(ageThreshold - 1).getMillis()); + String metaTag = VolumeTaggingMonkey.makeMetaTag(null, null, lastDetachTime); + resource.setTag(JanitorMonkey.JANITOR_META_TAG, metaTag); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + @Test + public void testAchedVolume() { + int ageThreshold = 5; + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + String metaTag = VolumeTaggingMonkey.makeMetaTag("i-123", "owner", null); + resource.setTag(JanitorMonkey.JANITOR_META_TAG, metaTag); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + Assert.assertTrue(rule.isValid(resource)); + Assert.assertNull(resource.getExpectedTerminationTime()); + } + + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + DateTime now = DateTime.now(); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + int ageThreshold = 5; + Resource resource = new AWSResource().withId("vol-123").withResourceType(AWSResourceType.EBS_VOLUME) + .withLaunchTime(new Date(now.minusDays(ageThreshold + 1).getMillis())); + ((AWSResource) resource).setAWSResourceState("available"); + Date lastDetachTime = new Date(now.minusDays(ageThreshold + 1).getMillis()); + String metaTag = VolumeTaggingMonkey.makeMetaTag(null, null, lastDetachTime); + resource.setTag(JanitorMonkey.JANITOR_META_TAG, metaTag); + int retentionDays = 4; + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), + ageThreshold, retentionDays); + resource.setExpectedTerminationTime(oldTermDate); + resource.setTerminationReason(oldTermReason); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullResource() { + OldDetachedVolumeRule rule = new OldDetachedVolumeRule(new TestMonkeyCalendar(), 5, 4); + rule.isValid(null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeAgeThreshold() { + new OldDetachedVolumeRule(new TestMonkeyCalendar(), -1, 4); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNgativeRetentionDaysWithOwner() { + new OldDetachedVolumeRule(new TestMonkeyCalendar(), 5, -4); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testNullCalendar() { + new OldDetachedVolumeRule(null, 5, 4); + } + + /** Verify that the termination date is roughly rentionDays from now **/ + private void verifyTerminationTime(Resource resource, int retentionDays, DateTime now) { + long days = (resource.getExpectedTerminationTime().getTime() - now.getMillis()) / (24 * 60 * 60 * 1000); + Assert.assertEquals(days, retentionDays); + } +} diff --git a/src/test/java/com/netflix/simianarmy/basic/TestBasicCalendar.java b/src/test/java/com/netflix/simianarmy/basic/TestBasicCalendar.java index 28456652..4e908a94 100644 --- a/src/test/java/com/netflix/simianarmy/basic/TestBasicCalendar.java +++ b/src/test/java/com/netflix/simianarmy/basic/TestBasicCalendar.java @@ -18,25 +18,20 @@ */ package com.netflix.simianarmy.basic; -import com.netflix.simianarmy.Monkey; -import com.netflix.simianarmy.TestMonkey; +import java.util.Calendar; +import java.util.Properties; +import java.util.TimeZone; -import org.testng.annotations.Test; import org.testng.Assert; import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; -import java.util.TimeZone; -import java.util.Calendar; -import java.util.Properties; +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.TestMonkey; // CHECKSTYLE IGNORE MagicNumberCheck -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class TestBasicCalendar extends BasicCalendar { - private static final Logger LOGGER = LoggerFactory.getLogger(TestBasicCalendar.class); - private static final Properties PROPS = new Properties(); private static final BasicConfiguration CFG = new BasicConfiguration(PROPS); @@ -59,6 +54,7 @@ public void testConstructors() { private Calendar now = super.now(); + @Override public Calendar now() { return (Calendar) now.clone(); } @@ -122,7 +118,7 @@ public Object[][] holidayDataProvider() { } @Test(dataProvider = "holidayDataProvider") - void testHolidays(int month, int dayOfMonth) { + public void testHolidays(int month, int dayOfMonth) { Calendar test = Calendar.getInstance(); test.set(Calendar.YEAR, 2012); test.set(Calendar.MONTH, month); @@ -132,4 +128,83 @@ void testHolidays(int month, int dayOfMonth) { Assert.assertTrue(isHoliday(test), test.getTime().toString() + " is a holiday?"); } + + @Test + public void testGetBusinessDayWihoutGap() { + // the days from 12/3/2012 to 12/7/2012 are all business days + int hour = 10; + Calendar test = Calendar.getInstance(); + test.set(Calendar.YEAR, 2012); + test.set(Calendar.MONTH, Calendar.DECEMBER); + test.set(Calendar.DAY_OF_MONTH, 3); + test.set(Calendar.HOUR_OF_DAY, hour); + int day = test.get(Calendar.DAY_OF_MONTH); + for (int n = 0; n <= 4; n++) { + Calendar businessDay = Calendar.getInstance(); + businessDay.setTime(getBusinessDay(test.getTime(), n)); + Assert.assertEquals(businessDay.get(Calendar.DAY_OF_MONTH), + day + n); + Assert.assertEquals(businessDay.get(Calendar.HOUR_OF_DAY), + hour); + } + } + + @Test + public void testGetBusinessDayWihWeekend() { + // 12/7/2012 is Friday + int hour = 10; + Calendar test = Calendar.getInstance(); + test.set(Calendar.YEAR, 2012); + test.set(Calendar.MONTH, Calendar.DECEMBER); + test.set(Calendar.DAY_OF_MONTH, 7); + test.set(Calendar.HOUR_OF_DAY, hour); + int day = test.get(Calendar.DAY_OF_MONTH); + for (int n = 1; n <= 5; n++) { + Calendar businessDay = Calendar.getInstance(); + businessDay.setTime(getBusinessDay(test.getTime(), n)); + Assert.assertEquals(businessDay.get(Calendar.DAY_OF_MONTH), + day + n + 2); + Assert.assertEquals(businessDay.get(Calendar.HOUR_OF_DAY), + hour); + } + } + + @Test + public void testGetBusinessDayWihHoliday() { + // 12/23/2012 is Monday and 12/24 - 12/26 are holidays + int hour = 10; + Calendar test = Calendar.getInstance(); + test.set(Calendar.YEAR, 2012); + test.set(Calendar.MONTH, Calendar.DECEMBER); + test.set(Calendar.DAY_OF_MONTH, 24); + test.set(Calendar.HOUR_OF_DAY, hour); + int day = test.get(Calendar.DAY_OF_MONTH); + + Calendar businessDay = Calendar.getInstance(); + businessDay.setTime(getBusinessDay(test.getTime(), 1)); + Assert.assertEquals(businessDay.get(Calendar.DAY_OF_MONTH), + day + 4); + Assert.assertEquals(businessDay.get(Calendar.HOUR_OF_DAY), + hour); + } + + @Test + public void testGetBusinessDayWihHolidayNextYear() { + // 12/28/2012 is Friday and 12/31 - 1/1 are holidays + int hour = 10; + Calendar test = Calendar.getInstance(); + test.set(Calendar.YEAR, 2012); + test.set(Calendar.MONTH, Calendar.DECEMBER); + test.set(Calendar.DAY_OF_MONTH, 28); + test.set(Calendar.HOUR_OF_DAY, hour); + + Calendar businessDay = Calendar.getInstance(); + businessDay.setTime(getBusinessDay(test.getTime(), 1)); + // The next business day should be 1/2/2013 + Assert.assertEquals(businessDay.get(Calendar.YEAR), 2013); + Assert.assertEquals(businessDay.get(Calendar.MONTH), Calendar.JANUARY); + Assert.assertEquals(businessDay.get(Calendar.DAY_OF_MONTH), 2); + Assert.assertEquals(businessDay.get(Calendar.HOUR_OF_DAY), + hour); + } } diff --git a/src/test/java/com/netflix/simianarmy/basic/TestBasicContext.java b/src/test/java/com/netflix/simianarmy/basic/TestBasicContext.java index a0d6ea42..b31b1b39 100644 --- a/src/test/java/com/netflix/simianarmy/basic/TestBasicContext.java +++ b/src/test/java/com/netflix/simianarmy/basic/TestBasicContext.java @@ -18,23 +18,22 @@ */ package com.netflix.simianarmy.basic; -import org.testng.annotations.Test; import org.testng.Assert; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.testng.annotations.Test; public class TestBasicContext { - private static final Logger LOGGER = LoggerFactory.getLogger(TestBasicContext.class); - @Test public void testContext() { - BasicContext ctx = new BasicContext(); + BasicChaosMonkeyContext ctx = new BasicChaosMonkeyContext(); Assert.assertNotNull(ctx.scheduler()); Assert.assertNotNull(ctx.calendar()); Assert.assertNotNull(ctx.configuration()); Assert.assertNotNull(ctx.cloudClient()); Assert.assertNotNull(ctx.chaosCrawler()); Assert.assertNotNull(ctx.chaosInstanceSelector()); + + Assert.assertTrue(ctx.configuration().getBool("simianarmy.calendar.isMonkeyTime")); + // Verify that the property in chaos.properties overrides the same property in simianarmy.properties + Assert.assertFalse(ctx.configuration().getBool("simianarmy.chaos.enabled")); } } diff --git a/src/test/java/com/netflix/simianarmy/basic/TestBasicMonkeyServer.java b/src/test/java/com/netflix/simianarmy/basic/TestBasicMonkeyServer.java index 1a78b92d..41752b76 100644 --- a/src/test/java/com/netflix/simianarmy/basic/TestBasicMonkeyServer.java +++ b/src/test/java/com/netflix/simianarmy/basic/TestBasicMonkeyServer.java @@ -18,30 +18,28 @@ */ package com.netflix.simianarmy.basic; -import org.testng.annotations.Test; import org.testng.Assert; +import org.testng.annotations.Test; import com.netflix.simianarmy.MonkeyRunner; import com.netflix.simianarmy.TestMonkey; import com.netflix.simianarmy.basic.chaos.BasicChaosMonkey; import com.netflix.simianarmy.chaos.TestChaosMonkeyContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - @SuppressWarnings("serial") public class TestBasicMonkeyServer extends BasicMonkeyServer { - private static final Logger LOGGER = LoggerFactory.getLogger(TestBasicMonkeyServer.class); private static final MonkeyRunner RUNNER = MonkeyRunner.getInstance(); private static boolean monkeyRan = false; public static class SillyMonkey extends TestMonkey { + @Override public void doMonkeyBusiness() { monkeyRan = true; } } + @Override public void addMonkeysToRun() { MonkeyRunner.getInstance().replaceMonkey(BasicChaosMonkey.class, TestChaosMonkeyContext.class); MonkeyRunner.getInstance().addMonkey(SillyMonkey.class); diff --git a/src/test/java/com/netflix/simianarmy/basic/janitor/TestBasicJanitorRuleEngine.java b/src/test/java/com/netflix/simianarmy/basic/janitor/TestBasicJanitorRuleEngine.java new file mode 100644 index 00000000..1e4e5db0 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/basic/janitor/TestBasicJanitorRuleEngine.java @@ -0,0 +1,106 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.janitor; + +import java.util.Date; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.janitor.Rule; + +public class TestBasicJanitorRuleEngine { + + @Test + public void testEmptyRuleSet() { + Resource resource = new AWSResource().withId("id"); + BasicJanitorRuleEngine engine = new BasicJanitorRuleEngine(); + Assert.assertTrue(engine.isValid(resource)); + } + + @Test + public void testAllValid() { + Resource resource = new AWSResource().withId("id"); + BasicJanitorRuleEngine engine = new BasicJanitorRuleEngine() + .addRule(new AlwaysValidRule()) + .addRule(new AlwaysValidRule()) + .addRule(new AlwaysValidRule()); + Assert.assertTrue(engine.isValid(resource)); + } + + @Test + public void testMixed() { + Resource resource = new AWSResource().withId("id"); + DateTime now = DateTime.now(); + BasicJanitorRuleEngine engine = new BasicJanitorRuleEngine() + .addRule(new AlwaysValidRule()) + .addRule(new AlwaysInvalidRule(now, 1)) + .addRule(new AlwaysValidRule()); + Assert.assertFalse(engine.isValid(resource)); + } + + @Test + public void testIsValidWithNearestTerminationTime() { + int[][] permutaions = {{1, 2, 3}, {1, 3, 2}, {2, 1, 3}, {2, 3, 1}, {3, 1, 2}, {3, 2, 1}}; + + for (int[] perm : permutaions) { + Resource resource = new AWSResource().withId("id"); + DateTime now = DateTime.now(); + BasicJanitorRuleEngine engine = new BasicJanitorRuleEngine() + .addRule(new AlwaysInvalidRule(now, perm[0])) + .addRule(new AlwaysInvalidRule(now, perm[1])) + .addRule(new AlwaysInvalidRule(now, perm[2])); + Assert.assertFalse(engine.isValid(resource)); + Assert.assertEquals( + resource.getExpectedTerminationTime().getTime(), + now.plusDays(1).getMillis()); + Assert.assertEquals(resource.getTerminationReason(), "1"); + } + } + +} + +class AlwaysValidRule implements Rule { + @Override + public boolean isValid(Resource resource) { + return true; + } +} + +class AlwaysInvalidRule implements Rule { + private final int retentionDays; + private final DateTime now; + + public AlwaysInvalidRule(DateTime now, int retentionDays) { + this.retentionDays = retentionDays; + this.now = now; + } + + @Override + public boolean isValid(Resource resource) { + resource.setExpectedTerminationTime( + new Date(now.plusDays(retentionDays).getMillis())); + resource.setTerminationReason(String.valueOf(retentionDays)); + return false; + } +} diff --git a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java index a93b4407..07f2aa12 100644 --- a/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/chaos/TestChaosMonkeyContext.java @@ -191,6 +191,22 @@ public CloudClient cloudClient() { public void terminateInstance(String instanceId) { terminated.add(instanceId); } + + @Override + public void createTagsForResources(Map keyValueMap, String... resourceIds) { + } + + @Override + public void deleteAutoScalingGroup(String asgName) { + } + + @Override + public void deleteVolume(String volumeId) { + } + + @Override + public void deleteSnapshot(String snapshotId) { + } }; } diff --git a/src/test/java/com/netflix/simianarmy/client/vsphere/TestVSphereContext.java b/src/test/java/com/netflix/simianarmy/client/vsphere/TestVSphereContext.java index 98a57b4b..e49ec301 100644 --- a/src/test/java/com/netflix/simianarmy/client/vsphere/TestVSphereContext.java +++ b/src/test/java/com/netflix/simianarmy/client/vsphere/TestVSphereContext.java @@ -30,7 +30,7 @@ public class TestVSphereContext { @Test public void shouldSetClientOfCorrectType() { VSphereContext context = new VSphereContext(); - AWSClient awsClient = context.getAwsClient(); + AWSClient awsClient = context.awsClient(); assertNotNull(awsClient); assertTrue(awsClient instanceof VSphereClient); } diff --git a/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java b/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java new file mode 100644 index 00000000..612bce20 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java @@ -0,0 +1,628 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.janitor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; +import com.netflix.simianarmy.basic.BasicConfiguration; +import com.netflix.simianarmy.basic.janitor.BasicJanitorRuleEngine; + + +public class TestAbstractJanitor extends AbstractJanitor { + + private static final String TEST_REGION = "test-region"; + + public TestAbstractJanitor(AbstractJanitor.Context ctx, Enum resourceType) { + super(ctx, resourceType); + this.idToResource = new HashMap(); + for (Resource r : ((TestJanitorCrawler) (ctx.janitorCrawler())).getCrawledResources()) { + this.idToResource.put(r.getId(), r); + } + } + + // The collection of all resources for testing. + private final Map idToResource; + + private final HashSet markedResourceIds = new HashSet(); + private final HashSet cleanedResourceIds = new HashSet(); + + @Override + protected void postMark(Resource resource) { + markedResourceIds.add(resource.getId()); + } + + @Override + protected void cleanup(Resource resource) { + if (!idToResource.containsKey(resource.getId())) { + throw new RuntimeException(); + } + // add a special case to throw exception + if (resource.getId().equals("11")) { + throw new RuntimeException("Magic number of id."); + } + idToResource.remove(resource.getId()); + } + + @Override + protected void postCleanup(Resource resource) { + cleanedResourceIds.add(resource.getId()); + } + + private static List generateTestingResources(int n) { + List resources = new ArrayList(n); + for (int i = 1; i <= n; i++) { + resources.add(new AWSResource().withId(String.valueOf(i)) + .withRegion(TEST_REGION) + .withResourceType(TestResourceType.TEST_RESOURCE_TYPE) + .withOptOutOfJanitor(false)); + } + return resources; + } + + @Test + public static void testJanitor() { + Collection crawledResources = new ArrayList(); + int n = 10; + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker( + new HashMap()); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n); + Assert.assertEquals(janitor.markedResourceIds.size(), 0); + janitor.markResources(); + Assert.assertEquals(janitor.getMarkedResources().size(), n / 2); + Assert.assertEquals(janitor.markedResourceIds.size(), n / 2); + for (int i = 1; i <= n; i += 2) { + Assert.assertTrue(janitor.markedResourceIds.contains(String.valueOf(i))); + } + + Assert.assertEquals(janitor.cleanedResourceIds.size(), 0); + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), n / 2); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(), + n / 2); + Assert.assertEquals(janitor.cleanedResourceIds.size(), n / 2); + for (int i = 1; i <= n; i += 2) { + Assert.assertTrue(janitor.cleanedResourceIds.contains(String.valueOf(i))); + } + } + + @Test + public static void testJanitorWithOptedOutResources() { + Collection crawledResources = new ArrayList(); + int n = 10; + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + // set some resources in the tracker as opted out + Date now = new Date(DateTime.now().minusDays(1).getMillis()); + Map trackedResources = new HashMap(); + for (Resource r : generateTestingResources(n)) { + int id = Integer.parseInt(r.getId()); + if (id % 4 == 1 || id % 4 == 2) { + r.setOptOutOfJanitor(true); + r.setState(CleanupState.MARKED); + r.setExpectedTerminationTime(now); + r.setMarkTime(now); + } + trackedResources.put(r.getId(), r); + } + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker( + trackedResources); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + 10); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + 6); // 1, 2, 5, 6, 9, 10 are marked + Assert.assertEquals(janitor.markedResourceIds.size(), 0); + janitor.markResources(); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + 5); // 1, 3, 5, 7, 9 are marked + Assert.assertEquals(janitor.getMarkedResources().size(), 2); // 3, 7 are newly marked. + Assert.assertEquals(janitor.markedResourceIds.size(), 2); + Assert.assertEquals(janitor.cleanedResourceIds.size(), 0); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + 5); // 1, 3, 5, 7, 9 are marked + Assert.assertEquals(janitor.getUnmarkedResources().size(), 3); // 2, 6, 10 got unmarked + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.UNMARKED, TEST_REGION).size(), + 3); + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), 2); // 3, 7 are cleaned + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(), + 2); + } + + @Test + public static void testJanitorWithCleanupFailure() { + Collection crawledResources = new ArrayList(); + int n = 20; + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + new TestJanitorResourceTracker(new HashMap()), + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n); + janitor.markResources(); + Assert.assertEquals(janitor.getMarkedResources().size(), n / 2); + + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - 1); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1); + } + + @Test + public static void testJanitorWithUnmarking() { + Collection crawledResources = new ArrayList(); + Map trackedResources = new HashMap(); + int n = 10; + DateTime now = DateTime.now(); + Date markTime = new Date(now.minusDays(5).getMillis()); + Date notifyTime = new Date(now.minusDays(4).getMillis()); + Date terminationTime = new Date(now.minusDays(1).getMillis()); + for (Resource r : generateTestingResources(n)) { + if (Integer.parseInt(r.getId()) % 3 == 0) { + trackedResources.put(r.getId(), r); + r.setState(CleanupState.MARKED); + r.setMarkTime(markTime); + r.setExpectedTerminationTime(terminationTime); + r.setNotificationTime(notifyTime); + } + } + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker(trackedResources); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + n / 3); + janitor.markResources(); + // (n/3-n/6) resources were already marked, so in the last run the marked resources + // should be n/2 - n/3 + n/6. + Assert.assertEquals(janitor.getMarkedResources().size(), n / 2 - n / 3 + n / 6); + Assert.assertEquals(janitor.getUnmarkedResources().size(), n / 6); + + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), n / 2); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + } + + + @Test + public static void testJanitorWithFutureTerminationTime() { + Collection crawledResources = new ArrayList(); + Map trackedResources = new HashMap(); + int n = 10; + DateTime now = DateTime.now(); + Date markTime = new Date(now.minusDays(5).getMillis()); + Date notifyTime = new Date(now.minusDays(4).getMillis()); + Date terminationTime = new Date(now.plusDays(10).getMillis()); + for (Resource r : generateTestingResources(n)) { + trackedResources.put(r.getId(), r); + r.setState(CleanupState.MARKED); + r.setNotificationTime(notifyTime); + r.setMarkTime(markTime); + r.setExpectedTerminationTime(terminationTime); + } + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker(trackedResources); + + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + n); + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), 0); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + } + + + @Test + public static void testJanitorWithoutNotification() { + Collection crawledResources = new ArrayList(); + Map trackedResources = new HashMap(); + int n = 10; + for (Resource r : generateTestingResources(n)) { + trackedResources.put(r.getId(), r); + r.setState(CleanupState.MARKED); + // The marking/cleanup is not notified so we the Janitor won't clean it up. + // r.setNotificationTime(new Date()); + r.setMarkTime(new Date()); + r.setExpectedTerminationTime(new Date(DateTime.now().plusDays(10).getMillis())); + } + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker(trackedResources); + + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + n); + + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), 0); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + } + + @Test + public static void testLeashedJanitorForMarking() { + Collection crawledResources = new ArrayList(); + int n = 10; + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker( + new HashMap()); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(true); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n); + janitor.markResources(); + Assert.assertEquals(janitor.getMarkedResources().size(), n / 2); + + // No resource is really changed in tracker + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + 0); + } + + + @Test + public static void testJanitorWithoutHoldingOffCleanup() { + Collection crawledResources = new ArrayList(); + int n = 10; + for (Resource r : generateTestingResources(n)) { + crawledResources.add(r); + } + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker(new HashMap()); + DateTime now = DateTime.now(); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new ImmediateCleanupRule(now)), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n); + Assert.assertEquals(janitor.markedResourceIds.size(), 0); + janitor.markResources(); + Assert.assertEquals(janitor.getMarkedResources().size(), n); + Assert.assertEquals(janitor.markedResourceIds.size(), n); + for (int i = 1; i <= n; i++) { + Assert.assertTrue(janitor.markedResourceIds.contains(String.valueOf(i))); + } + + Assert.assertEquals(janitor.cleanedResourceIds.size(), 0); + janitor.cleanupResources(); + // No resource is cleaned since the notification is later than expected termination time. + Assert.assertEquals(janitor.getCleanedResources().size(), n); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(), + n); + Assert.assertEquals(janitor.cleanedResourceIds.size(), n); + } + + @Test + public static void testJanitorWithUnmarkingUserTerminated() { + Collection crawledResources = new ArrayList(); + Map trackedResources = new HashMap(); + int n = 10; + DateTime now = DateTime.now(); + Date markTime = new Date(now.minusDays(5).getMillis()); + Date notifyTime = new Date(now.minusDays(4).getMillis()); + Date terminationTime = new Date(now.minusDays(1).getMillis()); + for (Resource r : generateTestingResources(n)) { + if (Integer.parseInt(r.getId()) % 3 != 0) { + crawledResources.add(r); + } else { + trackedResources.put(r.getId(), r); + r.setState(CleanupState.MARKED); + r.setMarkTime(markTime); + r.setNotificationTime(notifyTime); + r.setExpectedTerminationTime(terminationTime); + } + } + + TestJanitorCrawler crawler = new TestJanitorCrawler(crawledResources); + TestJanitorResourceTracker resourceTracker = new TestJanitorResourceTracker(trackedResources); + TestAbstractJanitor janitor = new TestAbstractJanitor( + new TestJanitorContext(TEST_REGION, + new BasicJanitorRuleEngine().addRule(new IsEvenRule()), + crawler, + resourceTracker, + new TestMonkeyCalendar()), TestResourceType.TEST_RESOURCE_TYPE); + janitor.setLeashed(false); + Assert.assertEquals( + crawler.resources(TestResourceType.TEST_RESOURCE_TYPE).size(), + n - n / 3); + Assert.assertEquals(resourceTracker.getResources( + TestResourceType.TEST_RESOURCE_TYPE, CleanupState.MARKED, TEST_REGION).size(), + n / 3); + janitor.markResources(); + // n/3 resources should be considered user terminated + Assert.assertEquals(janitor.getMarkedResources().size(), n / 2 - n / 3 + n / 6); + Assert.assertEquals(janitor.getUnmarkedResources().size(), n / 3); + + janitor.cleanupResources(); + Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - n / 3 + n / 6); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + } +} + +class TestJanitorCrawler implements JanitorCrawler { + private final Collection crawledResources; + public Collection getCrawledResources() { + return crawledResources; + } + + public TestJanitorCrawler(Collection crawledResources) { + this.crawledResources = crawledResources; + } + + @Override + public EnumSet resourceTypes() { + return EnumSet.of(TestResourceType.TEST_RESOURCE_TYPE); + } + + @Override + public List resources(Enum resourceType) { + return new ArrayList(crawledResources); + } + + @Override + public List resources(String... resourceIds) { + List result = new ArrayList(resourceIds.length); + Set idSet = new HashSet(Arrays.asList(resourceIds)); + for (Resource r : crawledResources) { + if (idSet.contains(r.getId())) { + result.add(r); + } + } + return result; + } + + @Override + public String getOwnerEmailForResource(Resource resource) { + return null; + } +} + +enum TestResourceType { + TEST_RESOURCE_TYPE +} + +class TestJanitorResourceTracker implements JanitorResourceTracker { + private final Map resources; + public TestJanitorResourceTracker(Map trackedResources) { + this.resources = trackedResources; + } + + @Override + public void addOrUpdate(Resource resource) { + resources.put(resource.getId(), resource); + } + + @Override + public List getResources(Enum resourceType, CleanupState state, String region) { + List result = new ArrayList(); + for (Resource r : resources.values()) { + if (r.getResourceType().equals(resourceType) + && (r.getState() != null && r.getState().equals(state)) + && r.getRegion().equals(region)) { + result.add(r.cloneResource()); + } + } + return result; + } + + @Override + public Resource getResource(String resourceId) { + return resources.get(resourceId); + } +} + +/** + * The rule considers all resources with an odd number as the id as cleanup candidate. + */ +class IsEvenRule implements Rule { + @Override + public boolean isValid(Resource resource) { + // returns true if the resource's id is an even integer + int id; + try { + id = Integer.parseInt(resource.getId()); + } catch (Exception e) { + return true; + } + DateTime now = DateTime.now(); + resource.setExpectedTerminationTime(new Date(now.minusDays(1).getMillis())); + // Set the resource as notified so it can be cleaned + // set the notification time at more than 1 day before the termination time + resource.setNotificationTime(new Date(now.minusDays(4).getMillis())); + return id % 2 == 0; + } +} + +/** + * The rule considers all resources as cleanup candidate and sets notification time + * after the termination time. + */ +class ImmediateCleanupRule implements Rule { + private final DateTime now; + public ImmediateCleanupRule(DateTime now) { + this.now = now; + } + @Override + public boolean isValid(Resource resource) { + resource.setExpectedTerminationTime(new Date(now.minusMinutes(10).getMillis())); + resource.setNotificationTime(new Date(now.getMillis())); + return false; + } +} + +class TestJanitorContext implements AbstractJanitor.Context { + private final String region; + private final JanitorRuleEngine ruleEngine; + private final JanitorCrawler crawler; + private final JanitorResourceTracker resourceTracker; + private final MonkeyCalendar calendar; + + public TestJanitorContext(String region, JanitorRuleEngine ruleEngine, JanitorCrawler crawler, + JanitorResourceTracker resourceTracker, MonkeyCalendar calendar) { + this.region = region; + this.resourceTracker = resourceTracker; + this.ruleEngine = ruleEngine; + this.crawler = crawler; + this.calendar = calendar; + } + + @Override + public String region() { + return region; + } + + @Override + public MonkeyCalendar calendar() { + return calendar; + } + + @Override + public JanitorRuleEngine janitorRuleEngine() { + return ruleEngine; + } + + @Override + public JanitorCrawler janitorCrawler() { + return crawler; + } + + @Override + public JanitorResourceTracker janitorResourceTracker() { + return resourceTracker; + } + + @Override + public MonkeyConfiguration configuration() { + return new BasicConfiguration(new Properties()); + } + + @Override + public MonkeyRecorder recorder() { + // No events to be recorded + return null; + } +} diff --git a/src/test/java/com/netflix/simianarmy/resources/chaos/TestChaosMonkeyResource.java b/src/test/java/com/netflix/simianarmy/resources/chaos/TestChaosMonkeyResource.java index 98e6d51b..202221a5 100644 --- a/src/test/java/com/netflix/simianarmy/resources/chaos/TestChaosMonkeyResource.java +++ b/src/test/java/com/netflix/simianarmy/resources/chaos/TestChaosMonkeyResource.java @@ -1,5 +1,5 @@ // CHECKSTYLE IGNORE Javadoc -// CHECKSTYLE IGNORE Javadoc +//CHECKSTYLE IGNORE MagicNumber /* * * Copyright 2012 Netflix, Inc. @@ -51,7 +51,6 @@ import com.netflix.simianarmy.chaos.TestChaosMonkeyContext; import com.sun.jersey.core.util.MultivaluedMapImpl; -//CHECKSTYLE IGNORE MagicNumber public class TestChaosMonkeyResource { private static final Logger LOGGER = LoggerFactory.getLogger(TestChaosMonkeyResource.class); diff --git a/src/test/resources/chaos.properties b/src/test/resources/chaos.properties new file mode 100644 index 00000000..eb7c1da7 --- /dev/null +++ b/src/test/resources/chaos.properties @@ -0,0 +1 @@ +simianarmy.chaos.enabled = false diff --git a/src/test/resources/simianarmy.properties b/src/test/resources/simianarmy.properties index 8a69a802..1ca72cc8 100644 --- a/src/test/resources/simianarmy.properties +++ b/src/test/resources/simianarmy.properties @@ -1 +1,2 @@ -simianarmy.chaos.enabled = false \ No newline at end of file +simianarmy.chaos.enabled = true +simianarmy.calendar.isMonkeyTime = true From aa702542b940f92ef14368538b03e7934657fcf2 Mon Sep 17 00:00:00 2001 From: michaelf Date: Wed, 2 Jan 2013 20:09:20 -0800 Subject: [PATCH 2/2] fix typos in comments --- src/main/java/com/netflix/simianarmy/MonkeyRunner.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/com/netflix/simianarmy/MonkeyRunner.java b/src/main/java/com/netflix/simianarmy/MonkeyRunner.java index bb79876a..548c386c 100644 --- a/src/main/java/com/netflix/simianarmy/MonkeyRunner.java +++ b/src/main/java/com/netflix/simianarmy/MonkeyRunner.java @@ -171,20 +171,11 @@ public void removeMonkey(Class monkeyClass) { * Example: * * 
-<<<<<<< HEAD
      *         {@code
      *         MonkeyRunner.getInstance().addMonkey(BasicChaosMonkey.class, BasicMonkeyContext.class);
      *         // This will actually return a BasicChaosMonkey since that is the only subclass that was registered
      *         ChaosMonkey monkey = MonkeyRunner.getInstance().factory(ChaosMonkey.class);
      *}
-=======
-     *          {@code
-     *          MonkeyRunner.getInstance().addMonkey(BasicChaosMonkey.class, BasicMonkeyContext.class);
-     *
-     *          // This will actualy return a BasicChaosMonkey since that is the only subclass that was registered
-     *          ChaosMonkey monkey = MonkeyRunner.getInstance().factory(ChaosMonkey.class);
-     * }
->>>>>>> fde87ebfbda161188f1dac96e5ea8e34bcc684f6
      *
* * @param