diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/generic/UntaggedRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/generic/UntaggedRule.java new file mode 100644 index 00000000..488ef4d4 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/generic/UntaggedRule.java @@ -0,0 +1,98 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.generic; + +import java.util.Date; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.janitor.Rule; + +/** + * The rule for checking the orphaned instances that do not belong to any ASGs and + * launched for certain days. + */ +public class UntaggedRule implements Rule { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(UntaggedRule.class); + + private static final String TERMINATION_REASON = "This resource is missing the required tags"; + + private final MonkeyCalendar calendar; + + private final Set tagNames; + + private final int retentionDaysWithOwner; + + private final int retentionDaysWithoutOwner; + + + /** + * Constructor for UntaggedInstanceRule. + * + * @param calendar + * The calendar used to calculate the termination time + * @param tagNames + * Set of tags that needs to be set + */ + public UntaggedRule(MonkeyCalendar calendar, Set tagNames, int retentionDaysWithOwner, int retentionDaysWithoutOwner) { + Validate.notNull(calendar); + Validate.notNull(tagNames); + this.calendar = calendar; + this.tagNames = tagNames; + this.retentionDaysWithOwner = retentionDaysWithOwner; + this.retentionDaysWithoutOwner = retentionDaysWithoutOwner; + } + + @Override + public boolean isValid(Resource resource) { + Validate.notNull(resource); + for (String tagName : this.tagNames) { + if (((AWSResource) resource).getTag(tagName) == null) { + String terminationReason = String.format("does not have the required tag %s", resource.getId(), + tagName); + LOGGER.error(String.format("The resource %s %s", resource.getId(), terminationReason)); + DateTime now = new DateTime(calendar.now().getTimeInMillis()); + if (resource.getExpectedTerminationTime() == null) { + int retentionDays = retentionDaysWithoutOwner; + if (resource.getOwnerEmail() != null) { + retentionDays = retentionDaysWithOwner; + } + Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays); + resource.setExpectedTerminationTime(terminationTime); + resource.setTerminationReason(terminationReason); + } + return false; + } else { + LOGGER.debug(String.format("The resource %s has the required tag %s", resource.getId(), tagName)); + } + } + LOGGER.info(String.format("The resource %s has all required tags", resource.getId())); + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java index f0860a7c..62f069bb 100644 --- a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java @@ -46,6 +46,7 @@ import com.netflix.simianarmy.aws.janitor.rule.asg.DummyASGInstanceValidator; import com.netflix.simianarmy.aws.janitor.rule.asg.OldEmptyASGRule; import com.netflix.simianarmy.aws.janitor.rule.asg.SuspendedASGRule; +import com.netflix.simianarmy.aws.janitor.rule.generic.UntaggedRule; import com.netflix.simianarmy.aws.janitor.rule.instance.OrphanedInstanceRule; import com.netflix.simianarmy.aws.janitor.rule.launchconfig.OldUnusedLaunchConfigRule; import com.netflix.simianarmy.aws.janitor.rule.snapshot.NoGeneratedAMIRule; @@ -186,6 +187,15 @@ private ASGJanitor getASGJanitor() { instanceValidator )); } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler crawler; if (configuration().getBoolOrElse("simianarmy.janitor.edda.enabled", false)) { crawler = new EddaASGJanitorCrawler(createEddaClient(), awsClient().region()); @@ -210,6 +220,15 @@ private InstanceJanitor getInstanceJanitor() { "simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithoutOwner", 8))); } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler instanceCrawler; if (configuration().getBoolOrElse("simianarmy.janitor.edda.enabled", false)) { instanceCrawler = new EddaInstanceJanitorCrawler(createEddaClient(), awsClient().region()); @@ -237,6 +256,15 @@ && configuration().getBoolOrElse("simianarmy.janitor.rule.deleteOnTerminationRul "simianarmy.janitor.rule.deleteOnTerminationRule.retentionDays", 3))); } } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler volumeCrawler; if (configuration().getBoolOrElse("simianarmy.janitor.edda.enabled", false)) { volumeCrawler = new EddaEBSVolumeJanitorCrawler(createEddaClient(), awsClient().region()); @@ -258,6 +286,15 @@ private EBSSnapshotJanitor getEBSSnapshotJanitor() { (int) configuration().getNumOrElse( "simianarmy.janitor.rule.noGeneratedAMIRule.retentionDays", 7))); } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler snapshotCrawler; if (configuration().getBoolOrElse("simianarmy.janitor.edda.enabled", false)) { snapshotCrawler = new EddaEBSSnapshotJanitorCrawler( @@ -281,6 +318,15 @@ private LaunchConfigJanitor getLaunchConfigJanitor() { (int) configuration().getNumOrElse( "simianarmy.janitor.rule.oldUnusedLaunchConfigRule.retentionDays", 3))); } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } + JanitorCrawler crawler; if (configuration().getBoolOrElse("simianarmy.janitor.edda.enabled", false)) { crawler = new EddaLaunchConfigJanitorCrawler( @@ -313,6 +359,14 @@ private ImageJanitor getImageJanitor() { (int) configuration().getNumOrElse( "simianarmy.janitor.rule.unusedImageRule.lastReferenceDaysThreshold", 45))); } + if (configuration().getBoolOrElse("simianarmy.janitor.rule.untaggedRule.enabled", false)) { + ruleEngine.addRule(new UntaggedRule(monkeyCalendar, getPropertySet("simianarmy.janitor.rule.untaggedRule.requiredTags"), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner", 3), + (int) configuration().getNumOrElse( + "simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner", + 8))); + } BasicJanitorContext janitorCtx = new BasicJanitorContext( monkeyRegion, ruleEngine, crawler, janitorResourceTracker, @@ -338,6 +392,17 @@ private Set getEnabledResourceSet() { return enabledResourceSet; } + private Set getPropertySet(String property) { + Set propertyValueSet = new HashSet(); + String propertyValue = configuration().getStr(property); + if (StringUtils.isNotBlank(propertyValue)) { + for (String propertyValueItem : propertyValue.split(",")) { + propertyValueSet.add(propertyValueItem.trim()); + } + } + return propertyValueSet; + } + public JanitorEmailNotifier.Context getJanitorEmailNotifierContext() { return new JanitorEmailNotifier.Context() { @Override diff --git a/src/main/resources/janitor.properties b/src/main/resources/janitor.properties index 3c5a08b2..7740119c 100644 --- a/src/main/resources/janitor.properties +++ b/src/main/resources/janitor.properties @@ -60,6 +60,18 @@ simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithOwner = 3 # when the instance has no owner. simianarmy.janitor.rule.orphanedInstanceRule.retentionDaysWithoutOwner = 8 +# The following properties are used by the Janitor rule for cleaning up untagged resources, +# i.e. instances that are missing any required tags +simianarmy.janitor.rule.untaggedRule.enabled = true +# List of tags that are required for each resource +simianarmy.janitor.rule.untaggedRule.requiredTags = owner, purpose, project +# The number of business days the resource is kept after a notification is sent for the deletion +# when the resource has an owner. +simianarmy.janitor.rule.untaggedRule.retentionDaysWithOwner = 2 +# The number of business days the resource is kept after a notification is sent for the deletion +# when the resource has no owner. +simianarmy.janitor.rule.untaggedRule.retentionDaysWithoutOwner = 2 + # The following properties are used by the Janitor rule for cleaning up volumes that have been # detached from instances for certain days. simianarmy.janitor.rule.oldDetachedVolumeRule.enabled = true diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/rule/generic/TestUntaggedRule.java b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/generic/TestUntaggedRule.java new file mode 100644 index 00000000..938512bc --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/rule/generic/TestUntaggedRule.java @@ -0,0 +1,131 @@ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.aws.janitor.rule.generic; + +import java.util.Date; +import java.util.HashSet; +import java.util.Set; + +import org.joda.time.DateTime; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.TestUtils; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import com.netflix.simianarmy.aws.janitor.crawler.InstanceJanitorCrawler; +import com.netflix.simianarmy.aws.janitor.rule.TestMonkeyCalendar; + +public class TestUntaggedRule { + + @Test + public void testUntaggedInstanceWithOwner() { + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withOwnerEmail("owner@foo.com"); + resource.setTag("tag1", "value1"); + ((AWSResource) resource).setAWSResourceState("running"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag2"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + UntaggedRule rule = new UntaggedRule(new TestMonkeyCalendar(), tags, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + TestUtils.verifyTerminationTimeRough(resource, retentionDaysWithOwner, now); + } + + @Test + public void testUntaggedInstanceWithoutOwner() { + DateTime now = DateTime.now(); + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE); + resource.setTag("tag1", "value1"); + ((AWSResource) resource).setAWSResourceState("running"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag2"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + UntaggedRule rule = new UntaggedRule(new TestMonkeyCalendar(), tags, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + TestUtils.verifyTerminationTimeRough(resource, retentionDaysWithoutOwner, now); + } + + @Test + public void testTaggedInstance() { + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE); + resource.setTag("tag1", "value1"); + resource.setTag("tag2", "value2"); + ((AWSResource) resource).setAWSResourceState("running"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag2"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + UntaggedRule rule = new UntaggedRule(new TestMonkeyCalendar(), tags, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertTrue(rule.isValid(resource)); + } + + @Test + public void testUntaggedResource() { + DateTime now = DateTime.now(); + Resource imageResource = new AWSResource().withId("ami-123123").withResourceType(AWSResourceType.IMAGE); + Resource asgResource = new AWSResource().withId("my-cool-asg").withResourceType(AWSResourceType.ASG); + Resource ebsSnapshotResource = new AWSResource().withId("snap-123123").withResourceType(AWSResourceType.EBS_SNAPSHOT); + Resource lauchConfigurationResource = new AWSResource().withId("my-cool-launch-configuration").withResourceType(AWSResourceType.LAUNCH_CONFIG); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag2"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + UntaggedRule rule = new UntaggedRule(new TestMonkeyCalendar(), tags, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(imageResource)); + Assert.assertFalse(rule.isValid(asgResource)); + Assert.assertFalse(rule.isValid(ebsSnapshotResource)); + Assert.assertFalse(rule.isValid(lauchConfigurationResource)); + TestUtils.verifyTerminationTimeRough(imageResource, retentionDaysWithoutOwner, now); + TestUtils.verifyTerminationTimeRough(asgResource, retentionDaysWithoutOwner, now); + TestUtils.verifyTerminationTimeRough(ebsSnapshotResource, retentionDaysWithoutOwner, now); + TestUtils.verifyTerminationTimeRough(lauchConfigurationResource, retentionDaysWithoutOwner, now); + } + + @Test + public void testResourceWithExpectedTerminationTimeSet() { + DateTime now = DateTime.now(); + Date oldTermDate = new Date(now.plusDays(10).getMillis()); + String oldTermReason = "Foo"; + Resource resource = new AWSResource().withId("i-12345678").withResourceType(AWSResourceType.INSTANCE) + .withExpectedTerminationTime(oldTermDate) + .withTerminationReason(oldTermReason); + ((AWSResource) resource).setAWSResourceState("running"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag2"); + int retentionDaysWithOwner = 4; + int retentionDaysWithoutOwner = 8; + UntaggedRule rule = new UntaggedRule(new TestMonkeyCalendar(), tags, retentionDaysWithOwner, retentionDaysWithoutOwner); + Assert.assertFalse(rule.isValid(resource)); + Assert.assertEquals(oldTermDate, resource.getExpectedTerminationTime()); + Assert.assertEquals(oldTermReason, resource.getTerminationReason()); + } + +}