diff --git a/build.gradle b/build.gradle index 8fcca5ca..cd5b00a6 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ dependencies { compile 'org.apache.jclouds.api:ec2:1.9.0' compile 'org.apache.jclouds.provider:aws-ec2:1.9.0' compile 'ch.qos.logback:logback-classic:1.0.13' - + compile 'com.netflix.servo:servo-core:0.9.4' testCompile 'org.testng:testng:6.3.1' testCompile 'org.mockito:mockito-core:1.8.5' diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java index 36e0c20b..44880c56 100644 --- a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java @@ -19,11 +19,15 @@ import java.util.Collection; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.netflix.servo.annotations.DataSourceType; +import com.netflix.servo.annotations.Monitor; +import com.netflix.servo.monitor.Monitors; import com.netflix.simianarmy.MonkeyCalendar; import com.netflix.simianarmy.MonkeyConfiguration; import com.netflix.simianarmy.MonkeyRecorder; @@ -60,6 +64,9 @@ public class BasicJanitorMonkey extends JanitorMonkey { private final MonkeyRecorder recorder; private final MonkeyCalendar calendar; + + /** Keep track of the number of monkey runs */ + protected final AtomicLong monkeyRuns = new AtomicLong(0); /** * Instantiates a new basic janitor monkey. @@ -78,6 +85,9 @@ public BasicJanitorMonkey(Context ctx) { resourceTracker = ctx.resourceTracker(); recorder = ctx.recorder(); calendar = ctx.calendar(); + + // register this janitor with servo + Monitors.registerObject("simianarmy.janitor", this); } /** {@inheritDoc} */ @@ -90,6 +100,7 @@ public void doMonkeyBusiness() { return; } else { LOGGER.info(String.format("Marking resources with %d janitors.", janitors.size())); + monkeyRuns.incrementAndGet(); for (AbstractJanitor janitor : janitors) { LOGGER.info(String.format("Running janitor for region %s", janitor.getRegion())); janitor.markResources(); @@ -221,4 +232,9 @@ private boolean isJanitorMonkeyEnabled() { LOGGER.info("JanitorMonkey disabled, set {}=true", prop); return false; } + + @Monitor(name="runs", type=DataSourceType.COUNTER) + public AtomicLong getMonkeyRuns() { + return monkeyRuns; + } } diff --git a/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java index 2def11e1..454213f7 100644 --- a/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java +++ b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java @@ -19,6 +19,9 @@ package com.netflix.simianarmy.janitor; import com.google.common.collect.Maps; +import com.netflix.servo.annotations.DataSourceType; +import com.netflix.servo.annotations.Monitor; +import com.netflix.servo.annotations.MonitorTags; import com.netflix.simianarmy.MonkeyCalendar; import com.netflix.simianarmy.MonkeyConfiguration; import com.netflix.simianarmy.MonkeyRecorder; @@ -41,6 +44,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import com.netflix.servo.monitor.Monitors; +import com.netflix.servo.tag.BasicTagList; +import com.netflix.servo.tag.TagList; /** * An abstract implementation of Janitor. It marks resources that the rule engine considers @@ -55,12 +63,28 @@ public abstract class AbstractJanitor implements Janitor { /** The Constant LOGGER. */ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJanitor.class); + /** Keep track of the number of cleaned resources */ + protected final AtomicLong cleanedResourcesCount = new AtomicLong(0); + + /** Keep track of the number of marked resources */ + protected final AtomicLong markedResourcesCount = new AtomicLong(0); + + /** Keep track of the number of failed to clean resources */ + protected final AtomicLong failedToCleanResourcesCount = new AtomicLong(0); + + /** Keep track of the number of unmarked resources */ + protected final AtomicLong unmarkedResourcesCount = new AtomicLong(0); + + /** Tags to attach to servo metrics */ + @MonitorTags + protected TagList tags; + private final String region; /** The region the janitor is running in. */ public String getRegion() { return region; } - + /** * The rule engine used to decide if a resource should be a cleanup * candidate. @@ -183,6 +207,13 @@ public AbstractJanitor(Context ctx, ResourceType resourceType) { Validate.notNull(resourceType); // recorder could be null and no events are recorded when it is. this.recorder = ctx.recorder(); + + // setup servo tags, currently just tag each published metric with the region + this.tags = BasicTagList.of("simianarmy.janitor.region", ctx.region()); + + // register this janitor with servo + String monitorObjName = String.format("simianarmy.janitor.%s.%s", this.resourceType.name(), this.region); + Monitors.registerObject(monitorObjName, this); } @Override @@ -222,6 +253,7 @@ public void markResources() { recorder.recordEvent(evt); } resourceTracker.addOrUpdate(resource); + markedResourcesCount.incrementAndGet(); postMark(resource); } else { LOGGER.info(String.format( @@ -247,6 +279,7 @@ public void markResources() { resource.getId())); } unmarkedResources.add(resource); + unmarkedResourcesCount.incrementAndGet(); } } @@ -291,6 +324,7 @@ public void cleanupResources() { recorder.recordEvent(evt); } cleanup(markedResource); + cleanedResourcesCount.incrementAndGet(); markedResource.setActualTerminationTime(now); markedResource.setState(Resource.CleanupState.JANITOR_TERMINATED); resourceTracker.addOrUpdate(markedResource); @@ -298,6 +332,7 @@ public void cleanupResources() { LOGGER.error(String.format("Failed to clean up the resource %s of type %s.", markedResource.getId(), markedResource.getResourceType().name()), e); failedToCleanResources.add(markedResource); + failedToCleanResourcesCount.incrementAndGet(); continue; } postCleanup(markedResource); @@ -392,7 +427,28 @@ private void unmarkUserTerminatedResources( markedResource.getId())); } unmarkedResources.add(markedResource); + unmarkedResourcesCount.incrementAndGet(); } } } + + @Monitor(name="cleanedResourcesCount", type=DataSourceType.COUNTER) + long getResourcesCleanedCount() { + return cleanedResourcesCount.get(); + } + + @Monitor(name="markedResourcesCount", type=DataSourceType.COUNTER) + long getMarkedResourcesCount() { + return markedResourcesCount.get(); + } + + @Monitor(name="failedToCleanResourcesCount", type=DataSourceType.COUNTER) + long getFailedToCleanResourcesCount() { + return failedToCleanResourcesCount.get(); + } + + @Monitor(name="unmarkedResourcesCount", type=DataSourceType.COUNTER) + long getUnmarkedResourcesCount() { + return unmarkedResourcesCount.get(); + } } diff --git a/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java b/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java index 9446a1a1..010d5f5b 100644 --- a/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java +++ b/src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java @@ -137,7 +137,10 @@ public static void testJanitor() { Assert.assertEquals(janitor.cleanedResourceIds.size(), n / 2); for (int i = 1; i <= n; i += 2) { Assert.assertTrue(janitor.cleanedResourceIds.contains(String.valueOf(i))); - } + } + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); } @Test @@ -196,7 +199,11 @@ public static void testJanitorWithOptedOutResources() { Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); Assert.assertEquals(resourceTracker.getResources( TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(), - 2); + 2); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); + Assert.assertEquals(janitor.getUnmarkedResourcesCount(), 3); } @Test @@ -222,7 +229,10 @@ public static void testJanitorWithCleanupFailure() { janitor.cleanupResources(); Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - 1); - Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1); + Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 1); } @Test @@ -271,6 +281,10 @@ public static void testJanitorWithUnmarking() { janitor.cleanupResources(); Assert.assertEquals(janitor.getCleanedResources().size(), n / 2); Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); + Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n/6); } @@ -310,6 +324,9 @@ public static void testJanitorWithFutureTerminationTime() { janitor.cleanupResources(); Assert.assertEquals(janitor.getCleanedResources().size(), 0); Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); } @@ -346,6 +363,9 @@ public static void testJanitorWithoutNotification() { janitor.cleanupResources(); Assert.assertEquals(janitor.getCleanedResources().size(), 0); Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); } @Test @@ -370,6 +390,8 @@ public static void testLeashedJanitorForMarking() { n); janitor.markResources(); Assert.assertEquals(janitor.getMarkedResources().size(), n / 2); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); // No resource is really changed in tracker Assert.assertEquals(resourceTracker.getResources( @@ -415,6 +437,9 @@ public static void testJanitorWithoutHoldingOffCleanup() { TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(), n); Assert.assertEquals(janitor.cleanedResourceIds.size(), n); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); } @Test @@ -461,6 +486,10 @@ public static void testJanitorWithUnmarkingUserTerminated() { janitor.cleanupResources(); Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - n / 3 + n / 6); Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0); + Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size()); + Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size()); + Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0); + Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n / 3); } }