这是indexloc提供的服务，不要输入任何密码
Skip to content
This repository was archived by the owner on Mar 4, 2021. It is now read-only.

Add Servo dependency. Add JMX/Servo metrics monitoring for monkey ru… #215

Merged
merged 1 commit into from
Nov 3, 2015
Merged

Add Servo dependency. Add JMX/Servo metrics monitoring for monkey ru… #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ dependencies {
compile 'org.apache.jclouds.api:ec2:1.9.0'
compile 'org.apache.jclouds.provider:aws-ec2:1.9.0'
compile 'ch.qos.logback:logback-classic:1.0.13'

compile 'com.netflix.servo:servo-core:0.9.4'

testCompile 'org.testng:testng:6.3.1'
testCompile 'org.mockito:mockito-core:1.8.5'
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@

import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.servo.annotations.DataSourceType;
import com.netflix.servo.annotations.Monitor;
import com.netflix.servo.monitor.Monitors;
import com.netflix.simianarmy.MonkeyCalendar;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder;
Expand Down Expand Up @@ -60,6 +64,9 @@ public class BasicJanitorMonkey extends JanitorMonkey {
private final MonkeyRecorder recorder;

private final MonkeyCalendar calendar;

/** Keep track of the number of monkey runs */
protected final AtomicLong monkeyRuns = new AtomicLong(0);

/**
* Instantiates a new basic janitor monkey.
Expand All @@ -78,6 +85,9 @@ public BasicJanitorMonkey(Context ctx) {
resourceTracker = ctx.resourceTracker();
recorder = ctx.recorder();
calendar = ctx.calendar();

// register this janitor with servo
Monitors.registerObject("simianarmy.janitor", this);
}

/** {@inheritDoc} */
Expand All @@ -90,6 +100,7 @@ public void doMonkeyBusiness() {
return;
} else {
LOGGER.info(String.format("Marking resources with %d janitors.", janitors.size()));
monkeyRuns.incrementAndGet();
for (AbstractJanitor janitor : janitors) {
LOGGER.info(String.format("Running janitor for region %s", janitor.getRegion()));
janitor.markResources();
Expand Down Expand Up @@ -221,4 +232,9 @@ private boolean isJanitorMonkeyEnabled() {
LOGGER.info("JanitorMonkey disabled, set {}=true", prop);
return false;
}

@Monitor(name="runs", type=DataSourceType.COUNTER)
public AtomicLong getMonkeyRuns() {
return monkeyRuns;
}
}
58 changes: 57 additions & 1 deletion src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
package com.netflix.simianarmy.janitor;

import com.google.common.collect.Maps;
import com.netflix.servo.annotations.DataSourceType;
import com.netflix.servo.annotations.Monitor;
import com.netflix.servo.annotations.MonitorTags;
import com.netflix.simianarmy.MonkeyCalendar;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder;
Expand All @@ -41,6 +44,11 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import com.netflix.servo.monitor.Monitors;
import com.netflix.servo.tag.BasicTagList;
import com.netflix.servo.tag.TagList;

/**
* An abstract implementation of Janitor. It marks resources that the rule engine considers
Expand All @@ -55,12 +63,28 @@ public abstract class AbstractJanitor implements Janitor {
/** The Constant LOGGER. */
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJanitor.class);

/** Keep track of the number of cleaned resources */
protected final AtomicLong cleanedResourcesCount = new AtomicLong(0);

/** Keep track of the number of marked resources */
protected final AtomicLong markedResourcesCount = new AtomicLong(0);

/** Keep track of the number of failed to clean resources */
protected final AtomicLong failedToCleanResourcesCount = new AtomicLong(0);

/** Keep track of the number of unmarked resources */
protected final AtomicLong unmarkedResourcesCount = new AtomicLong(0);

/** Tags to attach to servo metrics */
@MonitorTags
protected TagList tags;

private final String region;
/** The region the janitor is running in. */
public String getRegion() {
return region;
}

/**
* The rule engine used to decide if a resource should be a cleanup
* candidate.
Expand Down Expand Up @@ -183,6 +207,13 @@ public AbstractJanitor(Context ctx, ResourceType resourceType) {
Validate.notNull(resourceType);
// recorder could be null and no events are recorded when it is.
this.recorder = ctx.recorder();

// setup servo tags, currently just tag each published metric with the region
this.tags = BasicTagList.of("simianarmy.janitor.region", ctx.region());

// register this janitor with servo
String monitorObjName = String.format("simianarmy.janitor.%s.%s", this.resourceType.name(), this.region);
Monitors.registerObject(monitorObjName, this);
}

@Override
Expand Down Expand Up @@ -222,6 +253,7 @@ public void markResources() {
recorder.recordEvent(evt);
}
resourceTracker.addOrUpdate(resource);
markedResourcesCount.incrementAndGet();
postMark(resource);
} else {
LOGGER.info(String.format(
Expand All @@ -247,6 +279,7 @@ public void markResources() {
resource.getId()));
}
unmarkedResources.add(resource);
unmarkedResourcesCount.incrementAndGet();
}
}

Expand Down Expand Up @@ -291,13 +324,15 @@ public void cleanupResources() {
recorder.recordEvent(evt);
}
cleanup(markedResource);
cleanedResourcesCount.incrementAndGet();
markedResource.setActualTerminationTime(now);
markedResource.setState(Resource.CleanupState.JANITOR_TERMINATED);
resourceTracker.addOrUpdate(markedResource);
} catch (Exception e) {
LOGGER.error(String.format("Failed to clean up the resource %s of type %s.",
markedResource.getId(), markedResource.getResourceType().name()), e);
failedToCleanResources.add(markedResource);
failedToCleanResourcesCount.incrementAndGet();
continue;
}
postCleanup(markedResource);
Expand Down Expand Up @@ -392,7 +427,28 @@ private void unmarkUserTerminatedResources(
markedResource.getId()));
}
unmarkedResources.add(markedResource);
unmarkedResourcesCount.incrementAndGet();
}
}
}

@Monitor(name="cleanedResourcesCount", type=DataSourceType.COUNTER)
long getResourcesCleanedCount() {
return cleanedResourcesCount.get();
}

@Monitor(name="markedResourcesCount", type=DataSourceType.COUNTER)
long getMarkedResourcesCount() {
return markedResourcesCount.get();
}

@Monitor(name="failedToCleanResourcesCount", type=DataSourceType.COUNTER)
long getFailedToCleanResourcesCount() {
return failedToCleanResourcesCount.get();
}

@Monitor(name="unmarkedResourcesCount", type=DataSourceType.COUNTER)
long getUnmarkedResourcesCount() {
return unmarkedResourcesCount.get();
}
}
35 changes: 32 additions & 3 deletions src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,10 @@ public static void testJanitor() {
Assert.assertEquals(janitor.cleanedResourceIds.size(), n / 2);
for (int i = 1; i <= n; i += 2) {
Assert.assertTrue(janitor.cleanedResourceIds.contains(String.valueOf(i)));
}
}
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand Down Expand Up @@ -196,7 +199,11 @@ public static void testJanitorWithOptedOutResources() {
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(resourceTracker.getResources(
TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(),
2);
2);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), 3);
}

@Test
Expand All @@ -222,7 +229,10 @@ public static void testJanitorWithCleanupFailure() {

janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - 1);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 1);
}

@Test
Expand Down Expand Up @@ -271,6 +281,10 @@ public static void testJanitorWithUnmarking() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n/6);
}


Expand Down Expand Up @@ -310,6 +324,9 @@ public static void testJanitorWithFutureTerminationTime() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), 0);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}


Expand Down Expand Up @@ -346,6 +363,9 @@ public static void testJanitorWithoutNotification() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), 0);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand All @@ -370,6 +390,8 @@ public static void testLeashedJanitorForMarking() {
n);
janitor.markResources();
Assert.assertEquals(janitor.getMarkedResources().size(), n / 2);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());

// No resource is really changed in tracker
Assert.assertEquals(resourceTracker.getResources(
Expand Down Expand Up @@ -415,6 +437,9 @@ public static void testJanitorWithoutHoldingOffCleanup() {
TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(),
n);
Assert.assertEquals(janitor.cleanedResourceIds.size(), n);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand Down Expand Up @@ -461,6 +486,10 @@ public static void testJanitorWithUnmarkingUserTerminated() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - n / 3 + n / 6);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n / 3);
}
}

Expand Down