这是indexloc提供的服务，不要输入任何密码
Skip to content
This repository was archived by the owner on Mar 4, 2021. It is now read-only.

Request limit exceeded exponential backoff (Volume Tagging Monkey) #190

Closed
wants to merge 6 commits into from
Closed

Request limit exceeded exponential backoff (Volume Tagging Monkey) #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
23d34e3
Configurable Janitor owner tag as per netflix/SimianArmy#178
lantrix Apr 30, 2015
57f6987
Fix for java.lang.NullPointerException on BasicJanitorRuleEngine.
lantrix May 8, 2015
d463acc
Merge remote-tracking branch 'upstream/master'
lantrix May 18, 2015
a94f6a1
Handle RequestLimitExceeded with AWS API
lantrix May 26, 2015
4d79fab
Undo 23d34e3
lantrix May 26, 2015
efe736b
Merge branch 'master' into RequestLimitExceeded-exponentialBackoff
lantrix May 26, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 39 additions & 7 deletions src/main/java/com/netflix/simianarmy/aws/janitor/VolumeTaggingMonkey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;

import com.amazonaws.AmazonServiceException;
import com.google.common.collect.Maps;

import org.apache.commons.lang.StringUtils;
Expand Down Expand Up @@ -268,13 +269,37 @@ private Event updateJanitorMetaTag(Volume volume, String instance, String owner,
if (config.getBoolOrElse(prop, true)) {
LOGGER.info("Volume tagging monkey is leashed. No real change is made to the volume.");
} else {
try {
awsClient.createTagsForResources(janitorTags, volume.getVolumeId());
evt = context().recorder().newEvent(type(), EventTypes.TAGGING_JANITOR,
awsClient.region(), volume.getVolumeId());
evt.addField(JanitorMonkey.JANITOR_META_TAG, meta);
} catch (Exception e) {
LOGGER.error(String.format("Failed to update the tag for volume %s", volume.getVolumeId()));
int currentTry = 0;
int maxRetries = 5;
while(maxRetries-- > 0) {
try {
currentTry++;
//call Amazon to set Volume Meta Tag
awsClient.createTagsForResources(janitorTags, volume.getVolumeId());
evt = context().recorder().newEvent(type(), EventTypes.TAGGING_JANITOR,
awsClient.region(), volume.getVolumeId());
evt.addField(JanitorMonkey.JANITOR_META_TAG, meta);
} catch (AmazonServiceException e) {
if ("RequestLimitExceeded".equals(e.getErrorCode())) {
//exponentially increase the backoff duration on each consecutive RequestLimitExceeded failure
long backoffDuration = getSleepDuration(currentTry, 10, 5000);
LOGGER.debug(String.format("RequestLimitExceeded when tagging volume %s", volume.getVolumeId()));
if (maxRetries > 1) {
LOGGER.debug(String.format("Will retry after %s ms backoff", backoffDuration));
} else {
LOGGER.debug(String.format("Will not retry after backoff"));
}
try {
Thread.sleep(backoffDuration);
} catch (InterruptedException e1) {
LOGGER.error(String.format("Failed to update the tag for volume %s", volume.getVolumeId()));
}
} else {
LOGGER.error(String.format("Failed to update the tag for volume %s ; AWS Exception Error Code: %s", volume.getVolumeId(), e.getErrorCode()));
}
} catch (Exception e) {
LOGGER.error(String.format("Failed to update the tag for volume %s : Exception caught", volume.getVolumeId()));
}
}
}
return evt;
Expand All @@ -298,6 +323,13 @@ public static String makeMetaTag(String instance, String owner, Date lastDetachT
return meta.toString();
}

/* exponential backoff algorithm to deal with Rate Limit Exceeded */
public static long getSleepDuration(int currentTry, long minSleepMillis, long maxSleepMillis) {
currentTry = Math.max(0, currentTry);
long currentSleepMillis = (long) (minSleepMillis*Math.pow(2, currentTry));
return Math.min(currentSleepMillis, maxSleepMillis);
}

private static String getTagValue(String key, List<Tag> tags) {
for (Tag tag : tags) {
if (tag.getKey().equals(key)) {
Expand Down
11 changes: 7 additions & 4 deletions src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorRuleEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,13 @@ public boolean isValid(Resource resource) {
Date nearestTerminationTime = null;
for (Rule rule : rules) {
Resource clone = resource.cloneResource();
if (!rule.isValid(clone) && (nearestTerminationTime == null
|| nearestTerminationTime.after(clone.getExpectedTerminationTime()))) {
nearestRule = rule;
nearestTerminationTime = clone.getExpectedTerminationTime();
if (!rule.isValid(clone)) {
if (clone.getExpectedTerminationTime() != null) {
if (nearestTerminationTime == null || nearestTerminationTime.after(clone.getExpectedTerminationTime())) {
nearestRule = rule;
nearestTerminationTime = clone.getExpectedTerminationTime();
}
}
}
}
}
Expand Down