这是indexloc提供的服务，不要输入任何密码
Skip to content
This repository was archived by the owner on Mar 4, 2021. It is now read-only.

Use the creation time of image as the reference time when there is no reference record found #48

Merged
merged 1 commit into from
Apr 25, 2013
Merged

Use the creation time of image as the reference time when there is no reference record found #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -71,6 +72,7 @@ public class EddaImageJanitorCrawler implements JanitorCrawler {
private final Set<String> usedByLaunchConfig = Sets.newHashSet();
private final Set<String> usedNames = Sets.newHashSet();
private final Map<String, String> imageIdToName = Maps.newHashMap();
private final Map<String, Long> imageIdToCreationTime = Maps.newHashMap();
private final Set<String> ancestorImageIds = Sets.newHashSet();

private String ownerId;
Expand Down Expand Up @@ -128,6 +130,7 @@ private List<Resource> getAMIResources(String... imageIds) {
refreshIdToNameMap();
refreshAMIsUsedByInstance();
refreshAMIsUsedByLC();
refreshIdToCreationTime();
for (String excludedId : getExcludedImageIds()) {
String name = imageIdToName.get(excludedId);
usedNames.add(name);
Expand Down Expand Up @@ -194,6 +197,47 @@ private void refreshIdToNameMap() {
LOGGER.info(String.format("Got mapping from image id to name for %d ids", imageIdToName.size()));
}

/**
* AWS doesn't provide creation time for images. We use the ctime (the creation time of the image record in Edda)
* to approximate the creation time of the image.
*/
private void refreshIdToCreationTime() {
for (String region : regions) {
String url = eddaClient.getBaseUrl(region) + "/aws/images";
LOGGER.info(String.format("Getting the creation time for all AMIs in region %s", region));
if (StringUtils.isNotBlank(ownerId)) {
url += ";data.ownerId=" + ownerId;
}
url += ";_expand;_meta:(ctime,data:(imageId))";

JsonNode jsonNode = null;
try {
jsonNode = eddaClient.getJsonNodeFromUrl(url);
} catch (Exception e) {
LOGGER.error(String.format(
"Failed to get Jason node from edda for creation time of AMIs in region %s.", region), e);
}

if (jsonNode == null || !jsonNode.isArray()) {
throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
}

for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext();) {
JsonNode elem = it.next();
JsonNode data = elem.get("data");
String imageId = data.get("imageId").getTextValue();
JsonNode ctimeNode = elem.get("ctime");
if (ctimeNode != null && !ctimeNode.isNull()) {
long ctime = ctimeNode.asLong();
LOGGER.debug(String.format("The image record of %s was created in Edda at %s",
imageId, new DateTime(ctime)));
imageIdToCreationTime.put(imageId, ctime);
}
}
}
LOGGER.info(String.format("Got creation time for %d images", imageIdToCreationTime.size()));
}

private List<Resource> getAMIResourcesInRegion(
String region, Collection<String> excludedImageIds, String... imageIds) {
JsonNode jsonNode = getImagesInJson(region, imageIds);
Expand Down Expand Up @@ -241,6 +285,11 @@ private Resource parseJsonElementToresource(String region, JsonNode jsonNode) {
Resource resource = new AWSResource().withId(imageId).withRegion(region)
.withResourceType(AWSResourceType.IMAGE);

Long creationTime = imageIdToCreationTime.get(imageId);
if (creationTime != null) {
resource.setLaunchTime(new Date(creationTime));
}

JsonNode tags = jsonNode.get("tags");
if (tags == null || !tags.isArray() || tags.size() == 0) {
LOGGER.debug(String.format("No tags is found for %s", resource.getId()));
Expand Down
29 changes: 24 additions & 5 deletions src/main/java/com/netflix/simianarmy/aws/janitor/rule/ami/UnusedImageRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,11 @@ public boolean isValid(Resource resource) {
resource.getId()));
return true;
}
String instanceRefTime = resource.getAdditionalField(EddaImageJanitorCrawler.AMI_FIELD_LAST_INSTANCE_REF_TIME);
String lcRefTime = resource.getAdditionalField(EddaImageJanitorCrawler.AMI_FIELD_LAST_LC_REF_TIME);
long instanceRefTime = getRefTimeInMilis(resource, EddaImageJanitorCrawler.AMI_FIELD_LAST_INSTANCE_REF_TIME);
long lcRefTime = getRefTimeInMilis(resource, EddaImageJanitorCrawler.AMI_FIELD_LAST_LC_REF_TIME);
Date now = calendar.now().getTime();
long windowStart = new DateTime(now.getTime()).minusDays(lastReferenceDaysThreshold).getMillis();
boolean instanceOld = instanceRefTime != null && Long.parseLong(instanceRefTime) < windowStart;
boolean lcOld = lcRefTime != null && Long.parseLong(lcRefTime) < windowStart;
if (instanceRefTime == null && lcOld || lcRefTime == null && instanceOld || lcOld && instanceOld) {
if (instanceRefTime < windowStart && lcRefTime < windowStart) {
if (resource.getExpectedTerminationTime() == null) {
Date terminationTime = calendar.getBusinessDay(now, retentionDays);
resource.setExpectedTerminationTime(terminationTime);
Expand All @@ -100,4 +98,25 @@ public boolean isValid(Resource resource) {
}
return true;
}

/**
* Tries to get the long value from the provided field. If the field does not exist, try to use the
* creation time. If both do not exist, use the current time.
*/
private long getRefTimeInMilis(Resource resource, String field) {
String fieldValue = resource.getAdditionalField(field);
long refTime;
if (fieldValue != null) {
refTime = Long.parseLong(fieldValue);
} else if (resource.getLaunchTime() != null) {
LOGGER.info(String.format("No value in field %s is found, use the creation time %s as the ref time of %s",
field, resource.getLaunchTime(), resource.getId()));
refTime = resource.getLaunchTime().getTime();
} else {
// When there is no creation time or ref time is found, we consider the image is referenced.
LOGGER.info(String.format("Use the current time as the ref time of %s", resource.getId()));
refTime = DateTime.now().getMillis();
}
return refTime;
}
}