From 3e8be715d966eea053620d601412bd0995c2b157 Mon Sep 17 00:00:00 2001 From: michaelf Date: Thu, 25 Apr 2013 16:06:58 -0700 Subject: [PATCH] Use the creation time of image as the reference time when there is no reference record found. --- .../crawler/edda/EddaImageJanitorCrawler.java | 49 +++++++++++++++++++ .../aws/janitor/rule/ami/UnusedImageRule.java | 29 +++++++++-- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java index 29f4ce4f..2d7d1d9f 100644 --- a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.Iterator; import java.util.List; @@ -71,6 +72,7 @@ public class EddaImageJanitorCrawler implements JanitorCrawler { private final Set usedByLaunchConfig = Sets.newHashSet(); private final Set usedNames = Sets.newHashSet(); private final Map imageIdToName = Maps.newHashMap(); + private final Map imageIdToCreationTime = Maps.newHashMap(); private final Set ancestorImageIds = Sets.newHashSet(); private String ownerId; @@ -128,6 +130,7 @@ private List getAMIResources(String... imageIds) { refreshIdToNameMap(); refreshAMIsUsedByInstance(); refreshAMIsUsedByLC(); + refreshIdToCreationTime(); for (String excludedId : getExcludedImageIds()) { String name = imageIdToName.get(excludedId); usedNames.add(name); @@ -194,6 +197,47 @@ private void refreshIdToNameMap() { LOGGER.info(String.format("Got mapping from image id to name for %d ids", imageIdToName.size())); } + /** + * AWS doesn't provide creation time for images. We use the ctime (the creation time of the image record in Edda) + * to approximate the creation time of the image. + */ + private void refreshIdToCreationTime() { + for (String region : regions) { + String url = eddaClient.getBaseUrl(region) + "/aws/images"; + LOGGER.info(String.format("Getting the creation time for all AMIs in region %s", region)); + if (StringUtils.isNotBlank(ownerId)) { + url += ";data.ownerId=" + ownerId; + } + url += ";_expand;_meta:(ctime,data:(imageId))"; + + JsonNode jsonNode = null; + try { + jsonNode = eddaClient.getJsonNodeFromUrl(url); + } catch (Exception e) { + LOGGER.error(String.format( + "Failed to get Jason node from edda for creation time of AMIs in region %s.", region), e); + } + + if (jsonNode == null || !jsonNode.isArray()) { + throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode)); + } + + for (Iterator it = jsonNode.getElements(); it.hasNext();) { + JsonNode elem = it.next(); + JsonNode data = elem.get("data"); + String imageId = data.get("imageId").getTextValue(); + JsonNode ctimeNode = elem.get("ctime"); + if (ctimeNode != null && !ctimeNode.isNull()) { + long ctime = ctimeNode.asLong(); + LOGGER.debug(String.format("The image record of %s was created in Edda at %s", + imageId, new DateTime(ctime))); + imageIdToCreationTime.put(imageId, ctime); + } + } + } + LOGGER.info(String.format("Got creation time for %d images", imageIdToCreationTime.size())); + } + private List getAMIResourcesInRegion( String region, Collection excludedImageIds, String... imageIds) { JsonNode jsonNode = getImagesInJson(region, imageIds); @@ -241,6 +285,11 @@ private Resource parseJsonElementToresource(String region, JsonNode jsonNode) { Resource resource = new AWSResource().withId(imageId).withRegion(region) .withResourceType(AWSResourceType.IMAGE); + Long creationTime = imageIdToCreationTime.get(imageId); + if (creationTime != null) { + resource.setLaunchTime(new Date(creationTime)); + } + JsonNode tags = jsonNode.get("tags"); if (tags == null || !tags.isArray() || tags.size() == 0) { LOGGER.debug(String.format("No tags is found for %s", resource.getId())); diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/ami/UnusedImageRule.java b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/ami/UnusedImageRule.java index b6446734..7b22fd45 100644 --- a/src/main/java/com/netflix/simianarmy/aws/janitor/rule/ami/UnusedImageRule.java +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/rule/ami/UnusedImageRule.java @@ -76,13 +76,11 @@ public boolean isValid(Resource resource) { resource.getId())); return true; } - String instanceRefTime = resource.getAdditionalField(EddaImageJanitorCrawler.AMI_FIELD_LAST_INSTANCE_REF_TIME); - String lcRefTime = resource.getAdditionalField(EddaImageJanitorCrawler.AMI_FIELD_LAST_LC_REF_TIME); + long instanceRefTime = getRefTimeInMilis(resource, EddaImageJanitorCrawler.AMI_FIELD_LAST_INSTANCE_REF_TIME); + long lcRefTime = getRefTimeInMilis(resource, EddaImageJanitorCrawler.AMI_FIELD_LAST_LC_REF_TIME); Date now = calendar.now().getTime(); long windowStart = new DateTime(now.getTime()).minusDays(lastReferenceDaysThreshold).getMillis(); - boolean instanceOld = instanceRefTime != null && Long.parseLong(instanceRefTime) < windowStart; - boolean lcOld = lcRefTime != null && Long.parseLong(lcRefTime) < windowStart; - if (instanceRefTime == null && lcOld || lcRefTime == null && instanceOld || lcOld && instanceOld) { + if (instanceRefTime < windowStart && lcRefTime < windowStart) { if (resource.getExpectedTerminationTime() == null) { Date terminationTime = calendar.getBusinessDay(now, retentionDays); resource.setExpectedTerminationTime(terminationTime); @@ -100,4 +98,25 @@ public boolean isValid(Resource resource) { } return true; } + + /** + * Tries to get the long value from the provided field. If the field does not exist, try to use the + * creation time. If both do not exist, use the current time. + */ + private long getRefTimeInMilis(Resource resource, String field) { + String fieldValue = resource.getAdditionalField(field); + long refTime; + if (fieldValue != null) { + refTime = Long.parseLong(fieldValue); + } else if (resource.getLaunchTime() != null) { + LOGGER.info(String.format("No value in field %s is found, use the creation time %s as the ref time of %s", + field, resource.getLaunchTime(), resource.getId())); + refTime = resource.getLaunchTime().getTime(); + } else { + // When there is no creation time or ref time is found, we consider the image is referenced. + LOGGER.info(String.format("Use the current time as the ref time of %s", resource.getId())); + refTime = DateTime.now().getMillis(); + } + return refTime; + } }