这是indexloc提供的服务，不要输入任何密码
Skip to content
This repository was archived by the owner on Mar 4, 2021. It is now read-only.

Changes for Netflix environment compatibility #272

Merged
merged 2 commits into from
Sep 20, 2016
Merged

Changes for Netflix environment compatibility #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e786031
Changes for Netflix environment compatibility: AWS client to 1.11.9, …
ebukoski Aug 30, 2016
164ae81
Add region to janitor RDS operations. Add file headers to clear some…
ebukoski Sep 14, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies {
compile 'org.codehaus.jackson:jackson-core-asl:1.9.2'
compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2'
compile 'com.netflix.eureka:eureka-client:1.4.1'
compile 'com.amazonaws:aws-java-sdk:1.10.5.1'
compile 'com.amazonaws:aws-java-sdk:1.11.9'
compile 'commons-lang:commons-lang:2.6'
compile 'com.google.guava:guava:11.0.2'
compile 'org.apache.httpcomponents:httpclient:4.3'
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public Object emailValue(String email) {
@Override
public void addOrUpdate(Cluster cluster) {
Cluster orig = getCluster(cluster.getName(), cluster.getRegion());
LOGGER.debug(String.format("Saving cluster %s to RDB table %s", cluster.getName(), table));
LOGGER.debug(String.format("Saving cluster %s to RDB table %s in region %s", cluster.getName(), cluster.getRegion(), table));
Map<String, String> map = cluster.getFieldToValueMap();

String conformityJson;
Expand Down
42 changes: 34 additions & 8 deletions src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,13 @@ public Object emailValue(String email) {
/** {@inheritDoc} */
@Override
public void addOrUpdate(Resource resource) {
Resource orig = getResource(resource.getId());
LOGGER.debug(String.format("Saving resource %s to RDB table %s", resource.getId(), table));
Resource orig = getResource(resource.getId(), resource.getRegion());
LOGGER.debug(String.format("Saving resource %s to RDB table %s in region %s", resource.getId(), table, resource.getRegion()));
String json;
try {
json = new ObjectMapper().writeValueAsString(additionalFieldsAsMap(resource));
} catch (JsonProcessingException e) {
LOGGER.error("ERROR generating additonal field JSON when saving resource " + resource.getId(), e);
LOGGER.error("ERROR generating additional field JSON when saving resource " + resource.getId(), e);
return;
}

Expand Down Expand Up @@ -166,7 +166,8 @@ public void addOrUpdate(Resource resource) {
sb.append(AWSResource.FIELD_MARK_TIME).append("=?,");
sb.append(AWSResource.FIELD_OPT_OUT_OF_JANITOR).append("=?,");
sb.append("additionalFields").append("=? where ");
sb.append(AWSResource.FIELD_RESOURCE_ID).append("=?");
sb.append(AWSResource.FIELD_RESOURCE_ID).append("=? and ");
sb.append(AWSResource.FIELD_REGION).append("=?");

LOGGER.debug(String.format("Update statement is '%s'", sb));
int updated = this.jdbcTemplate.update(sb.toString(),
Expand All @@ -183,7 +184,8 @@ public void addOrUpdate(Resource resource) {
value(resource.getMarkTime()),
value(resource.isOptOutOfJanitor()),
json,
resource.getId());
resource.getId(),
resource.getRegion());
LOGGER.debug(String.format("%d rows updated", updated));
}
LOGGER.debug("Successfully saved.");
Expand Down Expand Up @@ -309,9 +311,33 @@ public Resource mapRow(ResultSet rs, int rowNum) throws SQLException {
resource = resources.get(0);
}
return resource;
}

/**
}

@Override
public Resource getResource(String resourceId, String region) {
Validate.notEmpty(resourceId);
Validate.notEmpty(region);
StringBuilder query = new StringBuilder();
query.append(String.format("select * from %s where resourceId=? and region=?", table));

LOGGER.debug(String.format("Query is '%s'", query));
List<Resource> resources = jdbcTemplate.query(query.toString(), new String[]{resourceId,region}, new RowMapper<Resource>() {
public Resource mapRow(ResultSet rs, int rowNum) throws SQLException {
return mapResource(rs);
}
});

Resource resource = null;
Validate.isTrue(resources.size() <= 1);
if (resources.size() == 0) {
LOGGER.info(String.format("Not found resource with id %s", resourceId));
} else {
resource = resources.get(0);
}
return resource;
}

/**
* Creates the RDS table, if it does not already exist.
*/
public void init() {
Expand Down
50 changes: 35 additions & 15 deletions src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,22 @@
*/
package com.netflix.simianarmy.aws.janitor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.services.simpledb.AmazonSimpleDB;
import com.amazonaws.services.simpledb.model.Attribute;
import com.amazonaws.services.simpledb.model.Item;
import com.amazonaws.services.simpledb.model.PutAttributesRequest;
import com.amazonaws.services.simpledb.model.ReplaceableAttribute;
import com.amazonaws.services.simpledb.model.SelectRequest;
import com.amazonaws.services.simpledb.model.SelectResult;
import com.amazonaws.services.simpledb.model.*;
import com.netflix.simianarmy.Resource;
import com.netflix.simianarmy.Resource.CleanupState;
import com.netflix.simianarmy.ResourceType;
import com.netflix.simianarmy.aws.AWSResource;
import com.netflix.simianarmy.client.aws.AWSClient;
import com.netflix.simianarmy.janitor.JanitorResourceTracker;
import org.apache.commons.lang.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* The JanitorResourceTracker implementation in SimpleDB.
Expand Down Expand Up @@ -151,6 +145,32 @@ public Resource getResource(String resourceId) {
}
}

@Override
public Resource getResource(String resourceId, String region) {
Validate.notEmpty(resourceId);
Validate.notEmpty(region);
StringBuilder query = new StringBuilder();
query.append(String.format("select * from `%s` where resourceId = '%s' and region = '%s'", domain, resourceId, region));

LOGGER.debug(String.format("Query is '%s'", query));

List<Item> items = querySimpleDBItems(query.toString());
Validate.isTrue(items.size() <= 1);
if (items.size() == 0) {
LOGGER.info(String.format("Not found resource with id %s and region %s", resourceId, region));
return null;
} else {
Resource resource = null;
try {
resource = parseResource(items.get(0));
} catch (Exception e) {
// Ignore the item that cannot be parsed.
LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a resource.", items.get(0)));
}
return resource;
}
}

/**
* Parses a SimpleDB item into an AWS resource.
* @param item the item from SimpleDB
Expand Down
25 changes: 20 additions & 5 deletions src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,23 +151,38 @@ public void doMonkeyBusiness() {

@Override
public Event optInResource(String resourceId) {
return optInOrOutResource(resourceId, true);
return optInOrOutResource(resourceId, true, region);
}

@Override
public Event optOutResource(String resourceId) {
return optInOrOutResource(resourceId, false);
return optInOrOutResource(resourceId, false, region);
}

private Event optInOrOutResource(String resourceId, boolean optIn) {
Resource resource = resourceTracker.getResource(resourceId);
@Override
public Event optInResource(String resourceId, String resourceRegion) {
return optInOrOutResource(resourceId, true, resourceRegion);
}

@Override
public Event optOutResource(String resourceId, String resourceRegion) {
return optInOrOutResource(resourceId, false, resourceRegion);
}

private Event optInOrOutResource(String resourceId, boolean optIn, String resourceRegion) {
if (resourceRegion == null) {
resourceRegion = region;
}

Resource resource = resourceTracker.getResource(resourceId, resourceRegion);
if (resource == null) {
return null;
}

EventTypes eventType = optIn ? EventTypes.OPT_IN_RESOURCE : EventTypes.OPT_OUT_RESOURCE;
long timestamp = calendar.now().getTimeInMillis();
// The same resource can have multiple events, so we add the timestamp to the id.
Event evt = recorder.newEvent(Type.JANITOR, eventType, region, resourceId + "@" + timestamp);
Event evt = recorder.newEvent(Type.JANITOR, eventType, resourceRegion, resourceId + "@" + timestamp);
recorder.recordEvent(evt);
resource.setOptOutOfJanitor(!optIn);
resourceTracker.addOrUpdate(resource);
Expand Down
63 changes: 11 additions & 52 deletions src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,54 +21,16 @@
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.autoscaling.AmazonAutoScalingClient;
import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
import com.amazonaws.services.autoscaling.model.AutoScalingInstanceDetails;
import com.amazonaws.services.autoscaling.model.DeleteAutoScalingGroupRequest;
import com.amazonaws.services.autoscaling.model.DeleteLaunchConfigurationRequest;
import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest;
import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult;
import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesRequest;
import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesResult;
import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsRequest;
import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsResult;
import com.amazonaws.services.autoscaling.model.LaunchConfiguration;
import com.amazonaws.services.autoscaling.model.*;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2Client;
import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest;
import com.amazonaws.services.ec2.model.CreateSecurityGroupResult;
import com.amazonaws.services.ec2.model.CreateTagsRequest;
import com.amazonaws.services.ec2.model.DeleteSnapshotRequest;
import com.amazonaws.services.ec2.model.DeleteVolumeRequest;
import com.amazonaws.services.ec2.model.DeregisterImageRequest;
import com.amazonaws.services.ec2.model.DescribeImagesRequest;
import com.amazonaws.services.ec2.model.DescribeImagesResult;
import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
import com.amazonaws.services.ec2.model.DescribeInstancesResult;
import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest;
import com.amazonaws.services.ec2.model.DescribeSnapshotsResult;
import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
import com.amazonaws.services.ec2.model.DescribeVolumesResult;
import com.amazonaws.services.ec2.model.DetachVolumeRequest;
import com.amazonaws.services.ec2.model.EbsInstanceBlockDevice;
import com.amazonaws.services.ec2.model.Image;
import com.amazonaws.services.ec2.model.*;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.InstanceBlockDeviceMapping;
import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
import com.amazonaws.services.ec2.model.Reservation;
import com.amazonaws.services.ec2.model.SecurityGroup;
import com.amazonaws.services.ec2.model.Snapshot;
import com.amazonaws.services.ec2.model.Tag;
import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
import com.amazonaws.services.ec2.model.Volume;
import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancingClient;
import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancerAttributesRequest;
import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancerAttributesResult;
import com.amazonaws.services.elasticloadbalancing.model.*;
import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersRequest;
import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersResult;
import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerAttributes;
import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription;
import com.amazonaws.services.simpledb.AmazonSimpleDB;
import com.amazonaws.services.simpledb.AmazonSimpleDBClient;
import com.google.common.base.Objects;
Expand All @@ -79,7 +41,6 @@
import com.google.inject.Module;
import com.netflix.simianarmy.CloudClient;
import com.netflix.simianarmy.NotFoundException;

import org.apache.commons.lang.Validate;
import org.jclouds.ContextBuilder;
import org.jclouds.compute.ComputeService;
Expand All @@ -89,20 +50,13 @@
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.NodeMetadataBuilder;
import org.jclouds.domain.LoginCredentials;
import org.jclouds.ec2.EC2ApiMetadata;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import org.jclouds.ssh.SshClient;
import org.jclouds.ssh.jsch.config.JschSshClientModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;


/**
Expand Down Expand Up @@ -515,8 +469,13 @@ public void deleteAutoScalingGroup(String asgName) {
LOGGER.info(String.format("Deleting auto-scaling group with name %s in region %s.", asgName, region));
AmazonAutoScalingClient asgClient = asgClient();
DeleteAutoScalingGroupRequest request = new DeleteAutoScalingGroupRequest()
.withAutoScalingGroupName(asgName);
asgClient.deleteAutoScalingGroup(request);
.withAutoScalingGroupName(asgName).withForceDelete(true);
try {
asgClient.deleteAutoScalingGroup(request);
LOGGER.info(String.format("Deleted auto-scaling group with name %s in region %s.", asgName, region));
}catch(Exception e) {
LOGGER.error("Got an exception deleting ASG " + asgName, e);
}
}

/** {@inheritDoc} */
Expand Down
20 changes: 18 additions & 2 deletions src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@
*/
package com.netflix.simianarmy.janitor;

import java.util.List;

import com.netflix.simianarmy.EventType;
import com.netflix.simianarmy.Monkey;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder.Event;
import com.netflix.simianarmy.MonkeyType;

import java.util.List;

/**
* The abstract class for a Janitor Monkey.
*/
Expand Down Expand Up @@ -150,4 +150,20 @@ public Context context() {
*/
public abstract Event optOutResource(String resourceId);

/**
* Opt in a resource for Janitor Monkey.
* @param resourceId the resource id
* @param region the region of the resource
* @return the opt-in event
*/
public abstract Event optInResource(String resourceId, String region);

/**
* Opt out a resource for Janitor Monkey.
* @param resourceId the resource id
* @param region the region of the resource
* @return the opt-out event
*/
public abstract Event optOutResource(String resourceId, String region);

}
12 changes: 10 additions & 2 deletions src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
*/
package com.netflix.simianarmy.janitor;

import java.util.List;

import com.netflix.simianarmy.Resource;
import com.netflix.simianarmy.ResourceType;

import java.util.List;

/**
* The interface to track the resources marked/cleaned by the Janitor Monkey.
*
Expand Down Expand Up @@ -52,4 +52,12 @@ public interface JanitorResourceTracker {
*/
Resource getResource(String resourceId);

/** Gets the resource of a specific id.
*
* @param resourceId the resource id
* @param regionId the region id
* @return the resource that matches the resource id and region
*/
Resource getResource(String resourceId, String regionId);

}
Loading