diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/SimpleDBConformityClusterTracker.java b/src/main/java/com/netflix/simianarmy/aws/conformity/SimpleDBConformityClusterTracker.java new file mode 100644 index 00000000..0d557610 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/SimpleDBConformityClusterTracker.java @@ -0,0 +1,224 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity; + +import com.amazonaws.services.simpledb.AmazonSimpleDB; +import com.amazonaws.services.simpledb.model.Attribute; +import com.amazonaws.services.simpledb.model.DeleteAttributesRequest; +import com.amazonaws.services.simpledb.model.Item; +import com.amazonaws.services.simpledb.model.PutAttributesRequest; +import com.amazonaws.services.simpledb.model.ReplaceableAttribute; +import com.amazonaws.services.simpledb.model.SelectRequest; +import com.amazonaws.services.simpledb.model.SelectResult; +import com.google.common.collect.Lists; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.ConformityClusterTracker; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The ConformityResourceTracker implementation in SimpleDB. + */ +public class SimpleDBConformityClusterTracker implements ConformityClusterTracker { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(SimpleDBConformityClusterTracker.class); + + /** The domain. */ + private final String domain; + + /** The SimpleDB client. */ + private final AmazonSimpleDB simpleDBClient; + + /** + * Instantiates a new simple db cluster tracker for conformity monkey. + * + * @param awsClient + * the AWS Client + * @param domain + * the domain + */ + public SimpleDBConformityClusterTracker(AWSClient awsClient, String domain) { + Validate.notNull(awsClient); + Validate.notNull(domain); + this.domain = domain; + this.simpleDBClient = awsClient.sdbClient(); + } + + /** + * Gets the SimpleDB client. + * @return the SimpleDB client + */ + protected AmazonSimpleDB getSimpleDBClient() { + return simpleDBClient; + } + + /** {@inheritDoc} */ + @Override + public void addOrUpdate(Cluster cluster) { + List attrs = new ArrayList(); + Map fieldToValueMap = cluster.getFieldToValueMap(); + for (Map.Entry entry : fieldToValueMap.entrySet()) { + attrs.add(new ReplaceableAttribute(entry.getKey(), entry.getValue(), true)); + } + PutAttributesRequest putReqest = new PutAttributesRequest(domain, getSimpleDBItemName(cluster), attrs); + LOGGER.debug(String.format("Saving cluster %s to SimpleDB domain %s", + cluster.getName(), domain)); + this.simpleDBClient.putAttributes(putReqest); + LOGGER.debug("Successfully saved."); + } + + + + /** + * Gets the clusters for a list of regions. If the regions parameter is empty, returns the clusters + * for all regions. + */ + @Override + public List getAllClusters(String... regions) { + return getClusters(null, regions); + } + + @Override + public List getNonconformingClusters(String... regions) { + return getClusters(false, regions); + } + + @Override + public Cluster getCluster(String clusterName, String region) { + Validate.notEmpty(clusterName); + Validate.notEmpty(region); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where cluster = '%s' and region = '%s'", + domain, clusterName, region)); + + LOGGER.info(String.format("Query is to get the cluster is '%s'", query)); + + List items = querySimpleDBItems(query.toString()); + Validate.isTrue(items.size() <= 1); + if (items.size() == 0) { + LOGGER.info(String.format("Not found cluster with name %s in region %s", clusterName, region)); + return null; + } else { + Cluster cluster = null; + try { + cluster = parseCluster(items.get(0)); + } catch (Exception e) { + // Ignore the item that cannot be parsed. + LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a cluster.", items.get(0))); + } + return cluster; + } + } + + @Override + public void deleteClusters(Cluster... clusters) { + Validate.notNull(clusters); + LOGGER.info(String.format("Deleting %d clusters", clusters.length)); + for (Cluster cluster : clusters) { + LOGGER.info(String.format("Deleting cluster %s", cluster.getName())); + simpleDBClient.deleteAttributes(new DeleteAttributesRequest(domain, getSimpleDBItemName(cluster))); + LOGGER.info(String.format("Successfully deleted cluster %s", cluster.getName())); + } + } + + private List getClusters(Boolean conforming, String... regions) { + Validate.notNull(regions); + List clusters = Lists.newArrayList(); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where cluster is not null and ", domain)); + boolean needsAnd = false; + if (regions.length != 0) { + query.append(String.format("region in ('%s') ", StringUtils.join(regions, "','"))); + needsAnd = true; + } + if (conforming != null) { + if (needsAnd) { + query.append(" and "); + } + query.append(String.format("isConforming = '%s'", conforming)); + } + + LOGGER.info(String.format("Query to retrieve clusters for regions %s is '%s'", + StringUtils.join(regions, "','"), query.toString())); + + List items = querySimpleDBItems(query.toString()); + for (Item item : items) { + try { + clusters.add(parseCluster(item)); + } catch (Exception e) { + // Ignore the item that cannot be parsed. + LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a cluster.", item), e); + } + } + LOGGER.info(String.format("Retrieved %d clusters from SimpleDB in domain %s and regions %s", + clusters.size(), domain, StringUtils.join(regions, "','"))); + return clusters; + } + + /** + * Parses a SimpleDB item into a cluster. + * @param item the item from SimpleDB + * @return the cluster for the SimpleDB item + */ + protected Cluster parseCluster(Item item) { + Map fieldToValue = new HashMap(); + for (Attribute attr : item.getAttributes()) { + String name = attr.getName(); + String value = attr.getValue(); + if (name != null && value != null) { + fieldToValue.put(name, value); + } + } + return Cluster.parseFieldToValueMap(fieldToValue); + } + + /** + * Gets the unique SimpleDB item name for a cluster. The subclass can override this + * method to generate the item name differently. + * @param cluster + * @return the SimpleDB item name for the cluster + */ + protected String getSimpleDBItemName(Cluster cluster) { + return String.format("%s-%s", cluster.getName(), cluster.getRegion()); + } + + private List querySimpleDBItems(String query) { + Validate.notNull(query); + String nextToken = null; + List items = new ArrayList(); + do { + SelectRequest request = new SelectRequest(query); + request.setNextToken(nextToken); + request.setConsistentRead(Boolean.TRUE); + SelectResult result = this.simpleDBClient.select(request); + items.addAll(result.getItems()); + nextToken = result.getNextToken(); + } while (nextToken != null); + + return items; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/crawler/AWSClusterCrawler.java b/src/main/java/com/netflix/simianarmy/aws/conformity/crawler/AWSClusterCrawler.java new file mode 100644 index 00000000..92281c09 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/crawler/AWSClusterCrawler.java @@ -0,0 +1,141 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.crawler; + +import com.amazonaws.services.autoscaling.model.AutoScalingGroup; +import com.amazonaws.services.autoscaling.model.Instance; +import com.amazonaws.services.autoscaling.model.SuspendedProcess; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.ClusterCrawler; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +/** + * The class implementing a crawler that gets the auto scaling groups from AWS. + */ +public class AWSClusterCrawler implements ClusterCrawler { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(AWSClusterCrawler.class); + + private static final String NS = "simianarmy.conformity.cluster"; + + /** The map from region to the aws client in the region. */ + private final Map regionToAwsClient = Maps.newHashMap(); + + private final MonkeyConfiguration cfg; + + /** + * Instantiates a new cluster crawler. + * + * @param regionToAwsClient + * the map from region to the correponding aws client for the region + */ + public AWSClusterCrawler(Map regionToAwsClient, MonkeyConfiguration cfg) { + Validate.notNull(regionToAwsClient); + Validate.notNull(cfg); + for (Map.Entry entry : regionToAwsClient.entrySet()) { + this.regionToAwsClient.put(entry.getKey(), entry.getValue()); + } + this.cfg = cfg; + } + + /** + * In this implementation, every auto scaling group is considered a cluster. + * @param clusterNames + * the cluster names + * @return the list of clusters matching the names, when names are empty, return all clusters + */ + @Override + public List clusters(String... clusterNames) { + List list = Lists.newArrayList(); + for (Map.Entry entry : regionToAwsClient.entrySet()) { + String region = entry.getKey(); + AWSClient awsClient = entry.getValue(); + LOGGER.info(String.format("Crawling clusters in region %s", region)); + for (AutoScalingGroup asg : awsClient.describeAutoScalingGroups(clusterNames)) { + List instances = Lists.newArrayList(); + for (Instance instance : asg.getInstances()) { + instances.add(instance.getInstanceId()); + } + com.netflix.simianarmy.conformity.AutoScalingGroup conformityAsg = + new com.netflix.simianarmy.conformity.AutoScalingGroup( + asg.getAutoScalingGroupName(), + instances.toArray(new String[instances.size()])); + + for (SuspendedProcess sp : asg.getSuspendedProcesses()) { + if ("AddToLoadBalancer".equals(sp.getProcessName())) { + LOGGER.info(String.format("ASG %s is suspeneded: %s", asg.getAutoScalingGroupName(), + asg.getSuspendedProcesses())); + conformityAsg.setSuspended(true); + } + } + Cluster cluster = new Cluster(asg.getAutoScalingGroupName(), region, conformityAsg); + list.add(cluster); + updateExcludedConformityRules(cluster); + cluster.setOwnerEmail(getOwnerEmailForCluster(cluster)); + String prop = String.format("simianarmy.conformity.cluster.%s.optedOut", cluster.getName()); + if (cfg.getBoolOrElse(prop, false)) { + LOGGER.info(String.format("Cluster %s is opted out of Conformity Monkey.", cluster.getName())); + cluster.setOptOutOfConformity(true); + } else { + cluster.setOptOutOfConformity(false); + } + } + } + return list; + } + + /** + * Gets the owner email from the monkey configuration. + * @param cluster + * the cluster + * @return the owner email if it is defined in the configuration, null otherwise. + */ + @Override + public String getOwnerEmailForCluster(Cluster cluster) { + String prop = String.format("%s.%s.ownerEmail", NS, cluster.getName()); + String ownerEmail = cfg.getStr(prop); + if (ownerEmail == null) { + LOGGER.info(String.format("No owner email is found for cluster %s in configuration" + + "please set property %s for it.", cluster.getName(), prop)); + } else { + LOGGER.info(String.format("Found owner email %s for cluster %s in configuration.", + ownerEmail, cluster.getName())); + } + return ownerEmail; + } + + @Override + public void updateExcludedConformityRules(Cluster cluster) { + String prop = String.format("%s.%s.excludedRules", NS, cluster.getName()); + String excludedRules = cfg.getStr(prop); + if (StringUtils.isNotBlank(excludedRules)) { + LOGGER.info(String.format("Excluded rules for cluster %s are : %s", cluster.getName(), excludedRules)); + cluster.excludeRules(StringUtils.split(excludedRules, ",")); + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/BasicConformityEurekaClient.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/BasicConformityEurekaClient.java new file mode 100644 index 00000000..418cb4f2 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/BasicConformityEurekaClient.java @@ -0,0 +1,89 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.netflix.appinfo.InstanceInfo; +import com.netflix.discovery.DiscoveryClient; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Set; + +/** + * The class implementing a client to access Eureda for getting instance information that is used + * by Conformity Monkey. + */ +public class BasicConformityEurekaClient implements ConformityEurekaClient { + private static final Logger LOGGER = LoggerFactory.getLogger(BasicConformityEurekaClient.class); + + private final DiscoveryClient discoveryClient; + + /** + * Constructor. + * @param discoveryClient the client to access Discovery/Eureka service. + */ + public BasicConformityEurekaClient(DiscoveryClient discoveryClient) { + this.discoveryClient = discoveryClient; + } + + @Override + public boolean hasHealthCheckUrl(String region, String instanceId) { + List instanceInfos = discoveryClient.getInstancesById(instanceId); + for (InstanceInfo info : instanceInfos) { + Set healthCheckUrls = info.getHealthCheckUrls(); + if (healthCheckUrls != null && !healthCheckUrls.isEmpty()) { + return true; + } + } + return false; + } + + @Override + public boolean hasStatusUrl(String region, String instanceId) { + List instanceInfos = discoveryClient.getInstancesById(instanceId); + for (InstanceInfo info : instanceInfos) { + String statusPageUrl = info.getStatusPageUrl(); + if (!StringUtils.isEmpty(statusPageUrl)) { + return true; + } + } + return false; + } + + @Override + public boolean isHealthy(String region, String instanceId) { + List instanceInfos = discoveryClient.getInstancesById(instanceId); + if (instanceInfos.isEmpty()) { + LOGGER.info(String.format("Instance %s is not registered in Eureka in region %s.", instanceId, region)); + return false; + } else { + for (InstanceInfo info : instanceInfos) { + InstanceInfo.InstanceStatus status = info.getStatus(); + if (!status.equals(InstanceInfo.InstanceStatus.UP) + && !status.equals(InstanceInfo.InstanceStatus.STARTING)) { + LOGGER.info(String.format("Instance %s is not healthy in Eureka with status %s.", + instanceId, status.name())); + return false; + } + } + } + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/ConformityEurekaClient.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/ConformityEurekaClient.java new file mode 100644 index 00000000..bd79959e --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/ConformityEurekaClient.java @@ -0,0 +1,47 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +/** + * The interface for a client to access Eureka service to get the status of instances for Conformity Monkey. + */ +public interface ConformityEurekaClient { + /** + * Checks whether an instance has health check url in Eureka. + * @param region the region of the instance + * @param instanceId the instance id + * @return true if the instance has health check url in Eureka, false otherwise. + */ + boolean hasHealthCheckUrl(String region, String instanceId); + + /** + * Checks whether an instance has status url in Eureka. + * @param region the region of the instance + * @param instanceId the instance id + * @return true if the instance has status url in Eureka, false otherwise. + */ + boolean hasStatusUrl(String region, String instanceId); + + /** + * Checks whether an instance is healthy in Eureka. + * @param region the region of the instance + * @param instanceId the instance id + * @return true if the instance is healthy in Eureka, false otherwise. + */ + boolean isHealthy(String region, String instanceId); +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasHealthCheckUrl.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasHealthCheckUrl.java new file mode 100644 index 00000000..e204a224 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasHealthCheckUrl.java @@ -0,0 +1,80 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.google.common.collect.Lists; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +/** + * The class implementing a conformity rule that checks if all instances in a cluster has health check url + * in Discovery/Eureka. + */ +public class InstanceHasHealthCheckUrl implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceHasHealthCheckUrl.class); + + private static final String RULE_NAME = "InstanceHasHealthCheckUrl"; + private static final String REASON = "Health check url not defined"; + + private final ConformityEurekaClient conformityEurekaClient; + + /** + * Constructor. + * @param conformityEurekaClient + * the client to access the Discovery/Eureka service for checking the status of instances. + */ + public InstanceHasHealthCheckUrl(ConformityEurekaClient conformityEurekaClient) { + Validate.notNull(conformityEurekaClient); + this.conformityEurekaClient = conformityEurekaClient; + } + + @Override + public Conformity check(Cluster cluster) { + Collection failedComponents = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + if (asg.isSuspended()) { + continue; + } + for (String instance : asg.getInstances()) { + if (!conformityEurekaClient.hasHealthCheckUrl(cluster.getRegion(), instance)) { + LOGGER.info(String.format("Instance %s does not have health check url in discovery.", + instance)); + failedComponents.add(instance); + } + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return REASON; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasStatusUrl.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasStatusUrl.java new file mode 100644 index 00000000..2a91cb3d --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceHasStatusUrl.java @@ -0,0 +1,79 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.google.common.collect.Lists; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +/** + * The class implementing a conformity rule that checks if all instances in a cluster has status url. + */ +public class InstanceHasStatusUrl implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceHasStatusUrl.class); + + private static final String RULE_NAME = "InstanceHasStatusUrl"; + private static final String REASON = "Status url not defined"; + + private final ConformityEurekaClient conformityEurekaClient; + + /** + * Constructor. + * @param conformityEurekaClient + * the client to access the Discovery/Eureka service for checking the status of instances. + */ + public InstanceHasStatusUrl(ConformityEurekaClient conformityEurekaClient) { + Validate.notNull(conformityEurekaClient); + this.conformityEurekaClient = conformityEurekaClient; + } + + @Override + public Conformity check(Cluster cluster) { + Collection failedComponents = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + if (asg.isSuspended()) { + continue; + } + for (String instance : asg.getInstances()) { + if (!conformityEurekaClient.hasStatusUrl(cluster.getRegion(), instance)) { + LOGGER.info(String.format("Instance %s does not have a status page url in discovery.", + instance)); + failedComponents.add(instance); + } + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return REASON; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceInSecurityGroup.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceInSecurityGroup.java new file mode 100644 index 00000000..0bfbcbc3 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceInSecurityGroup.java @@ -0,0 +1,154 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.amazonaws.services.ec2.model.GroupIdentifier; +import com.amazonaws.services.ec2.model.Instance; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * The class implementing a conformity rule that checks whether or not all instances in a cluster are in + * specific security groups. + */ +public class InstanceInSecurityGroup implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceHasStatusUrl.class); + + private static final String RULE_NAME = "InstanceInSecurityGroup"; + private final String reason; + + private final Collection requiredSecurityGroupNames = Sets.newHashSet(); + + /** + * Constructor. + * @param requiredSecurityGroupNames + * The security group names that are required to have for every instance of a cluster. + */ + public InstanceInSecurityGroup(String... requiredSecurityGroupNames) { + Validate.notNull(requiredSecurityGroupNames); + for (String sgName : requiredSecurityGroupNames) { + Validate.notNull(sgName); + this.requiredSecurityGroupNames.add(sgName.trim()); + } + this.reason = String.format("Instances are not part of security groups (%s)", + StringUtils.join(this.requiredSecurityGroupNames, ",")); + } + + @Override + public Conformity check(Cluster cluster) { + List instanceIds = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + instanceIds.addAll(asg.getInstances()); + } + Collection failedComponents = Lists.newArrayList(); + if (instanceIds.size() != 0) { + Map> instanceIdToSecurityGroup = getInstanceSecurityGroups( + cluster.getRegion(), instanceIds.toArray(new String[instanceIds.size()])); + + for (Map.Entry> entry : instanceIdToSecurityGroup.entrySet()) { + String instanceId = entry.getKey(); + if (!checkSecurityGroups(entry.getValue())) { + LOGGER.info(String.format("Instance %s does not have all required security groups", instanceId)); + failedComponents.add(instanceId); + } + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return reason; + } + + /** + * Checks whether the collection of security group names are valid. The default implementation here is to check + * whether the security groups contain the required security groups. The method can be overriden for different + * rules. + * @param sgNames + * The collection of security group names + * @return + * true if the security group names are valid, false otherwise. + */ + protected boolean checkSecurityGroups(Collection sgNames) { + for (String requiredSg : requiredSecurityGroupNames) { + if (!sgNames.contains(requiredSg)) { + LOGGER.info(String.format("Required security group %s is not found.", requiredSg)); + return false; + } + } + return true; + } + + /** + * Gets the security groups for a list of instance ids of the same region. The default implementation + * is using an AWS client. The method can be overriden in subclasses to get the security groups differently. + * @param region + * the region of the instances + * @param instanceIds + * the instance ids, all instances should be in the same region. + * @return + * the map from instance id to the list of security group names the instance has + */ + protected Map> getInstanceSecurityGroups(String region, String... instanceIds) { + Map> result = Maps.newHashMap(); + if (instanceIds == null || instanceIds.length == 0) { + return result; + } + AWSClient awsClient = new AWSClient(region); + for (Instance instance : awsClient.describeInstances(instanceIds)) { + // Ignore instances that are in VPC + if (StringUtils.isNotEmpty(instance.getVpcId())) { + LOGGER.info(String.format("Instance %s is in VPC and is ignored.", instance.getInstanceId())); + continue; + } + + if (!"running".equals(instance.getState().getName())) { + LOGGER.info(String.format("Instance %s is not running, state is %s.", + instance.getInstanceId(), instance.getState().getName())); + continue; + } + + List sgs = Lists.newArrayList(); + for (GroupIdentifier groupId : instance.getSecurityGroups()) { + sgs.add(groupId.getGroupName()); + } + result.put(instance.getInstanceId(), sgs); + } + return result; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceIsHealthyInEureka.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceIsHealthyInEureka.java new file mode 100644 index 00000000..44a75018 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceIsHealthyInEureka.java @@ -0,0 +1,80 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.google.common.collect.Lists; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +/** + * The class implements a conformity rule to check if all instances in the cluster are healthy in Discovery. + */ +public class InstanceIsHealthyInEureka implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceIsHealthyInEureka.class); + + private static final String RULE_NAME = "InstanceIsHealthyInEureka"; + private static final String REASON = "Instances are not 'UP' in Eureka."; + + private final ConformityEurekaClient conformityEurekaClient; + + /** + * Constructor. + * @param conformityEurekaClient + * the client to access the Discovery/Eureka service for checking the status of instances. + */ + public InstanceIsHealthyInEureka(ConformityEurekaClient conformityEurekaClient) { + Validate.notNull(conformityEurekaClient); + this.conformityEurekaClient = conformityEurekaClient; + } + + @Override + public Conformity check(Cluster cluster) { + Collection failedComponents = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + // ignore suspended ASGs + if (asg.isSuspended()) { + LOGGER.info(String.format("ASG %s is supspended, ingore.", asg.getName())); + continue; + } + for (String instance : asg.getInstances()) { + if (!conformityEurekaClient.isHealthy(cluster.getRegion(), instance)) { + LOGGER.info(String.format("Instance %s is not healthy in Eureka.", instance)); + failedComponents.add(instance); + } + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return REASON; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceTooOld.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceTooOld.java new file mode 100644 index 00000000..aaa0fa11 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/InstanceTooOld.java @@ -0,0 +1,118 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.amazonaws.services.ec2.model.Instance; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * The class implementing a conformity rule that checks if there are instances that are older than certain days. + * Instances are not considered to be permanent in the cloud, so sometimes having too old instances could indicate + * potential issues. + */ +public class InstanceTooOld implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceHasStatusUrl.class); + + private static final String RULE_NAME = "InstanceTooOld"; + private final String reason; + private final int instanceAgeThreshold; + + /** + * Constructor. + * @param instanceAgeThreshold + * The age in days that makes an instance be considered too old. + */ + public InstanceTooOld(int instanceAgeThreshold) { + Validate.isTrue(instanceAgeThreshold > 0); + this.instanceAgeThreshold = instanceAgeThreshold; + this.reason = String.format("Instances are older than %d days", instanceAgeThreshold); + } + + @Override + public Conformity check(Cluster cluster) { + List instanceIds = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + instanceIds.addAll(asg.getInstances()); + } + Map instanceIdToLaunchTime = getInstanceLaunchTimes( + cluster.getRegion(), instanceIds.toArray(new String[instanceIds.size()])); + + Collection failedComponents = Lists.newArrayList(); + long creationTimeThreshold = DateTime.now().minusDays(instanceAgeThreshold).getMillis(); + for (Map.Entry entry : instanceIdToLaunchTime.entrySet()) { + String instanceId = entry.getKey(); + if (creationTimeThreshold > entry.getValue()) { + LOGGER.info(String.format("Instance %s was created more than %d days ago", + instanceId, instanceAgeThreshold)); + failedComponents.add(instanceId); + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return reason; + } + + /** + * Gets the launch time (in milliseconds) for a list of instance ids of the same region. The default + * implementation is using an AWS client. The method can be overriden in subclasses to get the instance + * launch times differently. + * @param region + * the region of the instances + * @param instanceIds + * the instance ids, all instances should be in the same region. + * @return + * the map from instance id to the launch time in milliseconds + */ + protected Map getInstanceLaunchTimes(String region, String... instanceIds) { + Map result = Maps.newHashMap(); + if (instanceIds == null || instanceIds.length == 0) { + return result; + } + AWSClient awsClient = new AWSClient(region); + for (Instance instance : awsClient.describeInstances(instanceIds)) { + if (instance.getLaunchTime() != null) { + result.put(instance.getInstanceId(), instance.getLaunchTime().getTime()); + } else { + LOGGER.warn(String.format("No launch time found for instance %s", instance.getInstanceId())); + } + } + return result; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/rule/SameZonesInElbAndAsg.java b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/SameZonesInElbAndAsg.java new file mode 100644 index 00000000..07c85354 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/rule/SameZonesInElbAndAsg.java @@ -0,0 +1,157 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity.rule; + +import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.AutoScalingGroup; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The class implementing a conformity rule that checks if the zones in ELB and ASG are the same. + */ +public class SameZonesInElbAndAsg implements ConformityRule { + private static final Logger LOGGER = LoggerFactory.getLogger(InstanceHasStatusUrl.class); + + private final Map regionToAwsClient = Maps.newHashMap(); + private static final String RULE_NAME = "SameZonesInElbAndAsg"; + private static final String REASON = "Avaliability zones of ELB and ASG are different"; + + @Override + public Conformity check(Cluster cluster) { + List asgNames = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + asgNames.add(asg.getName()); + } + Collection failedComponents = Lists.newArrayList(); + for (AutoScalingGroup asg : cluster.getAutoScalingGroups()) { + List asgZones = getAvailabilityZonesForAsg(cluster.getRegion(), asg.getName()); + for (String lbName : getLoadBalancerNamesForAsg(cluster.getRegion(), asg.getName())) { + List lbZones = getAvailabilityZonesForLoadBalancer(cluster.getRegion(), lbName); + if (!haveSameZones(asgZones, lbZones)) { + LOGGER.info(String.format("ASG %s and ELB %s do not have the same availability zones", + asgZones, lbZones)); + failedComponents.add(lbName); + } + } + } + return new Conformity(getName(), failedComponents); + } + + @Override + public String getName() { + return RULE_NAME; + } + + @Override + public String getNonconformingReason() { + return REASON; + } + + /** + * Gets the load blancer names of an ASG. Can be overriden in subclasses. + * @param region the region + * @param asgName the ASG name + * @return the list of load balancer names + */ + protected List getLoadBalancerNamesForAsg(String region, String asgName) { + List asgs = + getAwsClient(region).describeAutoScalingGroups(asgName); + if (asgs.isEmpty()) { + LOGGER.error(String.format("Not found ASG with name %s", asgName)); + return Collections.emptyList(); + } else { + return asgs.get(0).getLoadBalancerNames(); + } + } + + /** + * Gets the list of availability zones for an ASG. Can be overriden in subclasses. + * @param region the region + * @param asgName the ASG name. + * @return the list of the availability zones that the ASG has. + */ + protected List getAvailabilityZonesForAsg(String region, String asgName) { + List asgs = + getAwsClient(region).describeAutoScalingGroups(asgName); + if (asgs.isEmpty()) { + LOGGER.error(String.format("Not found ASG with name %s", asgName)); + return null; + } else { + return asgs.get(0).getAvailabilityZones(); + } + } + + /** + * Gets the list of availability zones for a load balancer. Can be overriden in subclasses. + * @param region the region + * @param lbName the load balancer name. + * @return the list of the availability zones that the load balancer has. + */ + protected List getAvailabilityZonesForLoadBalancer(String region, String lbName) { + List lbs = + getAwsClient(region).describeElasticLoadBalancers(lbName); + if (lbs.isEmpty()) { + LOGGER.error(String.format("Not found load balancer with name %s", lbName)); + return null; + } else { + return lbs.get(0).getAvailabilityZones(); + } + } + + private AWSClient getAwsClient(String region) { + AWSClient awsClient = regionToAwsClient.get(region); + if (awsClient == null) { + awsClient = new AWSClient(region); + regionToAwsClient.put(region, awsClient); + } + return awsClient; + } + + + private boolean haveSameZones(List zones1, List zones2) { + if (zones1 == null || zones2 == null) { + return true; + } + if (zones1.size() != zones1.size()) { + return false; + } + for (String zone : zones1) { + if (!zones2.contains(zone)) { + return false; + } + } + for (String zone : zones2) { + if (!zones1.contains(zone)) { + return false; + } + } + return true; + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaEBSVolumeJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaEBSVolumeJanitorCrawler.java index 755163ec..02e3500b 100644 --- a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaEBSVolumeJanitorCrawler.java +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaEBSVolumeJanitorCrawler.java @@ -39,7 +39,7 @@ public class EddaEBSVolumeJanitorCrawler implements JanitorCrawler { private static final int BATCH_SIZE = 50; // The value below specifies how many days we want to look back in Edda to find the owner of old instances. - // In case of Edda keeps too much history data, without a reasonal data range, the query may fail. + // In case of Edda keeps too much history data, without a reasonable date range, the query may fail. private static final int LOOKBACK_DAYS = 90; /** diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java index 8e48ef73..99e1f5a0 100644 --- a/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/crawler/edda/EddaImageJanitorCrawler.java @@ -219,7 +219,8 @@ private void refreshIdToCreationTime() { } if (jsonNode == null || !jsonNode.isArray()) { - throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", url, jsonNode)); + throw new RuntimeException(String.format("Failed to get valid document from %s, got: %s", + url, jsonNode)); } for (Iterator it = jsonNode.getElements(); it.hasNext();) { diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java b/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java index 7b0aefe6..acb1725b 100644 --- a/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java +++ b/src/main/java/com/netflix/simianarmy/basic/BasicMonkeyServer.java @@ -24,6 +24,8 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; +import com.netflix.simianarmy.basic.conformity.BasicConformityMonkey; +import com.netflix.simianarmy.basic.conformity.BasicConformityMonkeyContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +56,8 @@ public void addMonkeysToRun() { RUNNER.replaceMonkey(VolumeTaggingMonkey.class, BasicVolumeTaggingMonkeyContext.class); LOGGER.info("Adding Janitor Monkey."); RUNNER.replaceMonkey(BasicJanitorMonkey.class, BasicJanitorMonkeyContext.class); + LOGGER.info("Adding Conformity Monkey."); + RUNNER.replaceMonkey(BasicConformityMonkey.class, BasicConformityMonkeyContext.class); } /** diff --git a/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityEmailBuilder.java b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityEmailBuilder.java new file mode 100644 index 00000000..b4250ab3 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityEmailBuilder.java @@ -0,0 +1,137 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.conformity; + +import com.google.common.collect.Maps; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityEmailBuilder; +import com.netflix.simianarmy.conformity.ConformityRule; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; + +/** The basic implementation of the email builder for Conformity monkey. */ +public class BasicConformityEmailBuilder extends ConformityEmailBuilder { + private static final String[] TABLE_COLUMNS = {"Cluster", "Region", "Rule", "Failed Components"}; + private static final String AHREF_TEMPLATE = "%s"; + private static final Logger LOGGER = LoggerFactory.getLogger(BasicConformityEmailBuilder.class); + + private Map> emailToClusters; + private final Map idToRule = Maps.newHashMap(); + + @Override + public void setEmailToClusters(Map> clustersByEmail, Collection rules) { + Validate.notNull(clustersByEmail); + Validate.notNull(rules); + this.emailToClusters = clustersByEmail; + idToRule.clear(); + for (ConformityRule rule : rules) { + idToRule.put(rule.getName(), rule); + } + } + + @Override + protected String getHeader() { + StringBuilder header = new StringBuilder(); + header.append("

Conformity Report

"); + header.append("The following is a list of failed conformity rules for your cluster(s).
"); + return header.toString(); + } + + @Override + protected String getEntryTable(String emailAddress) { + StringBuilder table = new StringBuilder(); + table.append(getHtmlTableHeader(getTableColumns())); + for (Cluster cluster : emailToClusters.get(emailAddress)) { + for (Conformity conformity : cluster.getConformties()) { + if (!conformity.getFailedComponents().isEmpty()) { + table.append(getClusterRow(cluster, conformity)); + } + } + } + table.append(""); + return table.toString(); + } + + @Override + protected String getFooter() { + return "
Conformity Monkey wiki: https://github.com/Netflix/SimianArmy/wiki
"; + } + + /** + * Gets the url to view the details of the cluster. + * @param cluster the cluster + * @return the url to view/edit the cluster. + */ + protected String getClusterUrl(Cluster cluster) { + return null; + } + + /** + * Gets the string when displaying the cluster, e.g. the id. + * @param cluster the cluster to display + * @return the string to represent the cluster + */ + protected String getClusterDisplay(Cluster cluster) { + return cluster.getName(); + } + + /** Gets the table columns for the table in the email. + * + * @return the array of column names + */ + protected String[] getTableColumns() { + return TABLE_COLUMNS; + } + + /** + * Gets the row for a cluster and a failed conformity check in the table in the email body. + * @param cluster the cluster to display + * @param conformity the failed conformity check + * @return the table row in the email body + */ + protected String getClusterRow(Cluster cluster, Conformity conformity) { + StringBuilder message = new StringBuilder(); + message.append(""); + String clusterUrl = getClusterUrl(cluster); + if (!StringUtils.isEmpty(clusterUrl)) { + message.append(getHtmlCell(String.format(AHREF_TEMPLATE, clusterUrl, getClusterDisplay(cluster)))); + } else { + message.append(getHtmlCell(getClusterDisplay(cluster))); + } + message.append(getHtmlCell(cluster.getRegion())); + ConformityRule rule = idToRule.get(conformity.getRuleId()); + String ruleDesc; + if (rule == null) { + LOGGER.warn(String.format("Not found rule with name %s", conformity.getRuleId())); + ruleDesc = conformity.getRuleId(); + } else { + ruleDesc = rule.getNonconformingReason(); + } + message.append(getHtmlCell(ruleDesc)); + message.append(getHtmlCell(StringUtils.join(conformity.getFailedComponents(), ","))); + message.append(""); + return message.toString(); + } + +} diff --git a/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkey.java b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkey.java new file mode 100644 index 00000000..b46349e1 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkey.java @@ -0,0 +1,244 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.basic.conformity; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.MonkeyConfiguration; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.ClusterCrawler; +import com.netflix.simianarmy.conformity.ConformityClusterTracker; +import com.netflix.simianarmy.conformity.ConformityEmailNotifier; +import com.netflix.simianarmy.conformity.ConformityMonkey; +import com.netflix.simianarmy.conformity.ConformityRuleEngine; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** The basic implementation of Conformity Monkey. */ +public class BasicConformityMonkey extends ConformityMonkey { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicConformityMonkey.class); + + /** The Constant NS. */ + private static final String NS = "simianarmy.conformity."; + + /** The cfg. */ + private final MonkeyConfiguration cfg; + + private final ClusterCrawler crawler; + + private final ConformityEmailNotifier emailNotifier; + + private final Collection regions = Lists.newArrayList(); + + private final ConformityClusterTracker clusterTracker; + + private final MonkeyCalendar calendar; + + private final ConformityRuleEngine ruleEngine; + + /** Flag to indicate whether the monkey is leashed. */ + private boolean leashed; + + /** + * Clusters that are not conforming in the last check. + */ + private final Map> nonconformingClusters = Maps.newHashMap(); + + /** + * Clusters that are conforming in the last check. + */ + private final Map> conformingClusters = Maps.newHashMap(); + + /** + * Clusters that the monkey failed to check for some reason. + */ + private final Map> failedClusters = Maps.newHashMap(); + + /** + * Clusters that do not exist in the cloud anymore. + */ + private final Map> nonexistentClusters = Maps.newHashMap(); + + /** + * Instantiates a new basic conformity monkey. + * + * @param ctx + * the ctx + */ + public BasicConformityMonkey(Context ctx) { + super(ctx); + cfg = ctx.configuration(); + crawler = ctx.clusterCrawler(); + ruleEngine = ctx.ruleEngine(); + emailNotifier = ctx.emailNotifier(); + for (String region : ctx.regions()) { + regions.add(region); + } + clusterTracker = ctx.clusterTracker(); + calendar = ctx.calendar(); + leashed = ctx.isLeashed(); + } + + /** {@inheritDoc} */ + @Override + public void doMonkeyBusiness() { + cfg.reload(); + context().resetEventReport(); + + if (!isConformityMonkeyEnabled()) { + return; + } else { + nonconformingClusters.clear(); + conformingClusters.clear(); + failedClusters.clear(); + nonexistentClusters.clear(); + + List clusters = crawler.clusters(); + Set existingClusterNames = Sets.newHashSet(); + for (Cluster cluster : clusters) { + existingClusterNames.add(cluster.getName()); + } + List trackedClusters = clusterTracker.getAllClusters(regions.toArray(new String[regions.size()])); + for (Cluster trackedCluster : trackedClusters) { + if (!existingClusterNames.contains(trackedCluster.getName())) { + addCluster(nonexistentClusters, trackedCluster); + } + } + for (String region : regions) { + Collection toDelete = nonexistentClusters.get(region); + if (toDelete != null) { + clusterTracker.deleteClusters(toDelete.toArray(new Cluster[toDelete.size()])); + } + } + + LOGGER.info(String.format("Performing conformity check for %d crawled clusters.", clusters.size())); + Date now = calendar.now().getTime(); + for (Cluster cluster : clusters) { + boolean conforming = true; + try { + conforming = ruleEngine.check(cluster); + } catch (Exception e) { + LOGGER.error(String.format("Failed to perform conformity check for cluster %s", cluster.getName()), + e); + addCluster(failedClusters, cluster); + continue; + } + cluster.setUpdateTime(now); + cluster.setConforming(conforming); + if (conforming) { + LOGGER.info(String.format("Cluster %s is conforming", cluster.getName())); + addCluster(conformingClusters, cluster); + } else { + LOGGER.info(String.format("Cluster %s is not conforming", cluster.getName())); + addCluster(nonconformingClusters, cluster); + } + if (!leashed) { + LOGGER.info(String.format("Saving cluster %s", cluster.getName())); + clusterTracker.addOrUpdate(cluster); + } else { + LOGGER.info(String.format( + "The conformity monkey is leashed, no data change is made for cluster %s.", + cluster.getName())); + } + } + if (!leashed) { + emailNotifier.sendNotifications(); + } else { + LOGGER.info("Conformity monkey is leashed, no notification is sent."); + } + sendConformitySummaryEmail(); + } + } + + private static void addCluster(Map> map, Cluster cluster) { + Collection clusters = map.get(cluster.getRegion()); + if (clusters == null) { + clusters = Lists.newArrayList(); + map.put(cluster.getRegion(), clusters); + } + clusters.add(cluster); + } + + /** + * Send a summary email with about the last run of the conformity monkey. + */ + protected void sendConformitySummaryEmail() { + String summaryEmailTarget = cfg.getStr(NS + "summaryEmail.to"); + if (!StringUtils.isEmpty(summaryEmailTarget)) { + if (!emailNotifier.isValidEmail(summaryEmailTarget)) { + LOGGER.error(String.format("The email target address '%s' for Conformity summary email is invalid", + summaryEmailTarget)); + return; + } + StringBuilder message = new StringBuilder(); + for (String region : regions) { + appendSummary(message, "nonconforming", nonconformingClusters, region, true); + appendSummary(message, "failed to check", failedClusters, region, true); + appendSummary(message, "nonexistent", nonexistentClusters, region, true); + appendSummary(message, "conforming", conformingClusters, region, false); + } + String subject = getSummaryEmailSubject(); + emailNotifier.sendEmail(summaryEmailTarget, subject, message.toString()); + } + } + + private void appendSummary(StringBuilder message, String summaryName, + Map> regionToClusters, String region, boolean showDetails) { + Collection clusters = regionToClusters.get(region); + if (clusters == null) { + clusters = Lists.newArrayList(); + } + message.append(String.format("Total %s clusters = %d in region %s
", + summaryName, clusters.size(), region)); + if (showDetails) { + List clusterNames = Lists.newArrayList(); + for (Cluster cluster : clusters) { + clusterNames.add(cluster.getName()); + } + message.append(String.format("List: %s

", StringUtils.join(clusterNames, ","))); + } + } + + /** + * Gets the summary email subject for the last run of conformity monkey. + * @return the subject of the summary email + */ + protected String getSummaryEmailSubject() { + return String.format("Conformity monkey execution summary (%s)", StringUtils.join(regions, ",")); + } + + private boolean isConformityMonkeyEnabled() { + String prop = NS + "enabled"; + if (cfg.getBoolOrElse(prop, true)) { + return true; + } + LOGGER.info("Conformity Monkey is disabled, set {}=true", prop); + return false; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java new file mode 100644 index 00000000..161b2ccf --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java @@ -0,0 +1,241 @@ +/* + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// CHECKSTYLE IGNORE MagicNumberCheck +package com.netflix.simianarmy.basic.conformity; + +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.netflix.discovery.DiscoveryClient; +import com.netflix.discovery.DiscoveryManager; +import com.netflix.simianarmy.aws.conformity.SimpleDBConformityClusterTracker; +import com.netflix.simianarmy.aws.conformity.crawler.AWSClusterCrawler; +import com.netflix.simianarmy.aws.conformity.rule.BasicConformityEurekaClient; +import com.netflix.simianarmy.aws.conformity.rule.ConformityEurekaClient; +import com.netflix.simianarmy.aws.conformity.rule.InstanceHasHealthCheckUrl; +import com.netflix.simianarmy.aws.conformity.rule.InstanceHasStatusUrl; +import com.netflix.simianarmy.aws.conformity.rule.InstanceInSecurityGroup; +import com.netflix.simianarmy.aws.conformity.rule.InstanceIsHealthyInEureka; +import com.netflix.simianarmy.aws.conformity.rule.InstanceTooOld; +import com.netflix.simianarmy.aws.conformity.rule.SameZonesInElbAndAsg; +import com.netflix.simianarmy.basic.BasicSimianArmyContext; +import com.netflix.simianarmy.client.aws.AWSClient; +import com.netflix.simianarmy.conformity.ClusterCrawler; +import com.netflix.simianarmy.conformity.ConformityClusterTracker; +import com.netflix.simianarmy.conformity.ConformityEmailBuilder; +import com.netflix.simianarmy.conformity.ConformityEmailNotifier; +import com.netflix.simianarmy.conformity.ConformityMonkey; +import com.netflix.simianarmy.conformity.ConformityRule; +import com.netflix.simianarmy.conformity.ConformityRuleEngine; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; + +/** + * The basic implementation of the context class for Conformity monkey. + */ +public class BasicConformityMonkeyContext extends BasicSimianArmyContext implements ConformityMonkey.Context { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(BasicConformityMonkeyContext.class); + + /** The email notifier. */ + private final ConformityEmailNotifier emailNotifier; + + private final ConformityClusterTracker clusterTracker; + + private final Collection regions; + + private final ClusterCrawler clusterCrawler; + + private final AmazonSimpleEmailServiceClient sesClient; + + private final ConformityEmailBuilder conformityEmailBuilder; + + private final String defaultEmail; + + private final String[] ccEmails; + + private final String sourceEmail; + + private final ConformityRuleEngine ruleEngine; + + private final boolean leashed; + + private final Map regionToAwsClient = Maps.newHashMap(); + + /** + * The constructor. + */ + public BasicConformityMonkeyContext() { + super("simianarmy.properties", "client.properties", "conformity.properties"); + regions = Lists.newArrayList(region()); + + // By default, the monkey is leashed + leashed = configuration().getBoolOrElse("simianarmy.conformity.leashed", true); + + LOGGER.info(String.format("Conformity Monkey is running in: %s", regions)); + + String sdbDomain = configuration().getStrOrElse("simianarmy.conformity.sdb.domain", "SIMIAN_ARMY"); + + clusterTracker = new SimpleDBConformityClusterTracker(awsClient(), sdbDomain); + + ruleEngine = new ConformityRuleEngine(); + boolean eurekaEnabled = configuration().getBoolOrElse("simianarmy.conformity.Eureka.enabled", false); + + if (eurekaEnabled) { + LOGGER.info("Initializing Discovery client."); + DiscoveryClient discoveryClient = DiscoveryManager.getInstance().getDiscoveryClient(); + ConformityEurekaClient conformityEurekaClient = new BasicConformityEurekaClient(discoveryClient); + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.InstanceIsHealthyInEureka.enabled", false)) { + ruleEngine.addRule(new InstanceIsHealthyInEureka(conformityEurekaClient)); + } + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.InstanceHasHealthCheckUrl.enabled", false)) { + ruleEngine.addRule(new InstanceHasHealthCheckUrl(conformityEurekaClient)); + } + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.InstanceHasStatusUrl.enabled", false)) { + ruleEngine.addRule(new InstanceHasStatusUrl(conformityEurekaClient)); + } + } else { + LOGGER.info("Discovery/Eureka is not enabled, the conformity rules that need Eureka are not added."); + } + + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.InstanceInSecurityGroup.enabled", false)) { + String requiredSecurityGroups = configuration().getStr( + "simianarmy.conformity.rule.InstanceInSecurityGroup.requiredSecurityGroups"); + if (!StringUtils.isBlank(requiredSecurityGroups)) { + ruleEngine.addRule(new InstanceInSecurityGroup(StringUtils.split(requiredSecurityGroups, ","))); + } else { + LOGGER.info("No required security groups is specified, " + + "the conformity rule InstanceInSecurityGroup is ignored."); + } + } + + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.InstanceTooOld.enabled", false)) { + ruleEngine.addRule(new InstanceTooOld((int) configuration().getNumOrElse( + "simianarmy.conformity.rule.InstanceTooOld.instanceAgeThreshold", 180))); + } + + if (configuration().getBoolOrElse( + "simianarmy.conformity.rule.SameZonesInElbAndAsg.enabled", false)) { + ruleEngine().addRule(new SameZonesInElbAndAsg()); + } + + regionToAwsClient.put(region(), new AWSClient(region())); + clusterCrawler = new AWSClusterCrawler(regionToAwsClient, configuration()); + sesClient = new AmazonSimpleEmailServiceClient(); + defaultEmail = configuration().getStrOrElse("simianarmy.conformity.notification.defaultEmail", null); + ccEmails = StringUtils.split( + configuration().getStrOrElse("simianarmy.conformity.notification.ccEmails", ""), ","); + sourceEmail = configuration().getStrOrElse("simianarmy.conformity.notification.sourceEmail", null); + conformityEmailBuilder = new BasicConformityEmailBuilder(); + emailNotifier = new ConformityEmailNotifier(getConformityEmailNotifierContext()); + } + + public ConformityEmailNotifier.Context getConformityEmailNotifierContext() { + return new ConformityEmailNotifier.Context() { + @Override + public AmazonSimpleEmailServiceClient sesClient() { + return sesClient; + } + + @Override + public int opentHour() { + return (int) configuration().getNumOrElse("simianarmy.conformity.notification.openHour", 0); + } + + @Override + public int closeHour() { + return (int) configuration().getNumOrElse("simianarmy.conformity.notification.closeHour", 24); + } + + @Override + public String defaultEmail() { + return defaultEmail; + } + + @Override + public Collection regions() { + return regions; + } + + @Override + public ConformityClusterTracker clusterTracker() { + return clusterTracker; + } + + @Override + public ConformityEmailBuilder emailBuilder() { + return conformityEmailBuilder; + } + + @Override + public String[] ccEmails() { + return ccEmails; + } + + @Override + public Collection rules() { + return ruleEngine.rules(); + } + + @Override + public String sourceEmail() { + return sourceEmail; + } + }; + } + + @Override + public ClusterCrawler clusterCrawler() { + return clusterCrawler; + } + + @Override + public ConformityRuleEngine ruleEngine() { + return ruleEngine; + } + + /** {@inheritDoc} */ + @Override + public ConformityEmailNotifier emailNotifier() { + return emailNotifier; + } + + @Override + public Collection regions() { + return regions; + } + + @Override + public boolean isLeashed() { + return leashed; + } + + @Override + public ConformityClusterTracker clusterTracker() { + return clusterTracker; + } +} diff --git a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java index 971483bd..0ad4f8df 100644 --- a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java +++ b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java @@ -52,6 +52,10 @@ import com.amazonaws.services.ec2.model.Tag; import com.amazonaws.services.ec2.model.TerminateInstancesRequest; import com.amazonaws.services.ec2.model.Volume; +import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancingClient; +import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersRequest; +import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersResult; +import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription; import com.amazonaws.services.simpledb.AmazonSimpleDB; import com.amazonaws.services.simpledb.AmazonSimpleDBClient; import com.netflix.simianarmy.CloudClient; @@ -170,6 +174,22 @@ protected AmazonAutoScalingClient asgClient() { return client; } + /** + * Amazon ELB client. Abstracted to aid testing. + * + * @return the Amazon ELB client + */ + protected AmazonElasticLoadBalancingClient elbClient() { + AmazonElasticLoadBalancingClient client; + if (awsCredentialsProvider == null) { + client = new AmazonElasticLoadBalancingClient(); + } else { + client = new AmazonElasticLoadBalancingClient(awsCredentialsProvider); + } + client.setEndpoint("elasticloadbalancing." + region + ".amazonaws.com"); + return client; + } + /** * Amazon SimpleDB client. * @@ -234,6 +254,28 @@ public List describeAutoScalingGroups(String... names) { return asgs; } + /** + * Describe a set of specific ELBs. + * + * @param names the ELB names + * @return the ELBs + */ + public List describeElasticLoadBalancers(String... names) { + if (names == null || names.length == 0) { + LOGGER.info(String.format("Getting all ELBs in region %s.", region)); + } else { + LOGGER.info(String.format("Getting ELBs for %d names in region %s.", names.length, region)); + } + + AmazonElasticLoadBalancingClient elbClient = elbClient(); + DescribeLoadBalancersRequest request = new DescribeLoadBalancersRequest().withLoadBalancerNames(names); + DescribeLoadBalancersResult result = elbClient.describeLoadBalancers(request); + List elbs = result.getLoadBalancerDescriptions(); + LOGGER.info(String.format("Got %d ELBs in region %s.", elbs.size(), region)); + return elbs; + } + + /** * Describe a set of specific auto-scaling instances. * diff --git a/src/main/java/com/netflix/simianarmy/conformity/AutoScalingGroup.java b/src/main/java/com/netflix/simianarmy/conformity/AutoScalingGroup.java new file mode 100644 index 00000000..cc5d0184 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/AutoScalingGroup.java @@ -0,0 +1,83 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.Validate; + +import java.util.Collection; +import java.util.Collections; + +/** + * The class implementing the auto scaling groups. + */ +public class AutoScalingGroup { + private final String name; + private final Collection instances = Lists.newArrayList(); + private boolean isSuspended; + + /** + * Constructor. + * @param name + * the name of the auto scaling group + * @param instances + * the instance ids in the auto scaling group + */ + public AutoScalingGroup(String name, String... instances) { + Validate.notNull(instances); + this.name = name; + for (String instance : instances) { + this.instances.add(instance); + } + this.isSuspended = false; + } + + /** + * Gets the name of the auto scaling group. + * @return + * the name of the auto scaling group + */ + public String getName() { + return name; + } + + /** + * * Gets the instances of the auto scaling group. + * @return + * the instances of the auto scaling group + */ + public Collection getInstances() { + return Collections.unmodifiableCollection(instances); + } + + /** + * Gets the flag to indicate whether the ASG is suspended. + * @return true if the ASG is suspended, false otherwise + */ + public boolean isSuspended() { + return isSuspended; + } + + /** + * Sets the flag to indicate whether the ASG is suspended. + * @param suspended true if the ASG is suspended, false otherwise + */ + public void setSuspended(boolean suspended) { + isSuspended = suspended; + } +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/Cluster.java b/src/main/java/com/netflix/simianarmy/conformity/Cluster.java new file mode 100644 index 00000000..e823ca1e --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/Cluster.java @@ -0,0 +1,297 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * The class implementing clusters. Cluster is the basic unit of conformity check. It can be a single ASG or + * a group of ASGs that belong to the same application, for example, a cluster in the Asgard deployment system. + */ +public class Cluster { + private static final String OWNER_EMAIL = "ownerEmail"; + private static final String CLUSTER = "cluster"; + private static final String REGION = "region"; + private static final String IS_CONFORMING = "isConforming"; + private static final String IS_OPTEDOUT = "isOptedOut"; + private static final String UPDATE_TIMESTAMP = "updateTimestamp"; + private static final String EXCLUDED_RULES = "excludedRules"; + private static final String CONFORMITY_RULES = "conformityRules"; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); + + private final String name; + private final Collection autoScalingGroups = Lists.newArrayList(); + private final String region; + private String ownerEmail; + private Date updateTime; + private final Map conformities = Maps.newHashMap(); + private final Collection excludedConformityRules = Sets.newHashSet(); + private boolean isConforming; + private boolean isOptOutOfConformity; + + /** + * Constructor. + * @param name + * the name of the cluster + * @param autoScalingGroups + * the auto scaling groups in the cluster + */ + public Cluster(String name, String region, AutoScalingGroup... autoScalingGroups) { + Validate.notNull(name); + Validate.notNull(region); + Validate.notNull(autoScalingGroups); + this.name = name; + this.region = region; + for (AutoScalingGroup asg : autoScalingGroups) { + this.autoScalingGroups.add(asg); + } + } + + /** + * Gets the name of the cluster. + * @return + * the name of the cluster + */ + public String getName() { + return name; + } + + /** + * Gets the region of the cluster. + * @return + * the region of the cluster + */ + public String getRegion() { + return region; + } + + /** + * * Gets the auto scaling groups of the auto scaling group. + * @return + * the auto scaling groups in the cluster + */ + public Collection getAutoScalingGroups() { + return Collections.unmodifiableCollection(autoScalingGroups); + } + + /** + * Gets the owner email of the cluster. + * @return + * the owner email of the cluster + */ + public String getOwnerEmail() { + return ownerEmail; + } + + /** + * Sets the owner email of the cluster. + * @param ownerEmail + * the owner email of the cluster + */ + public void setOwnerEmail(String ownerEmail) { + this.ownerEmail = ownerEmail; + } + + /** + * Gets the update time of the cluster. + * @return + * the update time of the cluster + */ + public Date getUpdateTime() { + return new Date(updateTime.getTime()); + } + + /** + * Sets the update time of the cluster. + * @param updateTime + * the update time of the cluster + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = new Date(updateTime.getTime()); + } + + /** + * Gets all conformity check information of the cluster. + * @return + * all conformity check information of the cluster + */ + public Collection getConformties() { + return conformities.values(); + } + + /** + * Gets the conformity information for a conformity rule. + * @param rule + * the conformity rule + * @return + * the conformity for the rule + */ + public Conformity getConformity(ConformityRule rule) { + Validate.notNull(rule); + return conformities.get(rule.getName()); + } + + /** + * Updates the cluster with a new conformity check result. + * @param conformity + * the conformity to update + * @return + * the cluster itself + * + */ + public Cluster updateConformity(Conformity conformity) { + Validate.notNull(conformity); + conformities.put(conformity.getRuleId(), conformity); + return this; + } + + /** + * Clears the conformity check results. + */ + public void clearConformities() { + conformities.clear(); + } + + /** + * Gets the boolean flag to indicate whether the cluster is conforming to + * all non-excluded conformity rules. + * @return + * true if the cluster is conforming against all non-excluded rules, + * false otherwise + */ + public boolean isConforming() { + return isConforming; + } + + /** + * Sets the boolean flag to indicate whether the cluster is conforming to + * all non-excluded conformity rules. + * @param conforming + * true if the cluster is conforming against all non-excluded rules, + * false otherwise + */ + public void setConforming(boolean conforming) { + isConforming = conforming; + } + + /** + * Gets names of all excluded conformity rules for this cluster. + * @return + * names of all excluded conformity rules for this cluster + */ + public Collection getExcludedRules() { + return Collections.unmodifiableCollection(excludedConformityRules); + } + + /** + * Excludes rules for the cluster. + * @param ruleIds + * the rule ids to exclude + * @return + * the cluster itself + */ + public Cluster excludeRules(String... ruleIds) { + Validate.notNull(ruleIds); + for (String ruleId : ruleIds) { + Validate.notNull(ruleId); + excludedConformityRules.add(ruleId.trim()); + } + return this; + } + + /** + * Gets the flag to indicate whether the cluster is opted out of Conformity monkey. + * @return true if the cluster is not handled by Conformity monkey, false otherwise + */ + public boolean isOptOutOfConformity() { + return isOptOutOfConformity; + } + + /** + * Sets the flag to indicate whether the cluster is opted out of Conformity monkey. + * @param optOutOfConformity + * true if the cluster is not handled by Conformity monkey, false otherwise + */ + public void setOptOutOfConformity(boolean optOutOfConformity) { + isOptOutOfConformity = optOutOfConformity; + } + + /** + * Gets a map from fields of resources to corresponding values. Values are represented + * as Strings so they can be displayed or stored in databases like SimpleDB. + * @return a map from field name to field value + */ + public Map getFieldToValueMap() { + Map map = Maps.newHashMap(); + putToMapIfNotNull(map, CLUSTER, name); + putToMapIfNotNull(map, REGION, region); + putToMapIfNotNull(map, OWNER_EMAIL, ownerEmail); + putToMapIfNotNull(map, UPDATE_TIMESTAMP, String.valueOf(DATE_FORMATTER.print(updateTime.getTime()))); + putToMapIfNotNull(map, IS_CONFORMING, String.valueOf(isConforming)); + putToMapIfNotNull(map, IS_OPTEDOUT, String.valueOf(isOptOutOfConformity)); + putToMapIfNotNull(map, EXCLUDED_RULES, StringUtils.join(excludedConformityRules, ",")); + List ruleIds = Lists.newArrayList(); + for (Conformity conformity : conformities.values()) { + map.put(conformity.getRuleId(), StringUtils.join(conformity.getFailedComponents(), ",")); + ruleIds.add(conformity.getRuleId()); + } + putToMapIfNotNull(map, CONFORMITY_RULES, StringUtils.join(ruleIds, ",")); + return map; + } + + /** + * Parse a map from field name to value to a cluster. + * @param fieldToValue the map from field name to value + * @return the cluster that is de-serialized from the map + */ + public static Cluster parseFieldToValueMap(Map fieldToValue) { + Validate.notNull(fieldToValue); + Cluster cluster = new Cluster(fieldToValue.get(CLUSTER), + fieldToValue.get(REGION)); + cluster.setOwnerEmail(fieldToValue.get(OWNER_EMAIL)); + cluster.setConforming(Boolean.parseBoolean(fieldToValue.get(IS_CONFORMING))); + cluster.setOptOutOfConformity(Boolean.parseBoolean(fieldToValue.get(IS_OPTEDOUT))); + cluster.excludeRules(StringUtils.split(fieldToValue.get(EXCLUDED_RULES), ",")); + cluster.setUpdateTime(new Date(DATE_FORMATTER.parseDateTime(fieldToValue.get(UPDATE_TIMESTAMP)).getMillis())); + for (String ruleId : StringUtils.split(fieldToValue.get(CONFORMITY_RULES), ",")) { + cluster.updateConformity(new Conformity(ruleId, + Lists.newArrayList(StringUtils.split(fieldToValue.get(ruleId), ",")))); + } + return cluster; + } + + private static void putToMapIfNotNull(Map map, String key, String value) { + Validate.notNull(map); + Validate.notNull(key); + if (value != null) { + map.put(key, value); + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ClusterCrawler.java b/src/main/java/com/netflix/simianarmy/conformity/ClusterCrawler.java new file mode 100644 index 00000000..da55f1a7 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ClusterCrawler.java @@ -0,0 +1,50 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import java.util.List; + +/** + * The interface of the crawler for Conformity Monkey to get the cluster information. + */ +public interface ClusterCrawler { + + /** + * Gets the up to date information for a collection of clusters. When the input argument is null + * or empty, the method returns all clusters. + * + * @param clusterNames + * the cluster names + * @return the list of clusters + */ + List clusters(String... clusterNames); + + /** + * Gets the owner email for a cluster to set the ownerEmail field when crawl. + * @param cluster + * the cluster + * @return the owner email of the cluster + */ + String getOwnerEmailForCluster(Cluster cluster); + + /** + * Updates the excluded conformity rules for the given cluster. + * @param cluster + */ + void updateExcludedConformityRules(Cluster cluster); +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/Conformity.java b/src/main/java/com/netflix/simianarmy/conformity/Conformity.java new file mode 100644 index 00000000..070b4f5b --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/Conformity.java @@ -0,0 +1,68 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.Validate; + +import java.util.Collection; +import java.util.Collections; + +/** + * The class defining the result of a conformity check. + */ +public class Conformity { + + private final String ruleId; + private final Collection failedComponenets = Lists.newArrayList(); + + /** + * Constructor. + * @param ruleId + * the conformity rule id + * @param failedComponenets + * the components that cause the conformity check to fail, if there is + * no failed componenets, it means the conformity check passes. + */ + public Conformity(String ruleId, Collection failedComponenets) { + Validate.notNull(ruleId); + Validate.notNull(failedComponenets); + this.ruleId = ruleId; + for (String failedComponent : failedComponenets) { + this.failedComponenets.add(failedComponent); + } + } + + /** + * Gets the conformity rule id. + * @return + * the conformity rule id + */ + public String getRuleId() { + return ruleId; + } + + /** + * Gets the components that cause the conformity check to fail. + * @return + * the components that cause the conformity check to fail + */ + public Collection getFailedComponents() { + return Collections.unmodifiableCollection(failedComponenets); + } +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityClusterTracker.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityClusterTracker.java new file mode 100644 index 00000000..da9029de --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityClusterTracker.java @@ -0,0 +1,67 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import java.util.List; + +/** + * The interface that defines the tracker to manage clusters for Conformity monkey to use. + */ +public interface ConformityClusterTracker { + /** + * Adds a cluster to the tracker. If the cluster with the same name already exists, + * the method updates the record with the cluster parameter. + * @param cluster + * the cluster to add or update + */ + void addOrUpdate(Cluster cluster); + + /** + * Gets the list of clusters in a list of regions. + * @param regions + * the regions of the clusters, when the parameter is null or empty, the method returns + * clusters from all regions + * @return list of clusters in the given regions + */ + List getAllClusters(String... regions); + + /** + * Gets the list of non-conforming clusters in a list of regions. + * @param regions the regions of the clusters, when the parameter is null or empty, the method returns + * clusters from all regions + * @return list of clusters in the given regions + */ + List getNonconformingClusters(String... regions); + + /** + * Gets the cluster with a specific name from . + * @param name the cluster name + * @param region the region of the cluster + * @return the cluster with the name + */ + Cluster getCluster(String name, String region); + + + /** + * Deletes a list of clusters from the tracker. + * @param clusters the list of clusters to delete. The parameter cannot be null. If it is empty, + * no cluster is deleted. + */ + void deleteClusters(Cluster... clusters); + +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailBuilder.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailBuilder.java new file mode 100644 index 00000000..a2168b5e --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailBuilder.java @@ -0,0 +1,36 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.netflix.simianarmy.AbstractEmailBuilder; + +import java.util.Collection; +import java.util.Map; + +/** The abstract class for building Conformity monkey email notifications. */ +public abstract class ConformityEmailBuilder extends AbstractEmailBuilder { + + /** + * Sets the map from an owner email to the clusters that belong to the owner + * and need to send notifications for. + * @param emailToClusters the map from owner email to the owned clusters + * @param rules all conformity rules that are used to find the description of each rule to display + */ + public abstract void setEmailToClusters(Map> emailToClusters, + Collection rules); +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailNotifier.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailNotifier.java new file mode 100644 index 00000000..f2b9e9bb --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityEmailNotifier.java @@ -0,0 +1,236 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.netflix.simianarmy.aws.AWSEmailNotifier; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * The email notifier implemented for Janitor Monkey. + */ +public class ConformityEmailNotifier extends AWSEmailNotifier { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(ConformityEmailNotifier.class); + private static final String UNKNOWN_EMAIL = "UNKNOWN"; + + private final Collection regions = Lists.newArrayList(); + private final String defaultEmail; + private final List ccEmails = Lists.newArrayList(); + private final ConformityClusterTracker clusterTracker; + private final ConformityEmailBuilder emailBuilder; + private final String sourceEmail; + private final Map> invalidEmailToClusters = Maps.newHashMap(); + private final Collection rules = Lists.newArrayList(); + private final int openHour; + private final int closeHour; + + /** + * The Interface Context. + */ + public interface Context { + /** + * Gets the Amazon Simple Email Service client. + * @return the Amazon Simple Email Service client + */ + AmazonSimpleEmailServiceClient sesClient(); + + /** + * Gets the open hour the email notifications are sent. + * @return + * the open hour the email notifications are sent + */ + int opentHour(); + + /** + * Gets the close hour the email notifications are sent. + * @return + * the close hour the email notifications are sent + */ + int closeHour(); + + /** + * Gets the source email the notifier uses to send email. + * @return the source email + */ + String sourceEmail(); + + /** + * Gets the default email the notifier sends to when there is no owner specified for a cluster. + * @return the default email + */ + String defaultEmail(); + + /** + * Gets the regions the notifier is running in. + * @return the regions the notifier is running in. + */ + Collection regions(); + + /** Gets the Conformity Monkey's cluster tracker. + * @return the Conformity Monkey's cluster tracker + */ + ConformityClusterTracker clusterTracker(); + + /** Gets the Conformity email builder. + * @return the Conformity email builder + */ + ConformityEmailBuilder emailBuilder(); + + /** Gets the cc email addresses. + * @return the cc email addresses + */ + String[] ccEmails(); + + /** + * Gets all the conformity rules. + * @return all conformity rules. + */ + Collection rules(); + } + + /** + * Constructor. + * @param ctx the context. + */ + public ConformityEmailNotifier(Context ctx) { + super(ctx.sesClient()); + this.openHour = ctx.opentHour(); + this.closeHour = ctx.closeHour(); + for (String region : ctx.regions()) { + this.regions.add(region); + } + this.defaultEmail = ctx.defaultEmail(); + this.clusterTracker = ctx.clusterTracker(); + this.emailBuilder = ctx.emailBuilder(); + String[] ctxCCs = ctx.ccEmails(); + if (ctxCCs != null) { + for (String ccEmail : ctxCCs) { + this.ccEmails.add(ccEmail); + } + } + this.sourceEmail = ctx.sourceEmail(); + Validate.notNull(ctx.rules()); + for (ConformityRule rule : ctx.rules()) { + rules.add(rule); + } + } + + /** + * Gets all the clusters that are not conforming and sends email notifications to the owners. + */ + public void sendNotifications() { + int currentHour = DateTime.now().getHourOfDay(); + if (currentHour < openHour || currentHour > closeHour) { + LOGGER.info("It is not the time for Conformity Monkey to send notifications. You can change " + + "simianarmy.conformity.notification.openHour and simianarmy.conformity.notification.openHour" + + " to make it work at this hour."); + return; + } + + validateEmails(); + Map> emailToClusters = Maps.newHashMap(); + for (Cluster cluster : clusterTracker.getNonconformingClusters(regions.toArray(new String[regions.size()]))) { + if (cluster.isOptOutOfConformity()) { + LOGGER.info(String.format("Cluster %s is opted out of Conformity Monkey so no notification is sent.", + cluster.getName())); + continue; + } + if (!cluster.isConforming()) { + String email = cluster.getOwnerEmail(); + if (!isValidEmail(email)) { + if (defaultEmail != null) { + LOGGER.info(String.format("Email %s is not valid, send to the default email address %s", + email, defaultEmail)); + putEmailAndCluster(emailToClusters, defaultEmail, cluster); + } else { + if (email == null) { + email = UNKNOWN_EMAIL; + } + LOGGER.info(String.format("Email %s is not valid and default email is not set for cluster %s", + email, cluster.getName())); + putEmailAndCluster(invalidEmailToClusters, email, cluster); + } + } else { + putEmailAndCluster(emailToClusters, email, cluster); + } + } else { + LOGGER.debug(String.format("Cluster %s is conforming so no notification needs to be sent.", + cluster.getName())); + } + } + emailBuilder.setEmailToClusters(emailToClusters, rules); + for (Map.Entry> entry : emailToClusters.entrySet()) { + String email = entry.getKey(); + String emailBody = emailBuilder.buildEmailBody(email); + String subject = buildEmailSubject(email); + sendEmail(email, subject, emailBody); + for (Cluster cluster : entry.getValue()) { + LOGGER.debug(String.format("Notification is sent for cluster %s to %s", cluster.getName(), email)); + } + LOGGER.info(String.format("Email notification has been sent to %s for %d clusters.", + email, entry.getValue().size())); + } + } + + + @Override + public String buildEmailSubject(String to) { + return String.format("Conformity Monkey Notification for %s", to); + } + + @Override + public String[] getCcAddresses(String to) { + return ccEmails.toArray(new String[ccEmails.size()]); + } + + @Override + public String getSourceAddress(String to) { + return sourceEmail; + } + + private void validateEmails() { + if (defaultEmail != null) { + Validate.isTrue(isValidEmail(defaultEmail), String.format("Default email %s is invalid", defaultEmail)); + } + if (ccEmails != null) { + for (String ccEmail : ccEmails) { + Validate.isTrue(isValidEmail(ccEmail), String.format("CC email %s is invalid", ccEmail)); + } + } + } + + private void putEmailAndCluster(Map> map, String email, Cluster cluster) { + Collection clusters = map.get(email); + if (clusters == null) { + clusters = Lists.newArrayList(); + map.put(email, clusters); + } + clusters.add(cluster); + } +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityMonkey.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityMonkey.java new file mode 100644 index 00000000..f3ef9ead --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityMonkey.java @@ -0,0 +1,119 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.netflix.simianarmy.Monkey; +import com.netflix.simianarmy.MonkeyConfiguration; + +import java.util.Collection; + +/** + * The abstract class for Conformity Monkey. + */ +public abstract class ConformityMonkey extends Monkey { + + /** + * The Interface Context. + */ + public interface Context extends Monkey.Context { + + /** + * Configuration. + * + * @return the monkey configuration + */ + MonkeyConfiguration configuration(); + + /** + * Crawler that gets information of all clusters for conformity check. + * @return all clusters for conformity check + */ + ClusterCrawler clusterCrawler(); + + /** + * Conformity rule engine. + * @return the Conformity rule engine + */ + ConformityRuleEngine ruleEngine(); + + + /** + * Email notifier used to send notifications by the Conformity monkey. + * @return the email notifier + */ + ConformityEmailNotifier emailNotifier(); + + /** + * The regions the monkey is running in. + * @return the regions the monkey is running in. + */ + Collection regions(); + + /** + * The tracker of the clusters for conformity monkey to check. + * @return the tracker of the clusters for conformity monkey to check. + */ + ConformityClusterTracker clusterTracker(); + + /** + * Gets the flag to indicate whether the monkey is leashed. + * @return true if the monkey is leashed and does not make real change or send notifications to + * cluster owners, false otherwise. + */ + boolean isLeashed(); + } + + /** The context. */ + private final Context ctx; + + /** + * Instantiates a new Coformity monkey. + * + * @param ctx + * the context. + */ + public ConformityMonkey(Context ctx) { + super(ctx); + this.ctx = ctx; + } + + /** + * The monkey Type. + */ + public enum Type { + /** Conformity monkey. */ + CONFORMITY + } + + /** {@inheritDoc} */ + @Override + public final Enum type() { + return Type.CONFORMITY; + } + + /** {@inheritDoc} */ + @Override + public Context context() { + return ctx; + } + + /** {@inheritDoc} */ + @Override + public abstract void doMonkeyBusiness(); + +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityRule.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityRule.java new file mode 100644 index 00000000..53ad3117 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityRule.java @@ -0,0 +1,47 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.netflix.simianarmy.conformity; + +/** + * Interface for a conformity check rule. + */ +public interface ConformityRule { + + /** + * Performs the conformity check against the rule. + * @param cluster + * the cluster to check for conformity + * @return + * the conformity check result + */ + Conformity check(Cluster cluster); + + /** + * Gets the name/id of the rule. + * @return + * the name of the rule + */ + String getName(); + + /** + * Gets the human-readable reason to explain why the cluster is not conforming. + * @return the human-readable reason to explain why the cluster is not conforming + */ + String getNonconformingReason(); +} diff --git a/src/main/java/com/netflix/simianarmy/conformity/ConformityRuleEngine.java b/src/main/java/com/netflix/simianarmy/conformity/ConformityRuleEngine.java new file mode 100644 index 00000000..ed0fa896 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/conformity/ConformityRuleEngine.java @@ -0,0 +1,87 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.conformity; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; + +/** + * The class implementing the conformity rule engine. + */ +public class ConformityRuleEngine { + private static final Logger LOGGER = LoggerFactory.getLogger(ConformityRuleEngine.class); + + private final Collection rules = Lists.newArrayList(); + + /** + * Checks whether a cluster is conforming or not against the rules in the engine. This + * method runs the checks the cluster against all the rules. + * + * @param cluster + * the cluster + * @return true if the cluster is conforming, false otherwise. + */ + public boolean check(Cluster cluster) { + Validate.notNull(cluster); + cluster.clearConformities(); + for (ConformityRule rule : rules) { + if (!cluster.getExcludedRules().contains(rule.getName())) { + LOGGER.info(String.format("Running conformity rule %s on cluster %s", + rule.getName(), cluster.getName())); + cluster.updateConformity(rule.check(cluster)); + } else { + LOGGER.info(String.format("Conformity rule %s is excluded on cluster %s", + rule.getName(), cluster.getName())); + } + } + boolean isConforming = true; + for (Conformity conformity : cluster.getConformties()) { + if (!conformity.getFailedComponents().isEmpty()) { + isConforming = false; + } + } + cluster.setConforming(isConforming); + return isConforming; + } + + /** + * Add a conformity rule. + * + * @param rule + * The conformity rule to add. + * @return The Conformity rule engine object. + */ + public ConformityRuleEngine addRule(ConformityRule rule) { + Validate.notNull(rule); + rules.add(rule); + return this; + } + + /** + * Gets all conformity rules in the rule engine. + * @return all conformity rules in the rule engine + */ + public Collection rules() { + return Collections.unmodifiableCollection(rules); + } +} diff --git a/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java index a24ac34b..21b66331 100644 --- a/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java +++ b/src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java @@ -273,12 +273,11 @@ protected Map getTrackedMarkedResources() { public void cleanupResources() { cleanedResources.clear(); failedToCleanResources.clear(); - List trackedMarkedResources = resourceTracker.getResources( - resourceType, Resource.CleanupState.MARKED, region); + Map trackedMarkedResources = getTrackedMarkedResources(); LOGGER.info(String.format("Checking %d marked resources for cleanup.", trackedMarkedResources.size())); Date now = calendar.now().getTime(); - for (Resource markedResource : trackedMarkedResources) { + for (Resource markedResource : trackedMarkedResources.values()) { if (canClean(markedResource, now)) { LOGGER.info(String.format("Cleaning up resource %s of type %s", markedResource.getId(), markedResource.getResourceType().name())); diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java index 5bc5d46b..44e9e782 100644 --- a/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorEmailNotifier.java @@ -17,6 +17,16 @@ */ package com.netflix.simianarmy.janitor; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.netflix.simianarmy.MonkeyCalendar; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.aws.AWSEmailNotifier; +import org.apache.commons.lang.Validate; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -24,27 +34,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; - -import org.apache.commons.lang.Validate; -import org.joda.time.DateTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; -import com.netflix.simianarmy.MonkeyCalendar; -import com.netflix.simianarmy.Resource; -import com.netflix.simianarmy.Resource.CleanupState; -import com.netflix.simianarmy.aws.AWSEmailNotifier; /** The email notifier implemented for Janitor Monkey. */ public class JanitorEmailNotifier extends AWSEmailNotifier { /** The Constant LOGGER. */ private static final Logger LOGGER = LoggerFactory.getLogger(JanitorEmailNotifier.class); - private static final String EMAIL_PATTERN = - "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@" - + "[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$"; private static final String UNKNOWN_EMAIL = "UNKNOWN"; /** * If the sheduled termination date is within 2 hours of notification date + headsup days, @@ -55,7 +50,6 @@ public class JanitorEmailNotifier extends AWSEmailNotifier { private final String region; private final String defaultEmail; private final List ccEmails; - private final Pattern emailPattern; private final JanitorResourceTracker resourceTracker; private final JanitorEmailBuilder emailBuilder; private final MonkeyCalendar calendar; @@ -126,7 +120,6 @@ public interface Context { public JanitorEmailNotifier(Context ctx) { super(ctx.sesClient()); this.region = ctx.region(); - this.emailPattern = Pattern.compile(EMAIL_PATTERN); this.defaultEmail = ctx.defaultEmail(); this.daysBeforeTermination = ctx.daysBeforeTermination(); this.resourceTracker = ctx.resourceTracker(); @@ -215,19 +208,6 @@ private void validateEmails() { } } - @Override - public boolean isValidEmail(String email) { - if (email == null) { - return false; - } - if (emailPattern.matcher(email).matches()) { - return true; - } else { - LOGGER.error(String.format("Invalid email address: %s", email)); - return false; - } - } - @Override public String buildEmailSubject(String email) { return String.format("Janitor Monkey Notification for %s", email); diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java index 1a3a526b..d0c6e2e7 100644 --- a/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java +++ b/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java @@ -47,7 +47,7 @@ public interface JanitorResourceTracker { /** Gets the resource of a specific id. * * @param resourceId the resource id - * @return list of resources that match the resource id + * @return the resource that matches the resource id */ Resource getResource(String resourceId); diff --git a/src/main/resources/client.properties b/src/main/resources/client.properties index 55dcf1d6..c72f9fea 100644 --- a/src/main/resources/client.properties +++ b/src/main/resources/client.properties @@ -1,33 +1,33 @@ -##################################################################### -### Configure which client and context to use. -##################################################################### - -### The default implementation is to use an AWS Client, equaling a property like the following: -# -#simianarmy.client.context.class=com.netflix.simianarmy.basic.BasicContext -# -### to use an VSphereClient instead, uncomment this: -# -#simianarmy.client.context.class=com.netflix.simianarmy.client.vsphere.VSphereContext -# -### configure the specific selected client, e.g for VSphere these are -# -#simianarmy.client.vsphere.url=https://YOUR_VSPHERE_SERVER/sdk -#simianarmy.client.vsphere.username=YOUR_SERVICE_ACCOUNT_USERNAME -#simianarmy.client.vsphere.password=YOUR_SERVICE_ACCOUNT_PASSWORD -# -### configure the specific selected client, e.g for AWS these are - -### both "accountKey" and "secretKey" can be left blank or be removed, -### if the credentials are provided as environment variable or -### an instance role is used to handle permissions -### see: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html -simianarmy.client.aws.accountKey = fakeAccount -simianarmy.client.aws.secretKey = fakeSecret -simianarmy.client.aws.region = us-east-1 - -### The VSpehere client uses a TerminationStrategy for killing VirtualMachines -### You can configure which property and value for it to set prior to resetting the VirtualMachine -# -#simianarmy.client.vsphere.terminationStrategy.property.name=Force Boot -#simianarmy.client.vsphere.terminationStrategy.property.value=server +##################################################################### +### Configure which client and context to use. +##################################################################### + +### The default implementation is to use an AWS Client, equaling a property like the following: +# +#simianarmy.client.context.class=com.netflix.simianarmy.basic.BasicContext +# +### to use an VSphereClient instead, uncomment this: +# +#simianarmy.client.context.class=com.netflix.simianarmy.client.vsphere.VSphereContext +# +### configure the specific selected client, e.g for VSphere these are +# +#simianarmy.client.vsphere.url=https://YOUR_VSPHERE_SERVER/sdk +#simianarmy.client.vsphere.username=YOUR_SERVICE_ACCOUNT_USERNAME +#simianarmy.client.vsphere.password=YOUR_SERVICE_ACCOUNT_PASSWORD +# +### configure the specific selected client, e.g for AWS these are + +### both "accountKey" and "secretKey" can be left blank or be removed, +### if the credentials are provided as environment variable or +### an instance role is used to handle permissions +### see: http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html +simianarmy.client.aws.accountKey = fakeAccount +simianarmy.client.aws.secretKey = fakeSecret +simianarmy.client.aws.region = us-east-1 + +### The VSpehere client uses a TerminationStrategy for killing VirtualMachines +### You can configure which property and value for it to set prior to resetting the VirtualMachine +# +#simianarmy.client.vsphere.terminationStrategy.property.name=Force Boot +#simianarmy.client.vsphere.terminationStrategy.property.value=server \ No newline at end of file diff --git a/src/main/resources/conformity.properties b/src/main/resources/conformity.properties new file mode 100644 index 00000000..760b7c45 --- /dev/null +++ b/src/main/resources/conformity.properties @@ -0,0 +1,85 @@ +# let Conformity monkey run +simianarmy.conformity.enabled = true + +# dryrun mode, no email notification to the owner of nonconforming clusters is sent +simianarmy.conformity.leashed = true + +# By default Conformity Monkey wakes up every hour +simianarmy.scheduler.frequency = 1 +simianarmy.scheduler.frequencyUnit = HOURS +simianarmy.scheduler.threads = 1 + +# Conformity Monkey runs every hour. +simianarmy.calendar.openHour = 0 +simianarmy.calendar.closeHour = 24 +simianarmy.calendar.timezone = America/Los_Angeles + +# override to force monkey time, useful for debugging off hours +#simianarmy.calendar.isMonkeyTime = true + +# Conformity monkey sends notifications to the owner of unconforming clusters between the open hour and close +# hour only. In other hours, only summary email is sent. The default setting is to always send email notifications +# after each run. +simianarmy.conformity.notification.openHour = 0 +simianarmy.conformity.notification.closeHour = 24 + +simianarmy.conformity.sdb.domain = SIMIAN_ARMY + +# The property below needs to be a valid email address to receive the summary email of Conformity Monkey +# after each run +simianarmy.conformity.summaryEmail.to = michaelf@netflix.com + +# The property below needs to be a valid email address to send notifications for Conformity monkey +simianarmy.conformity.notification.defaultEmail = michaelf@netflix.com + +# The property below needs to be a valid email address to send notifications for Conformity Monkey +simianarmy.conformity.notification.sourceEmail = cloudmonkey@saasmail.netflix.com + +# By default Eureka is not enabled. The conformity rules that need to access Eureka are not added +# when Eureka is not enabled. +simianarmy.conformity.Eureka.enabled = false + +# The following property is used to enable the conformity rule to check whether there is mismatch of availability +# zones between any auto scaling group and its ELBs in a cluster. +simianarmy.conformity.rule.SameZonesInElbAndAsg.enabled = true + +# The following property is used to enable the conformity rule to check whether all instances in the cluster +# are in required security groups. +simianarmy.conformity.rule.InstanceInSecurityGroup.enabled = true + +# The following property specifies the required security groups in the InstanceInSecurityGroup conformity rule. +simianarmy.conformity.rule.InstanceInSecurityGroup.requiredSecurityGroups = nf-infrastructure, nf-datacenter + +# The following property is used to enable the conformity rule to check whether there is any instance that is +# older than certain days. +simianarmy.conformity.rule.InstanceTooOld.enabled = true + +# The following property specifies the number of days used in the InstanceInSecurityGroup, any instance that is +# old than this number of days is consider nonconforming. +simianarmy.conformity.rule.InstanceTooOld.instanceAgeThreshold = 180 + +# The following property is used to enable the conformity rule to check whether all instances in the cluster +# have a status url defined according to Discovery/Eureka. +simianarmy.conformity.rule.InstanceHasStatusUrl.enabled = true + +# The following property is used to enable the conformity rule to check whether all instances in the cluster +# have a health check url defined according to Discovery/Eureka. +simianarmy.conformity.rule.InstanceHasHealthCheckUrl.enabled = true + +# The following property is used to enable the conformity rule to check whether there are unhealthy instances +# in the cluster accoring to Discovery/Eureka. +simianarmy.conformity.rule.InstanceIsHealthyInEureka.enabled = true + +# You can override a cluster's owner email by providing a property here. For example, the line below overrides +# the owner email of cluster foo to foo@bar.com +# simianarmy.conformity.cluster.foo.ownerEmail = foo@bar.com + +# You can exclude specific conformity rules for a cluster using this property. For example, the line below excludes +# the conformity rule rule1 and rule2 on cluster foo. +# simianarmy.conformity.cluster.foo.excludedRules = rule1,rule2 + +# You can opt out a cluster completely from Conformity Monkey by using this property. After a cluster is opted out, +# no notification is sent for it no matter it is conforming or not. For example, the line below opts out the cluster +# foo. +# simianarmy.conformity.cluster.foo.optedOut = true +