diff --git a/build.gradle b/build.gradle index 9a144bc0..4f5ecd74 100644 --- a/build.gradle +++ b/build.gradle @@ -48,6 +48,8 @@ dependencies { compile 'org.apache.jclouds.api:ec2:1.9.0' compile 'org.apache.jclouds.provider:aws-ec2:1.9.0' compile 'com.netflix.servo:servo-core:0.9.4' + compile 'org.springframework:spring-jdbc:4.2.5.RELEASE' + compile 'com.zaxxer:HikariCP:2.4.7' testCompile 'org.testng:testng:6.3.1' testCompile 'org.mockito:mockito-core:1.8.5' diff --git a/src/main/java/com/netflix/simianarmy/aws/RDSRecorder.java b/src/main/java/com/netflix/simianarmy/aws/RDSRecorder.java new file mode 100644 index 00000000..160e2a84 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/RDSRecorder.java @@ -0,0 +1,241 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws; + +import com.amazonaws.AmazonClientException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.simianarmy.EventType; +import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.MonkeyType; +import com.netflix.simianarmy.basic.BasicRecorderEvent; +import com.zaxxer.hikari.HikariDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * The Class RDSRecorder. Records events to and fetched events from a RDS table (default SIMIAN_ARMY) + */ +@SuppressWarnings("serial") +public class RDSRecorder implements MonkeyRecorder { + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(RDSRecorder.class); + + private final String region; + + /** The table. */ + private final String table; + + /** the jdbcTemplate */ + JdbcTemplate jdbcTemplate = null; + + public static final String FIELD_ID = "eventId"; + public static final String FIELD_EVENT_TIME = "eventTime"; + public static final String FIELD_MONKEY_TYPE = "monkeyType"; + public static final String FIELD_EVENT_TYPE = "eventType"; + public static final String FIELD_REGION = "region"; + public static final String FIELD_DATA_JSON = "dataJson"; + + /** + * Instantiates a new RDS recorder. + * + */ + public RDSRecorder(String dbDriver, String dbUser, + String dbPass, String dbUrl, String dbTable, String region) { + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dbDriver); + dataSource.setJdbcUrl(dbUrl); + dataSource.setUsername(dbUser); + dataSource.setPassword(dbPass); + this.jdbcTemplate = new JdbcTemplate(dataSource); + this.table = dbTable; + this.region = region; + } + + /** + * Instantiates a new RDS recorder. This constructor is intended + * for unit testing. + * + */ + public RDSRecorder(JdbcTemplate jdbcTemplate, String table, String region) { + this.jdbcTemplate = jdbcTemplate; + this.table = table; + this.region = region; + } + + public JdbcTemplate getJdbcTemplate() { + return jdbcTemplate; + } + + /** {@inheritDoc} */ + @Override + public Event newEvent(MonkeyType monkeyType, EventType eventType, String reg, String id) { + return new BasicRecorderEvent(monkeyType, eventType, reg, id); + } + + /** {@inheritDoc} */ + @Override + public void recordEvent(Event evt) { + String evtTime = String.valueOf(evt.eventTime().getTime()); + String name = String.format("%s-%s-%s-%s", evt.monkeyType().name(), evt.id(), region, evtTime); + String json; + try { + json = new ObjectMapper().writeValueAsString(evt.fields()); + } catch (JsonProcessingException e) { + LOGGER.error("ERROR generating JSON when saving resource " + name, e); + return; + } + + LOGGER.debug(String.format("Saving event %s to RDS table %s", name, table)); + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table); + sb.append(" ("); + sb.append(FIELD_ID).append(","); + sb.append(FIELD_EVENT_TIME).append(","); + sb.append(FIELD_MONKEY_TYPE).append(","); + sb.append(FIELD_EVENT_TYPE).append(","); + sb.append(FIELD_REGION).append(","); + sb.append(FIELD_DATA_JSON).append(") values (?,?,?,?,?,?)"); + + LOGGER.debug(String.format("Insert statement is '%s'", sb)); + int updated = this.jdbcTemplate.update(sb.toString(), + evt.id(), + evt.eventTime().getTime(), + SimpleDBRecorder.enumToValue(evt.monkeyType()), + SimpleDBRecorder.enumToValue(evt.eventType()), + evt.region(), + json); + LOGGER.debug(String.format("%d rows inserted", updated)); + } + + /** {@inheritDoc} */ + @Override + public List findEvents(Map query, Date after) { + return findEvents(null, null, query, after); + } + + /** {@inheritDoc} */ + @Override + public List findEvents(MonkeyType monkeyType, Map query, Date after) { + return findEvents(monkeyType, null, query, after); + } + + /** {@inheritDoc} */ + @Override + public List findEvents(MonkeyType monkeyType, EventType eventType, Map query, Date after) { + ArrayList args = new ArrayList<>(); + StringBuilder sqlquery = new StringBuilder( + String.format("select * from %s where region = ?", table, region)); + args.add(table); + + if (monkeyType != null) { + sqlquery.append(String.format(" and %s = ?", FIELD_MONKEY_TYPE)); + args.add(SimpleDBRecorder.enumToValue(monkeyType)); + } + + if (eventType != null) { + sqlquery.append(String.format(" and %s = ?", FIELD_EVENT_TYPE)); + args.add(SimpleDBRecorder.enumToValue(eventType)); + } + + for (Map.Entry pair : query.entrySet()) { + sqlquery.append(String.format(" and %s like ?", FIELD_DATA_JSON)); + args.add((String.format("%s: \"%s\"", pair.getKey(), pair.getValue()))); + } + sqlquery.append(String.format(" and %s > ? order by %s desc", FIELD_EVENT_TIME, FIELD_EVENT_TIME)); + args.add(new Long(after.getTime())); + + LOGGER.debug(String.format("Query is '%s'", sqlquery)); + List events = jdbcTemplate.query(sqlquery.toString(), args.toArray(), new RowMapper() { + public Event mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapEvent(rs); + } + }); + return events; + } + + private Event mapEvent(ResultSet rs) throws SQLException { + String json = rs.getString("dataJson"); + ObjectMapper mapper = new ObjectMapper(); + Event event = null; + try { + String id = rs.getString(FIELD_ID); + MonkeyType monkeyType = SimpleDBRecorder.valueToEnum(MonkeyType.class, rs.getString(FIELD_MONKEY_TYPE)); + EventType eventType = SimpleDBRecorder.valueToEnum(EventType.class, rs.getString(FIELD_EVENT_TYPE)); + String region = rs.getString(FIELD_REGION); + long time = rs.getLong(FIELD_EVENT_TIME); + event = new BasicRecorderEvent(monkeyType, eventType, region, id, time); + + TypeReference> typeRef = new TypeReference>() {}; + Map map = mapper.readValue(json, typeRef); + for(String key : map.keySet()) { + event.addField(key, map.get(key)); + } + + }catch(IOException ie) { + LOGGER.error("Error parsing resource from json", ie); + } + return event; + } + + + /** + * Creates the RDS table, if it does not already exist. + */ + public void init() { + try { + if (this.region == null || this.region.equals("region-null")) { + // This is a mock with an invalid region; avoid a slow timeout + LOGGER.debug("Region=null; skipping RDS table creation"); + return; + } + + LOGGER.info("Creating RDS table: {}", table); + String sql = String.format("create table if not exists %s (" + + " %s varchar(255)," + + " %s BIGINT," + + " %s varchar(255)," + + " %s varchar(255)," + + " %s varchar(255)," + + " %s varchar(4096) )", + table, + FIELD_ID, + FIELD_EVENT_TIME, + FIELD_MONKEY_TYPE, + FIELD_EVENT_TYPE, + FIELD_REGION, + FIELD_DATA_JSON); + LOGGER.debug("Create SQL is: '{}'", sql); + jdbcTemplate.execute(sql); + + } catch (AmazonClientException e) { + LOGGER.warn("Error while trying to auto-create RDS table", e); + } + } +} diff --git a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java index fc40ea07..06120a9e 100644 --- a/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java +++ b/src/main/java/com/netflix/simianarmy/aws/SimpleDBRecorder.java @@ -123,7 +123,7 @@ protected AmazonSimpleDB sdbClient() { * the e * @return the string */ - private static String enumToValue(NamedType e) { + public static String enumToValue(NamedType e) { return String.format("%s|%s", e.name(), e.getClass().getName()); } @@ -134,7 +134,7 @@ private static String enumToValue(NamedType e) { * the value * @return the enum */ - private static T valueToEnum( + public static T valueToEnum( Class type, String value) { // parts = [enum value, enum class type] String[] parts = value.split("\\|", 2); diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java b/src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java new file mode 100644 index 00000000..3caad919 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java @@ -0,0 +1,349 @@ +/* + * + * Copyright 2013 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.conformity; + +import com.amazonaws.AmazonClientException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; +import com.netflix.simianarmy.conformity.ConformityClusterTracker; +import com.zaxxer.hikari.HikariDataSource; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The RDSConformityClusterTracker implementation in RDS (relational database). + */ +public class RDSConformityClusterTracker implements ConformityClusterTracker { + + /** The Constant LOGGER. */ + private static final Logger LOGGER = LoggerFactory.getLogger(RDSConformityClusterTracker.class); + + /** The table. */ + private final String table; + + /** the jdbcTemplate */ + JdbcTemplate jdbcTemplate = null; + + /** + * Instantiates a new RDS db resource tracker. + * + */ + public RDSConformityClusterTracker(String dbDriver, String dbUser, + String dbPass, String dbUrl, String dbTable) { + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dbDriver); + dataSource.setJdbcUrl(dbUrl); + dataSource.setUsername(dbUser); + dataSource.setPassword(dbPass); + this.jdbcTemplate = new JdbcTemplate(dataSource); + this.table = dbTable; + } + + /** + * Instantiates a new RDS conformity cluster tracker. This constructor is intended + * for unit testing. + * + */ + public RDSConformityClusterTracker(JdbcTemplate jdbcTemplate, String table) { + this.jdbcTemplate = jdbcTemplate; + this.table = table; + } + + public JdbcTemplate getJdbcTemplate() { + return jdbcTemplate; + } + + public Object value(String value) { + return value == null ? Types.NULL : value; + } + + public Object value(Date value) { + return value == null ? Types.NULL : value.getTime(); + } + + public Object value(boolean value) { + return Boolean.toString(value); + } + + public Object emailValue(String email) { + if (StringUtils.isBlank(email)) return Types.NULL; + if (email.equals("0")) return Types.NULL; + return email; + } + + /** {@inheritDoc} */ + @Override + public void addOrUpdate(Cluster cluster) { + Cluster orig = getCluster(cluster.getName(), cluster.getRegion()); + LOGGER.debug(String.format("Saving cluster %s to RDB table %s", cluster.getName(), table)); + Map map = cluster.getFieldToValueMap(); + + String conformityJson; + try { + conformityJson = new ObjectMapper().writeValueAsString(conformitiesAsMap(cluster)); + } catch (JsonProcessingException e) { + LOGGER.error("ERROR generating conformities JSON when saving cluster " + cluster.getName() + ", " + cluster.getRegion(), e); + return; + } + + if (orig == null) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table); + sb.append(" ("); + sb.append(Cluster.CLUSTER).append(","); + sb.append(Cluster.REGION).append(","); + sb.append(Cluster.OWNER_EMAIL).append(","); + sb.append(Cluster.IS_CONFORMING).append(","); + sb.append(Cluster.IS_OPTEDOUT).append(","); + sb.append(Cluster.UPDATE_TIMESTAMP).append(","); + sb.append(Cluster.EXCLUDED_RULES).append(","); + sb.append("conformities").append(","); + sb.append(Cluster.CONFORMITY_RULES); + sb.append(") values (?,?,?,?,?,?,?,?,?)"); + + LOGGER.debug(String.format("Insert statement is '%s'", sb)); + this.jdbcTemplate.update(sb.toString(), + value(map.get(Cluster.CLUSTER)), + value(map.get(Cluster.REGION)), + emailValue(map.get(Cluster.OWNER_EMAIL)), + value(map.get(Cluster.IS_CONFORMING)), + value(map.get(Cluster.IS_OPTEDOUT)), + value(cluster.getUpdateTime()), + value(map.get(Cluster.EXCLUDED_RULES)), + value(conformityJson), + value(map.get(Cluster.CONFORMITY_RULES))); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("update ").append(table).append(" set "); + sb.append(Cluster.OWNER_EMAIL).append("=?,"); + sb.append(Cluster.IS_CONFORMING).append("=?,"); + sb.append(Cluster.IS_OPTEDOUT).append("=?,"); + sb.append(Cluster.UPDATE_TIMESTAMP).append("=?,"); + sb.append(Cluster.EXCLUDED_RULES).append("=?,"); + sb.append("conformities").append("=?,"); + sb.append(Cluster.CONFORMITY_RULES).append("=? where "); + sb.append(Cluster.CLUSTER).append("=? and "); + sb.append(Cluster.REGION).append("=?"); + + LOGGER.debug(String.format("Update statement is '%s'", sb)); + this.jdbcTemplate.update(sb.toString(), + emailValue(map.get(Cluster.OWNER_EMAIL)), + value(map.get(Cluster.IS_CONFORMING)), + value(map.get(Cluster.IS_OPTEDOUT)), + value(cluster.getUpdateTime()), + value(map.get(Cluster.EXCLUDED_RULES)), + value(conformityJson), + value(map.get(Cluster.CONFORMITY_RULES)), + value(cluster.getName()), + value(cluster.getRegion())); + } + LOGGER.debug("Successfully saved."); + } + + private HashMap conformitiesAsMap(Cluster cluster) { + HashMap map = new HashMap<>(); + + for(Conformity conformity : cluster.getConformties()) { + map.put(conformity.getRuleId(), StringUtils.join(conformity.getFailedComponents(), ",")); + } + + return map; + } + + /** + * Gets the clusters for a list of regions. If the regions parameter is empty, returns the clusters + * for all regions. + */ + @Override + public List getAllClusters(String... regions) { + return getClusters(null, regions); + } + + @Override + public List getNonconformingClusters(String... regions) { + return getClusters(false, regions); + } + + @Override + public Cluster getCluster(String clusterName, String region) { + Validate.notEmpty(clusterName); + Validate.notEmpty(region); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where cluster = ? and region = ?", table)); + LOGGER.info(String.format("Query is '%s'", query)); + + List clusters = jdbcTemplate.query(query.toString(), new String[] {clusterName, region}, new RowMapper() { + public Cluster mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapResource(rs); + } + }); + Validate.isTrue(clusters.size() <= 1); + if (clusters.size() == 0) { + LOGGER.info(String.format("Not found cluster with name %s in region %s", clusterName, region)); + return null; + } else { + Cluster cluster = clusters.get(0); + return cluster; + } + } + + private Cluster mapResource(ResultSet rs) throws SQLException { + Map map = conformityMapFromJson(rs.getString("conformities")); + map.put(Cluster.CLUSTER, rs.getString(Cluster.CLUSTER)); + map.put(Cluster.REGION, rs.getString(Cluster.REGION)); + map.put(Cluster.IS_CONFORMING, rs.getString(Cluster.IS_CONFORMING)); + map.put(Cluster.IS_OPTEDOUT, rs.getString(Cluster.IS_OPTEDOUT)); + + String email = rs.getString(Cluster.OWNER_EMAIL); + if (StringUtils.isBlank(email) || email.equals("0")) { + email = null; + } + map.put(Cluster.OWNER_EMAIL, email); + + String updatedTimestamp = millisToFormattedDate(rs.getString(Cluster.UPDATE_TIMESTAMP)); + if (updatedTimestamp != null) { + map.put(Cluster.UPDATE_TIMESTAMP, updatedTimestamp); + } + + map.put(Cluster.EXCLUDED_RULES, rs.getString(Cluster.EXCLUDED_RULES)); + map.put(Cluster.CONFORMITY_RULES, rs.getString(Cluster.CONFORMITY_RULES)); + return Cluster.parseFieldToValueMap(map); + } + + private String millisToFormattedDate(String millisStr) { + String datetime = null; + try { + long millis = Long.parseLong(millisStr); + datetime = AWSResource.DATE_FORMATTER.print(millis); + } catch(NumberFormatException nfe) { + LOGGER.error(String.format("Error parsing datetime %s when reading from RDS", millisStr)); + } + return datetime; + } + + private HashMap conformityMapFromJson(String json) throws SQLException { + HashMap map = new HashMap<>(); + + if (json != null) { + TypeReference> typeRef = new TypeReference>() {}; + + try { + ObjectMapper mapper = new ObjectMapper(); + map = mapper.readValue(json, typeRef); + }catch(IOException ie) { + String msg = "Error parsing conformities from result set"; + LOGGER.error(msg, ie); + throw new SQLException(msg); + } + } + return map; + } + + @Override + public void deleteClusters(Cluster... clusters) { + Validate.notNull(clusters); + LOGGER.info(String.format("Deleting %d clusters", clusters.length)); + for (Cluster cluster : clusters) { + LOGGER.info(String.format("Deleting cluster %s", cluster.getName())); + String stmt = String.format("delete from %s where %s=? and %s=?", table, Cluster.CLUSTER, Cluster.REGION); + jdbcTemplate.update(stmt, cluster.getName(), cluster.getRegion()); + LOGGER.info(String.format("Successfully deleted cluster %s", cluster.getName())); + } + } + + private List getClusters(Boolean conforming, String... regions) { + Validate.notNull(regions); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where cluster is not null and ", table)); + boolean needsAnd = false; + if (regions.length != 0) { + query.append(String.format("region in ('%s') ", StringUtils.join(regions, "','"))); + needsAnd = true; + } + if (conforming != null) { + if (needsAnd) { + query.append(" and "); + } + query.append(String.format("isConforming = '%s'", conforming)); + } + + LOGGER.info(String.format("Query to retrieve clusters for regions %s is '%s'", + StringUtils.join(regions, "','"), query.toString())); + + List clusters = jdbcTemplate.query(query.toString(), new RowMapper() { + public Cluster mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapResource(rs); + } + }); + + LOGGER.info(String.format("Retrieved %d clusters from RDS DB in table %s and regions %s", + clusters.size(), table, StringUtils.join(regions, "','"))); + return clusters; + } + + /** + * Creates the RDS table, if it does not already exist. + */ + public void init() { + try { + LOGGER.info("Creating RDS table: {}", table); + String sql = String.format("create table if not exists %s (" + + " %s varchar(255)," + + " %s varchar(25)," + + " %s varchar(255)," + + " %s varchar(10)," + + " %s varchar(10)," + + " %s BIGINT," + + " %s varchar(4096)," + + " %s varchar(4096)," + + " %s varchar(4096) )", + table, + Cluster.CLUSTER, + Cluster.REGION, + Cluster.OWNER_EMAIL, + Cluster.IS_CONFORMING, + Cluster.IS_OPTEDOUT, + Cluster.UPDATE_TIMESTAMP, + Cluster.EXCLUDED_RULES, + "conformities", + Cluster.CONFORMITY_RULES); + LOGGER.debug("Create SQL is: '{}'", sql); + jdbcTemplate.execute(sql); + + } catch (AmazonClientException e) { + LOGGER.warn("Error while trying to auto-create RDS table", e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java new file mode 100644 index 00000000..130d6235 --- /dev/null +++ b/src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java @@ -0,0 +1,365 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.simianarmy.aws.janitor; + +import com.amazonaws.AmazonClientException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.Resource.CleanupState; +import com.netflix.simianarmy.ResourceType; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.janitor.JanitorResourceTracker; +import com.zaxxer.hikari.HikariDataSource; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; + +import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.*; + +/** + * The JanitorResourceTracker implementation in AWS RDS. + */ +public class RDSJanitorResourceTracker implements JanitorResourceTracker { + + /** The Constant LOGGER. */ + public static final Logger LOGGER = LoggerFactory.getLogger(RDSJanitorResourceTracker.class); + + /** The table. */ + private final String table; + + /** the jdbcTemplate */ + JdbcTemplate jdbcTemplate = null; + + /** + * Instantiates a new RDS janitor resource tracker. + * + */ + public RDSJanitorResourceTracker(String dbDriver, String dbUser, + String dbPass, String dbUrl, String dbTable) { + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dbDriver); + dataSource.setJdbcUrl(dbUrl); + dataSource.setUsername(dbUser); + dataSource.setPassword(dbPass); + this.jdbcTemplate = new JdbcTemplate(dataSource); + this.table = dbTable; + } + + /** + * Instantiates a new RDS janitor resource tracker. This constructor is intended + * for unit testing. + * + */ + public RDSJanitorResourceTracker(JdbcTemplate jdbcTemplate, String table) { + this.jdbcTemplate = jdbcTemplate; + this.table = table; + } + + public JdbcTemplate getJdbcTemplate() { + return jdbcTemplate; + } + + public Object value(String value) { + return value == null ? Types.NULL : value; + } + + public Object value(Date value) { + return value == null ? Types.NULL : value.getTime(); + } + + public Object value(boolean value) { + return new Boolean(value).toString(); + } + + public Object emailValue(String email) { + if (StringUtils.isBlank(email)) return Types.NULL; + if (email.equals("0")) return Types.NULL; + return email; + } + + /** {@inheritDoc} */ + @Override + public void addOrUpdate(Resource resource) { + Resource orig = getResource(resource.getId()); + LOGGER.debug(String.format("Saving resource %s to RDB table %s", resource.getId(), table)); + String json; + try { + json = new ObjectMapper().writeValueAsString(additionalFieldsAsMap(resource)); + } catch (JsonProcessingException e) { + LOGGER.error("ERROR generating additonal field JSON when saving resource " + resource.getId(), e); + return; + } + + if (orig == null) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table); + sb.append(" ("); + sb.append(AWSResource.FIELD_RESOURCE_ID).append(","); + sb.append(AWSResource.FIELD_RESOURCE_TYPE).append(","); + sb.append(AWSResource.FIELD_REGION).append(","); + sb.append(AWSResource.FIELD_OWNER_EMAIL).append(","); + sb.append(AWSResource.FIELD_DESCRIPTION).append(","); + sb.append(AWSResource.FIELD_STATE).append(","); + sb.append(AWSResource.FIELD_TERMINATION_REASON).append(","); + sb.append(AWSResource.FIELD_EXPECTED_TERMINATION_TIME).append(","); + sb.append(AWSResource.FIELD_ACTUAL_TERMINATION_TIME).append(","); + sb.append(AWSResource.FIELD_NOTIFICATION_TIME).append(","); + sb.append(AWSResource.FIELD_LAUNCH_TIME).append(","); + sb.append(AWSResource.FIELD_MARK_TIME).append(","); + sb.append(AWSResource.FIELD_OPT_OUT_OF_JANITOR).append(","); + sb.append("additionalFields").append(") values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); + + LOGGER.debug(String.format("Insert statement is '%s'", sb)); + int updated = this.jdbcTemplate.update(sb.toString(), + resource.getId(), + value(resource.getResourceType().toString()), + value(resource.getRegion()), + emailValue(resource.getOwnerEmail()), + value(resource.getDescription()), + value(resource.getState().toString()), + value(resource.getTerminationReason()), + value(resource.getExpectedTerminationTime()), + value(resource.getActualTerminationTime()), + value(resource.getNotificationTime()), + value(resource.getLaunchTime()), + value(resource.getMarkTime()), + value(resource.isOptOutOfJanitor()), + json); + LOGGER.debug(String.format("%d rows inserted", updated)); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("update ").append(table).append(" set "); + sb.append(AWSResource.FIELD_RESOURCE_TYPE).append("=?,"); + sb.append(AWSResource.FIELD_REGION).append("=?,"); + sb.append(AWSResource.FIELD_OWNER_EMAIL).append("=?,"); + sb.append(AWSResource.FIELD_DESCRIPTION).append("=?,"); + sb.append(AWSResource.FIELD_STATE).append("=?,"); + sb.append(AWSResource.FIELD_TERMINATION_REASON).append("=?,"); + sb.append(AWSResource.FIELD_EXPECTED_TERMINATION_TIME).append("=?,"); + sb.append(AWSResource.FIELD_ACTUAL_TERMINATION_TIME).append("=?,"); + sb.append(AWSResource.FIELD_NOTIFICATION_TIME).append("=?,"); + sb.append(AWSResource.FIELD_LAUNCH_TIME).append("=?,"); + sb.append(AWSResource.FIELD_MARK_TIME).append("=?,"); + sb.append(AWSResource.FIELD_OPT_OUT_OF_JANITOR).append("=?,"); + sb.append("additionalFields").append("=? where "); + sb.append(AWSResource.FIELD_RESOURCE_ID).append("=?"); + + LOGGER.debug(String.format("Update statement is '%s'", sb)); + int updated = this.jdbcTemplate.update(sb.toString(), + resource.getResourceType().toString(), + value(resource.getRegion()), + emailValue(resource.getOwnerEmail()), + value(resource.getDescription()), + value(resource.getState().toString()), + value(resource.getTerminationReason()), + value(resource.getExpectedTerminationTime()), + value(resource.getActualTerminationTime()), + value(resource.getNotificationTime()), + value(resource.getLaunchTime()), + value(resource.getMarkTime()), + value(resource.isOptOutOfJanitor()), + json, + resource.getId()); + LOGGER.debug(String.format("%d rows updated", updated)); + } + LOGGER.debug("Successfully saved."); + } + + /** + * Returns a list of AWSResource objects. You need to override this method if more + * specific resource types (e.g. subtypes of AWSResource) need to be obtained from + * the Database. + */ + @Override + public List getResources(ResourceType resourceType, CleanupState state, String resourceRegion) { + Validate.notEmpty(resourceRegion); + StringBuilder query = new StringBuilder(); + ArrayList args = new ArrayList<>(); + query.append(String.format("select * from %s where ", table)); + if (resourceType != null) { + query.append("resourceType=? and "); + args.add(resourceType.toString()); + } + if (state != null) { + query.append("state=? and "); + args.add(state.toString()); + } + query.append("region=?"); + args.add(resourceRegion); + + LOGGER.debug(String.format("Query is '%s'", query)); + List resources = jdbcTemplate.query(query.toString(), args.toArray(), new RowMapper() { + public Resource mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapResource(rs); + } + }); + return resources; + } + + private Resource mapResource(ResultSet rs) throws SQLException { + String json = rs.getString("additionalFields"); + Resource resource = null; + try { + // put additional fields + Map map = new HashMap<>(); + if (json != null) { + TypeReference> typeRef = new TypeReference>() {}; + map = new ObjectMapper().readValue(json, typeRef); + } + + // put everything else + map.put(AWSResource.FIELD_RESOURCE_ID, rs.getString(AWSResource.FIELD_RESOURCE_ID)); + map.put(AWSResource.FIELD_RESOURCE_TYPE, rs.getString(AWSResource.FIELD_RESOURCE_TYPE)); + map.put(AWSResource.FIELD_REGION, rs.getString(AWSResource.FIELD_REGION)); + map.put(AWSResource.FIELD_DESCRIPTION, rs.getString(AWSResource.FIELD_DESCRIPTION)); + map.put(AWSResource.FIELD_STATE, rs.getString(AWSResource.FIELD_STATE)); + map.put(AWSResource.FIELD_TERMINATION_REASON, rs.getString(AWSResource.FIELD_TERMINATION_REASON)); + map.put(AWSResource.FIELD_OPT_OUT_OF_JANITOR, rs.getString(AWSResource.FIELD_OPT_OUT_OF_JANITOR)); + + String email = rs.getString(AWSResource.FIELD_OWNER_EMAIL); + if (StringUtils.isBlank(email) || email.equals("0")) { + email = null; + } + map.put(AWSResource.FIELD_OWNER_EMAIL, email); + + String expectedTerminationTime = millisToFormattedDate(rs.getString(AWSResource.FIELD_EXPECTED_TERMINATION_TIME)); + String actualTerminationTime = millisToFormattedDate(rs.getString(AWSResource.FIELD_ACTUAL_TERMINATION_TIME)); + String notificationTime = millisToFormattedDate(rs.getString(AWSResource.FIELD_NOTIFICATION_TIME)); + String launchTime = millisToFormattedDate(rs.getString(AWSResource.FIELD_LAUNCH_TIME)); + String markTime = millisToFormattedDate(rs.getString(AWSResource.FIELD_MARK_TIME)); + + if (expectedTerminationTime != null) { + map.put(AWSResource.FIELD_EXPECTED_TERMINATION_TIME, expectedTerminationTime); + } + if (actualTerminationTime != null) { + map.put(AWSResource.FIELD_ACTUAL_TERMINATION_TIME, actualTerminationTime); + } + if (notificationTime != null) { + map.put(AWSResource.FIELD_NOTIFICATION_TIME, notificationTime); + } + if (launchTime != null) { + map.put(AWSResource.FIELD_LAUNCH_TIME, launchTime); + } + if (markTime != null) { + map.put(AWSResource.FIELD_MARK_TIME, markTime); + } + + resource = AWSResource.parseFieldtoValueMap(map); + }catch(IOException ie) { + String msg = "Error parsing resource from result set"; + LOGGER.error(msg, ie); + throw new SQLException(msg); + } + return resource; + } + + private String millisToFormattedDate(String millisStr) { + String datetime = null; + try { + long millis = Long.parseLong(millisStr); + datetime = AWSResource.DATE_FORMATTER.print(millis); + } catch(NumberFormatException nfe) { + LOGGER.error(String.format("Error parsing datetime %s when reading from RDS", millisStr)); + } + return datetime; + } + + @Override + public Resource getResource(String resourceId) { + Validate.notEmpty(resourceId); + StringBuilder query = new StringBuilder(); + query.append(String.format("select * from %s where resourceId=?", table)); + + LOGGER.debug(String.format("Query is '%s'", query)); + List resources = jdbcTemplate.query(query.toString(), new String[]{resourceId}, new RowMapper() { + public Resource mapRow(ResultSet rs, int rowNum) throws SQLException { + return mapResource(rs); + } + }); + + Resource resource = null; + Validate.isTrue(resources.size() <= 1); + if (resources.size() == 0) { + LOGGER.info(String.format("Not found resource with id %s", resourceId)); + } else { + resource = resources.get(0); + } + return resource; + } + + /** + * Creates the RDS table, if it does not already exist. + */ + public void init() { + try { + LOGGER.info("Creating RDS table: {}", table); + String sql = String.format("create table if not exists %s (" + + " %s varchar(255), " + + " %s varchar(255), " + + " %s varchar(25), " + + " %s varchar(255), " + + " %s varchar(255), " + + " %s varchar(25), " + + " %s varchar(255), " + + " %s BIGINT, " + + " %s BIGINT, " + + " %s BIGINT, " + + " %s BIGINT, " + + " %s BIGINT, " + + " %s varchar(8), " + + " %s varchar(4096) )", + table, + AWSResource.FIELD_RESOURCE_ID, + AWSResource.FIELD_RESOURCE_TYPE, + AWSResource.FIELD_REGION, + AWSResource.FIELD_OWNER_EMAIL, + AWSResource.FIELD_DESCRIPTION, + AWSResource.FIELD_STATE, + AWSResource.FIELD_TERMINATION_REASON, + AWSResource.FIELD_EXPECTED_TERMINATION_TIME, + AWSResource.FIELD_ACTUAL_TERMINATION_TIME, + AWSResource.FIELD_NOTIFICATION_TIME, + AWSResource.FIELD_LAUNCH_TIME, + AWSResource.FIELD_MARK_TIME, + AWSResource.FIELD_OPT_OUT_OF_JANITOR, + "additionalFields"); + LOGGER.debug("Create SQL is: '{}'", sql); + jdbcTemplate.execute(sql); + + } catch (AmazonClientException e) { + LOGGER.warn("Error while trying to auto-create RDS table", e); + } + } + + private HashMap additionalFieldsAsMap(Resource resource) { + HashMap fields = new HashMap<>(); + for(String key : resource.getAdditionalFieldNames()) { + fields.put(key, resource.getAdditionalField(key)); + } + return fields; + } +} diff --git a/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java b/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java index 712e4d5d..d47f9593 100644 --- a/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java +++ b/src/main/java/com/netflix/simianarmy/basic/BasicSimianArmyContext.java @@ -17,23 +17,11 @@ */ package com.netflix.simianarmy.basic; -import java.io.InputStream; -import java.lang.reflect.Constructor; -import java.util.LinkedList; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; - import com.netflix.simianarmy.CloudClient; import com.netflix.simianarmy.Monkey; import com.netflix.simianarmy.MonkeyCalendar; @@ -41,9 +29,20 @@ import com.netflix.simianarmy.MonkeyRecorder; import com.netflix.simianarmy.MonkeyRecorder.Event; import com.netflix.simianarmy.MonkeyScheduler; -import com.netflix.simianarmy.aws.SimpleDBRecorder; +import com.netflix.simianarmy.aws.RDSRecorder; import com.netflix.simianarmy.aws.STSAssumeRoleSessionCredentialsProvider; +import com.netflix.simianarmy.aws.SimpleDBRecorder; import com.netflix.simianarmy.client.aws.AWSClient; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.util.LinkedList; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.concurrent.TimeUnit; /** * The Class BasicSimianArmyContext. @@ -156,6 +155,7 @@ protected BasicSimianArmyContext(String... configFiles) { assumeRoleArn = config.getStr("simianarmy.client.aws.assumeRoleArn"); if (assumeRoleArn != null) { this.awsCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(assumeRoleArn, awsClientConfig); + LOGGER.info("Using STSAssumeRoleSessionCredentialsProvider with assume role " + assumeRoleArn); } // if credentials are set explicitly make them available to the AWS SDK @@ -187,6 +187,7 @@ protected boolean isSafeToLog(Object propertyKey) { protected void loadConfigurationFileIntoProperties(String propertyFileName) { String propFile = System.getProperty(propertyFileName, "/" + propertyFileName); try { + LOGGER.info("loading properties file: " + propFile); InputStream is = BasicSimianArmyContext.class.getResourceAsStream(propFile); try { properties.load(is); @@ -212,7 +213,17 @@ private void createScheduler() { private void createRecorder() { @SuppressWarnings("rawtypes") Class recorderClass = loadClientClass("simianarmy.client.recorder.class"); - if (recorderClass == null || recorderClass.equals(SimpleDBRecorder.class)) { + if (recorderClass != null && recorderClass.equals(RDSRecorder.class)) { + String dbDriver = configuration().getStr("simianarmy.recorder.db.driver"); + String dbUser = configuration().getStr("simianarmy.recorder.db.user"); + String dbPass = configuration().getStr("simianarmy.recorder.db.pass"); + String dbUrl = configuration().getStr("simianarmy.recorder.db.url"); + String dbTable = configuration().getStr("simianarmy.recorder.db.table"); + + RDSRecorder rdsRecorder = new RDSRecorder(dbDriver, dbUser, dbPass, dbUrl, dbTable, client.region()); + rdsRecorder.init(); + setRecorder(rdsRecorder); + } else if (recorderClass == null || recorderClass.equals(SimpleDBRecorder.class)) { String domain = config.getStrOrElse("simianarmy.recorder.sdb.domain", "SIMIAN_ARMY"); if (client != null) { SimpleDBRecorder simpleDbRecorder = new SimpleDBRecorder(client, domain); diff --git a/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java index a266fd75..43e5e679 100644 --- a/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java +++ b/src/main/java/com/netflix/simianarmy/basic/conformity/BasicConformityMonkeyContext.java @@ -17,6 +17,13 @@ // CHECKSTYLE IGNORE MagicNumberCheck package com.netflix.simianarmy.basic.conformity; +import java.util.Collection; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; @@ -26,6 +33,7 @@ import com.google.inject.Injector; import com.netflix.discovery.DiscoveryClient; import com.netflix.discovery.guice.EurekaModule; +import com.netflix.simianarmy.aws.conformity.RDSConformityClusterTracker; import com.netflix.simianarmy.aws.conformity.SimpleDBConformityClusterTracker; import com.netflix.simianarmy.aws.conformity.crawler.AWSClusterCrawler; import com.netflix.simianarmy.aws.conformity.rule.BasicConformityEurekaClient; @@ -47,12 +55,6 @@ import com.netflix.simianarmy.conformity.ConformityMonkey; import com.netflix.simianarmy.conformity.ConformityRule; import com.netflix.simianarmy.conformity.ConformityRuleEngine; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.Map; /** * The basic implementation of the context class for Conformity monkey. @@ -101,7 +103,19 @@ public BasicConformityMonkeyContext() { String sdbDomain = configuration().getStrOrElse("simianarmy.conformity.sdb.domain", "SIMIAN_ARMY"); - clusterTracker = new SimpleDBConformityClusterTracker(awsClient(), sdbDomain); + String dbDriver = configuration().getStr("simianarmy.recorder.db.driver"); + String dbUser = configuration().getStr("simianarmy.recorder.db.user"); + String dbPass = configuration().getStr("simianarmy.recorder.db.pass"); + String dbUrl = configuration().getStr("simianarmy.recorder.db.url"); + String dbTable = configuration().getStr("simianarmy.conformity.resources.db.table"); + + if (dbDriver == null) { + clusterTracker = new SimpleDBConformityClusterTracker(awsClient(), sdbDomain); + } else { + RDSConformityClusterTracker rdsClusterTracker = new RDSConformityClusterTracker(dbDriver, dbUser, dbPass, dbUrl, dbTable); + rdsClusterTracker.init(); + clusterTracker = rdsClusterTracker; + } ruleEngine = new ConformityRuleEngine(); boolean eurekaEnabled = configuration().getBoolOrElse("simianarmy.conformity.Eureka.enabled", false); diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java index fecb702b..c3a25a99 100644 --- a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java @@ -17,27 +17,22 @@ */ package com.netflix.simianarmy.basic.janitor; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.netflix.servo.annotations.DataSourceType; import com.netflix.servo.annotations.Monitor; import com.netflix.servo.monitor.Monitors; -import com.netflix.simianarmy.MonkeyCalendar; -import com.netflix.simianarmy.MonkeyConfiguration; -import com.netflix.simianarmy.MonkeyRecorder; +import com.netflix.simianarmy.*; import com.netflix.simianarmy.MonkeyRecorder.Event; -import com.netflix.simianarmy.Resource; -import com.netflix.simianarmy.ResourceType; import com.netflix.simianarmy.janitor.AbstractJanitor; import com.netflix.simianarmy.janitor.JanitorEmailNotifier; import com.netflix.simianarmy.janitor.JanitorMonkey; import com.netflix.simianarmy.janitor.JanitorResourceTracker; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; /** The basic implementation of Janitor Monkey. */ public class BasicJanitorMonkey extends JanitorMonkey { @@ -140,7 +135,7 @@ public void doMonkeyBusiness() { janitor.cleanupResources(); } catch (Exception e) { monkeyErrors.incrementAndGet(); - LOGGER.error(String.format("Got an exception while %s janitor was cleaning for region %s", janitor.getResourceType()), janitor.getRegion(), e); + LOGGER.error(String.format("Got an exception while %s janitor was cleaning for region %s", janitor.getResourceType(), janitor.getRegion()), e); } LOGGER.info(String.format("Cleaned %d resources of type %s in the last run.", janitor.getCleanedResources().size(), janitor.getResourceType())); diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java index b9af7900..4f258176 100644 --- a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java +++ b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkeyContext.java @@ -17,6 +17,15 @@ // CHECKSTYLE IGNORE MagicNumberCheck package com.netflix.simianarmy.basic.janitor; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; @@ -33,6 +42,7 @@ import com.netflix.simianarmy.aws.janitor.ImageJanitor; import com.netflix.simianarmy.aws.janitor.InstanceJanitor; import com.netflix.simianarmy.aws.janitor.LaunchConfigJanitor; +import com.netflix.simianarmy.aws.janitor.RDSJanitorResourceTracker; import com.netflix.simianarmy.aws.janitor.SimpleDBJanitorResourceTracker; import com.netflix.simianarmy.aws.janitor.crawler.ASGJanitorCrawler; import com.netflix.simianarmy.aws.janitor.crawler.EBSSnapshotJanitorCrawler; @@ -66,14 +76,6 @@ import com.netflix.simianarmy.janitor.JanitorMonkey; import com.netflix.simianarmy.janitor.JanitorResourceTracker; import com.netflix.simianarmy.janitor.JanitorRuleEngine; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; /** * The basic implementation of the context class for Janitor monkey. @@ -122,7 +124,19 @@ public BasicJanitorMonkeyContext() { Set enabledResourceSet = getEnabledResourceSet(); - janitorResourceTracker = new SimpleDBJanitorResourceTracker(awsClient(), resourceDomain); + String dbDriver = configuration().getStr("simianarmy.recorder.db.driver"); + String dbUser = configuration().getStr("simianarmy.recorder.db.user"); + String dbPass = configuration().getStr("simianarmy.recorder.db.pass"); + String dbUrl = configuration().getStr("simianarmy.recorder.db.url"); + String dbTable = configuration().getStr("simianarmy.janitor.resources.db.table"); + + if (dbDriver == null) { + janitorResourceTracker = new SimpleDBJanitorResourceTracker(awsClient(), resourceDomain); + } else { + RDSJanitorResourceTracker rdsTracker = new RDSJanitorResourceTracker(dbDriver, dbUser, dbPass, dbUrl, dbTable); + rdsTracker.init(); + janitorResourceTracker = rdsTracker; + } janitorEmailBuilder = new BasicJanitorEmailBuilder(); sesClient = new AmazonSimpleEmailServiceClient(); diff --git a/src/main/java/com/netflix/simianarmy/conformity/Cluster.java b/src/main/java/com/netflix/simianarmy/conformity/Cluster.java index ba709e5e..8f3dfb4c 100644 --- a/src/main/java/com/netflix/simianarmy/conformity/Cluster.java +++ b/src/main/java/com/netflix/simianarmy/conformity/Cluster.java @@ -37,15 +37,15 @@ * a group of ASGs that belong to the same application, for example, a cluster in the Asgard deployment system. */ public class Cluster { - private static final String OWNER_EMAIL = "ownerEmail"; - private static final String CLUSTER = "cluster"; - private static final String REGION = "region"; - private static final String IS_CONFORMING = "isConforming"; - private static final String IS_OPTEDOUT = "isOptedOut"; - private static final String UPDATE_TIMESTAMP = "updateTimestamp"; - private static final String EXCLUDED_RULES = "excludedRules"; - private static final String CONFORMITY_RULES = "conformityRules"; - private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); + public static final String OWNER_EMAIL = "ownerEmail"; + public static final String CLUSTER = "cluster"; + public static final String REGION = "region"; + public static final String IS_CONFORMING = "isConforming"; + public static final String IS_OPTEDOUT = "isOptedOut"; + public static final String UPDATE_TIMESTAMP = "updateTimestamp"; + public static final String EXCLUDED_RULES = "excludedRules"; + public static final String CONFORMITY_RULES = "conformityRules"; + public static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"); private final String name; private final Collection autoScalingGroups = Lists.newArrayList(); diff --git a/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java b/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java index a6cc8bc3..52a09441 100644 --- a/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java +++ b/src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java @@ -17,18 +17,9 @@ */ package com.netflix.simianarmy.resources.janitor; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Map; - -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; - +import com.netflix.simianarmy.MonkeyRecorder.Event; +import com.netflix.simianarmy.MonkeyRunner; +import com.netflix.simianarmy.janitor.JanitorMonkey; import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.JsonEncoding; import org.codehaus.jackson.JsonGenerator; @@ -38,9 +29,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.netflix.simianarmy.MonkeyRecorder.Event; -import com.netflix.simianarmy.MonkeyRunner; -import com.netflix.simianarmy.janitor.JanitorMonkey; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; /** * The Class JanitorMonkeyResource for json REST apis. @@ -52,7 +50,7 @@ public class JanitorMonkeyResource { private static final MappingJsonFactory JSON_FACTORY = new MappingJsonFactory(); /** The monkey. */ - private final JanitorMonkey monkey; + private static JanitorMonkey monkey; /** The Constant LOGGER. */ private static final Logger LOGGER = LoggerFactory.getLogger(JanitorMonkeyResource.class); @@ -64,16 +62,18 @@ public class JanitorMonkeyResource { * the janitor monkey */ public JanitorMonkeyResource(JanitorMonkey monkey) { - this.monkey = monkey; + JanitorMonkeyResource.monkey = monkey; } /** * Instantiates a janitor monkey resource using a registered janitor monkey from factory. */ public JanitorMonkeyResource() { - this.monkey = MonkeyRunner.getInstance().factory(JanitorMonkey.class); + if (JanitorMonkeyResource.monkey == null ) { + JanitorMonkeyResource.monkey = MonkeyRunner.getInstance().factory(JanitorMonkey.class); + } } - + /** * GET /api/v1/janitor/addEvent will try to a add a new event with the information in the url query string. * This is the same as the regular POST addEvent except through a query string. This technically isn't diff --git a/src/test/java/com/netflix/simianarmy/aws/TestRDSRecorder.java b/src/test/java/com/netflix/simianarmy/aws/TestRDSRecorder.java new file mode 100644 index 00000000..dabb4afc --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/TestRDSRecorder.java @@ -0,0 +1,165 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// CHECKSTYLE IGNORE Javadoc +package com.netflix.simianarmy.aws; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.EventType; +import com.netflix.simianarmy.MonkeyType; +import com.netflix.simianarmy.basic.BasicRecorderEvent; + +// CHECKSTYLE IGNORE MagicNumberCheck +public class TestRDSRecorder extends RDSRecorder { + + public TestRDSRecorder() { + super(mock(JdbcTemplate.class), "recordertable", "us-west-1"); + } + + public enum Type implements MonkeyType { + MONKEY + } + + public enum EventTypes implements EventType { + EVENT + } + + @Test + public void testInit() { + TestRDSRecorder recorder = new TestRDSRecorder(); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + Mockito.doNothing().when(recorder.getJdbcTemplate()).execute(sqlCap.capture()); + recorder.init(); + Assert.assertEquals(sqlCap.getValue(), "create table if not exists recordertable ( eventId varchar(255), eventTime BIGINT, monkeyType varchar(255), eventType varchar(255), region varchar(255), dataJson varchar(4096) )"); + } + + @SuppressWarnings("unchecked") + @Test + public void testInsertNewRecordEvent() { + // mock the select query that is issued to see if the record already exists + ArrayList events = new ArrayList<>(); + TestRDSRecorder recorder = new TestRDSRecorder(); + when(recorder.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(events); + + Event evt = newEvent(Type.MONKEY, EventTypes.EVENT, "region", "testId"); + evt.addField("field1", "value1"); + evt.addField("field2", "value2"); + + // this will be ignored as it conflicts with reserved key + evt.addField("id", "ignoreThis"); + + ArgumentCaptor objCap = ArgumentCaptor.forClass(Object.class); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + when(recorder.getJdbcTemplate().update(sqlCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture())).thenReturn(1); + recorder.recordEvent(evt); + List args = objCap.getAllValues(); + Assert.assertEquals(sqlCap.getValue(), "insert into recordertable (eventId,eventTime,monkeyType,eventType,region,dataJson) values (?,?,?,?,?,?)"); + Assert.assertEquals(args.size(), 6); + Assert.assertEquals(args.get(0).toString(), evt.id()); + Assert.assertEquals(args.get(1).toString(), evt.eventTime().getTime() + ""); + Assert.assertEquals(args.get(2).toString(), SimpleDBRecorder.enumToValue(evt.monkeyType())); + Assert.assertEquals(args.get(3).toString(), SimpleDBRecorder.enumToValue(evt.eventType())); + Assert.assertEquals(args.get(4).toString(), evt.region()); + } + + private Event mkSelectResult(String id, Event evt) { + evt.addField("field1", "value1"); + evt.addField("field2", "value2"); + return evt; + } + + @SuppressWarnings("unchecked") + @Test + public void testFindEvent() { + Event evt1 = new BasicRecorderEvent(Type.MONKEY, EventTypes.EVENT, "region", "testId1", 1330538400000L); + mkSelectResult("testId1", evt1); + Event evt2 = new BasicRecorderEvent(Type.MONKEY, EventTypes.EVENT, "region", "testId2", 1330538400000L); + mkSelectResult("testId2", evt2); + + ArrayList events = new ArrayList<>(); + TestRDSRecorder recorder = new TestRDSRecorder(); + events.add(evt1); + events.add(evt2); + when(recorder.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(events); + + Map query = new LinkedHashMap(); + query.put("instanceId", "testId1"); + + verifyEvents(recorder.findEvents(query, new Date(0))); + } + + void verifyEvents(List events) { + Assert.assertEquals(events.size(), 2); + + Assert.assertEquals(events.get(0).id(), "testId1"); + Assert.assertEquals(events.get(0).eventTime().getTime(), 1330538400000L); + Assert.assertEquals(events.get(0).monkeyType(), Type.MONKEY); + Assert.assertEquals(events.get(0).eventType(), EventTypes.EVENT); + Assert.assertEquals(events.get(0).field("field1"), "value1"); + Assert.assertEquals(events.get(0).field("field2"), "value2"); + Assert.assertEquals(events.get(0).fields().size(), 2); + + Assert.assertEquals(events.get(1).id(), "testId2"); + Assert.assertEquals(events.get(1).eventTime().getTime(), 1330538400000L); + Assert.assertEquals(events.get(1).monkeyType(), Type.MONKEY); + Assert.assertEquals(events.get(1).eventType(), EventTypes.EVENT); + Assert.assertEquals(events.get(1).field("field1"), "value1"); + Assert.assertEquals(events.get(1).field("field2"), "value2"); + Assert.assertEquals(events.get(1).fields().size(), 2); + } + + @SuppressWarnings("unchecked") + @Test + public void testFindEventNotFound() { + ArrayList events = new ArrayList<>(); + TestRDSRecorder recorder = new TestRDSRecorder(); + when(recorder.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(events); + + List results = recorder.findEvents(new HashMap(), new Date()); + Assert.assertEquals(results.size(), 0); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/aws/conformity/TestRDSConformityClusterTracker.java b/src/test/java/com/netflix/simianarmy/aws/conformity/TestRDSConformityClusterTracker.java new file mode 100644 index 00000000..cca29bd8 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/conformity/TestRDSConformityClusterTracker.java @@ -0,0 +1,191 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +// CHECKSTYLE IGNORE ParameterNumber +package com.netflix.simianarmy.aws.conformity; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.netflix.simianarmy.conformity.Cluster; +import com.netflix.simianarmy.conformity.Conformity; + +public class TestRDSConformityClusterTracker extends RDSConformityClusterTracker { + public TestRDSConformityClusterTracker() { + super(mock(JdbcTemplate.class), "conformitytable"); + } + + @Test + public void testInit() { + TestRDSConformityClusterTracker recorder = new TestRDSConformityClusterTracker(); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + Mockito.doNothing().when(recorder.getJdbcTemplate()).execute(sqlCap.capture()); + recorder.init(); + Assert.assertEquals(sqlCap.getValue(), "create table if not exists conformitytable ( cluster varchar(255), region varchar(25), ownerEmail varchar(255), isConforming varchar(10), isOptedOut varchar(10), updateTimestamp BIGINT, excludedRules varchar(4096), conformities varchar(4096), conformityRules varchar(4096) )"); + } + + @SuppressWarnings("unchecked") + @Test + public void testInsertNewCluster() { + // mock the select query that is issued to see if the record already exists + ArrayList clusters = new ArrayList<>(); + TestRDSConformityClusterTracker tracker = new TestRDSConformityClusterTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(clusters); + + Cluster cluster1 = new Cluster("clustername1", "us-west-1"); + cluster1.setUpdateTime(new Date()); + ArrayList list = new ArrayList<>(); + list.add("one"); + list.add("two"); + cluster1.updateConformity(new Conformity("rule1",list)); + list.add("three"); + cluster1.updateConformity(new Conformity("rule2",list)); + + ArgumentCaptor objCap = ArgumentCaptor.forClass(Object.class); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + when(tracker.getJdbcTemplate().update(sqlCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture())).thenReturn(1); + tracker.addOrUpdate(cluster1); + + List args = objCap.getAllValues(); + Assert.assertEquals(args.size(), 9); + Assert.assertEquals(args.get(0).toString(), "clustername1"); + Assert.assertEquals(args.get(1).toString(), "us-west-1"); + Assert.assertEquals(args.get(7).toString(), "{\"rule1\":\"one,two\",\"rule2\":\"one,two,three\"}"); + Assert.assertEquals(args.get(8).toString(), "rule1,rule2"); + } + + @SuppressWarnings("unchecked") + @Test + public void testUpdateCluster() { + Cluster cluster1 = new Cluster("clustername1", "us-west-1"); + Date date = new Date(); + cluster1.setUpdateTime(date); + + // mock the select query that is issued to see if the record already exists + ArrayList clusters = new ArrayList<>(); + clusters.add(cluster1); + TestRDSConformityClusterTracker tracker = new TestRDSConformityClusterTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(clusters); + + cluster1.setOwnerEmail("newemail@test.com"); + ArgumentCaptor objCap = ArgumentCaptor.forClass(Object.class); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + when(tracker.getJdbcTemplate().update(sqlCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture())).thenReturn(1); + tracker.addOrUpdate(cluster1); + + List args = objCap.getAllValues(); + Assert.assertEquals(sqlCap.getValue(), "update conformitytable set ownerEmail=?,isConforming=?,isOptedOut=?,updateTimestamp=?,excludedRules=?,conformities=?,conformityRules=? where cluster=? and region=?"); + Assert.assertEquals(args.size(), 9); + Assert.assertEquals(args.get(0).toString(), "newemail@test.com"); + Assert.assertEquals(args.get(1).toString(), "false"); + Assert.assertEquals(args.get(2).toString(), "false"); + Assert.assertEquals(args.get(3).toString(), date.getTime() + ""); + Assert.assertEquals(args.get(7).toString(), "clustername1"); + Assert.assertEquals(args.get(8).toString(), "us-west-1"); + } + + @SuppressWarnings("unchecked") + @Test + public void testGetCluster() { + Cluster cluster1 = new Cluster("clustername1", "us-west-1"); + + ArrayList clusters = new ArrayList<>(); + clusters.add(cluster1); + TestRDSConformityClusterTracker tracker = new TestRDSConformityClusterTracker(); + + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + + when(tracker.getJdbcTemplate().query(sqlCap.capture(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(clusters); + Cluster result = tracker.getCluster("clustername1", "us-west-1"); + Assert.assertNotNull(result); + Assert.assertEquals(sqlCap.getValue(), "select * from conformitytable where cluster = ? and region = ?"); + } + + @SuppressWarnings("unchecked") + @Test + public void testGetClusters() { + Cluster cluster1 = new Cluster("clustername1", "us-west-1"); + Cluster cluster2 = new Cluster("clustername1", "us-west-2"); + Cluster cluster3 = new Cluster("clustername1", "us-east-1"); + + ArrayList clusters = new ArrayList<>(); + clusters.add(cluster1); + clusters.add(cluster2); + clusters.add(cluster3); + TestRDSConformityClusterTracker tracker = new TestRDSConformityClusterTracker(); + + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + + when(tracker.getJdbcTemplate().query(sqlCap.capture(), + Matchers.any(RowMapper.class))).thenReturn(clusters); + List results = tracker.getAllClusters("us-west-1", "us-west-2", "us-east-1"); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(sqlCap.getValue().toString().trim(), "select * from conformitytable where cluster is not null and region in ('us-west-1','us-west-2','us-east-1')"); + } + + @SuppressWarnings("unchecked") + @Test + public void testGetClusterNotFound() { + ArrayList clusters = new ArrayList<>(); + TestRDSConformityClusterTracker tracker = new TestRDSConformityClusterTracker(); + + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(clusters); + Cluster cluster = tracker.getCluster("clustername", "us-west-1"); + Assert.assertNull(cluster); + } + +} diff --git a/src/test/java/com/netflix/simianarmy/aws/janitor/TestRDSJanitorResourceTracker.java b/src/test/java/com/netflix/simianarmy/aws/janitor/TestRDSJanitorResourceTracker.java new file mode 100644 index 00000000..b3a7e7a9 --- /dev/null +++ b/src/test/java/com/netflix/simianarmy/aws/janitor/TestRDSJanitorResourceTracker.java @@ -0,0 +1,351 @@ +/* + * + * Copyright 2012 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +// CHECKSTYLE IGNORE Javadoc +// CHECKSTYLE IGNORE MagicNumberCheck +// CHECKSTYLE IGNORE ParameterNumber +package com.netflix.simianarmy.aws.janitor; + +import com.netflix.simianarmy.Resource; +import com.netflix.simianarmy.aws.AWSResource; +import com.netflix.simianarmy.aws.AWSResourceType; +import org.joda.time.DateTime; +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.*; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestRDSJanitorResourceTracker extends RDSJanitorResourceTracker { + + public TestRDSJanitorResourceTracker() { + super(mock(JdbcTemplate.class), "janitortable"); + } + + @Test + public void testInit() { + TestRDSJanitorResourceTracker recorder = new TestRDSJanitorResourceTracker(); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + Mockito.doNothing().when(recorder.getJdbcTemplate()).execute(sqlCap.capture()); + recorder.init(); + Assert.assertEquals(sqlCap.getValue(), "create table if not exists janitortable ( resourceId varchar(255), resourceType varchar(255), region varchar(25), ownerEmail varchar(255), description varchar(255), state varchar(25), terminationReason varchar(255), expectedTerminationTime BIGINT, actualTerminationTime BIGINT, notificationTime BIGINT, launchTime BIGINT, markTime BIGINT, optOutOfJanitor varchar(8), additionalFields varchar(4096) )"); + } + + @SuppressWarnings("unchecked") + @Test + public void testInsertNewResource() { + // mock the select query that is issued to see if the record already exists + ArrayList resources = new ArrayList<>(); + TestRDSJanitorResourceTracker tracker = new TestRDSJanitorResourceTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(resources); + + String id = "i-12345678901234567"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + Resource resource = new AWSResource().withId(id).withResourceType(resourceType) + .withDescription(description).withOwnerEmail(ownerEmail).withRegion(region) + .withState(state).withTerminationReason(terminationReason) + .withExpectedTerminationTime(expectedTerminationTime) + .withMarkTime(markTime).withOptOutOfJanitor(false) + .setAdditionalField(fieldName, fieldValue); + + ArgumentCaptor objCap = ArgumentCaptor.forClass(Object.class); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + when(tracker.getJdbcTemplate().update(sqlCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture())).thenReturn(1); + tracker.addOrUpdate(resource); + + List args = objCap.getAllValues(); + + Assert.assertEquals(sqlCap.getValue(), "insert into janitortable (resourceId,resourceType,region,ownerEmail,description,state,terminationReason,expectedTerminationTime,actualTerminationTime,notificationTime,launchTime,markTime,optOutOfJanitor,additionalFields) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); + Assert.assertEquals(args.size(), 14); + Assert.assertEquals(args.get(0).toString(), id); + Assert.assertEquals(args.get(1).toString(), AWSResourceType.INSTANCE.toString()); + Assert.assertEquals(args.get(2).toString(), region); + Assert.assertEquals(args.get(3).toString(), ownerEmail); + Assert.assertEquals(args.get(4).toString(), description); + Assert.assertEquals(args.get(5).toString(), state.toString()); + Assert.assertEquals(args.get(6).toString(), terminationReason); + Assert.assertEquals(args.get(7).toString(), expectedTerminationTime.getTime() + ""); + Assert.assertEquals(args.get(8).toString(), "0"); + Assert.assertEquals(args.get(9).toString(), "0"); + Assert.assertEquals(args.get(10).toString(), "0"); + Assert.assertEquals(args.get(11).toString(), markTime.getTime() + ""); + Assert.assertEquals(args.get(12).toString(), "false"); + Assert.assertEquals(args.get(13).toString(), "{\"fieldName123\":\"fieldValue456\"}"); + } + + @SuppressWarnings("unchecked") + @Test + public void testUpdateResource() { + String id = "i-12345678901234567"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + Resource resource = new AWSResource().withId(id).withResourceType(resourceType) + .withDescription(description).withOwnerEmail(ownerEmail).withRegion(region) + .withState(state).withTerminationReason(terminationReason) + .withExpectedTerminationTime(expectedTerminationTime) + .withMarkTime(markTime).withOptOutOfJanitor(false) + .setAdditionalField(fieldName, fieldValue); + + // mock the select query that is issued to see if the record already exists + ArrayList resources = new ArrayList<>(); + resources.add(resource); + TestRDSJanitorResourceTracker tracker = new TestRDSJanitorResourceTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(resources); + + // update the ownerEmail + ownerEmail = "owner2@test.com"; + Resource newResource = new AWSResource().withId(id).withResourceType(resourceType) + .withDescription(description).withOwnerEmail(ownerEmail).withRegion(region) + .withState(state).withTerminationReason(terminationReason) + .withExpectedTerminationTime(expectedTerminationTime) + .withMarkTime(markTime).withOptOutOfJanitor(false) + .setAdditionalField(fieldName, fieldValue); + + ArgumentCaptor objCap = ArgumentCaptor.forClass(Object.class); + ArgumentCaptor sqlCap = ArgumentCaptor.forClass(String.class); + when(tracker.getJdbcTemplate().update(sqlCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture(), + objCap.capture())).thenReturn(1); + tracker.addOrUpdate(newResource); + + List args = objCap.getAllValues(); + Assert.assertEquals(sqlCap.getValue(), "update janitortable set resourceType=?,region=?,ownerEmail=?,description=?,state=?,terminationReason=?,expectedTerminationTime=?,actualTerminationTime=?,notificationTime=?,launchTime=?,markTime=?,optOutOfJanitor=?,additionalFields=? where resourceId=?"); + Assert.assertEquals(args.size(), 14); + Assert.assertEquals(args.get(0).toString(), AWSResourceType.INSTANCE.toString()); + Assert.assertEquals(args.get(1).toString(), region); + Assert.assertEquals(args.get(2).toString(), ownerEmail); + Assert.assertEquals(args.get(3).toString(), description); + Assert.assertEquals(args.get(4).toString(), state.toString()); + Assert.assertEquals(args.get(5).toString(), terminationReason); + Assert.assertEquals(args.get(6).toString(), expectedTerminationTime.getTime() + ""); + Assert.assertEquals(args.get(7).toString(), "0"); + Assert.assertEquals(args.get(8).toString(), "0"); + Assert.assertEquals(args.get(9).toString(), "0"); + Assert.assertEquals(args.get(10).toString(), markTime.getTime() + ""); + Assert.assertEquals(args.get(11).toString(), "false"); + Assert.assertEquals(args.get(12).toString(), "{\"fieldName123\":\"fieldValue456\"}"); + Assert.assertEquals(args.get(13).toString(), id); + } + + + @SuppressWarnings("unchecked") + @Test + public void testGetResource() { + String id1 = "id-1"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + + AWSResource result1 = mkResource(id1, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, false, fieldName, fieldValue); + + ArrayList resources = new ArrayList<>(); + resources.add(result1); + TestRDSJanitorResourceTracker tracker = new TestRDSJanitorResourceTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(resources); + + Resource resource = tracker.getResource(id1); + ArrayList returnResources = new ArrayList<>(); + returnResources.add(resource); + + verifyResources(returnResources, + id1, null, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, fieldName, fieldValue); + } + + @SuppressWarnings("unchecked") + @Test + public void testGetResourceNotFound() { + ArrayList resources = new ArrayList<>(); + TestRDSJanitorResourceTracker tracker = new TestRDSJanitorResourceTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(resources); + Resource resource = tracker.getResource("id-2"); + Assert.assertNull(resource); + } + + @SuppressWarnings("unchecked") + @Test + public void testGetResources() { + String id1 = "id-1"; + String id2 = "id-2"; + AWSResourceType resourceType = AWSResourceType.INSTANCE; + Resource.CleanupState state = Resource.CleanupState.MARKED; + String description = "This is a test resource."; + String ownerEmail = "owner@test.com"; + String region = "us-east-1"; + String terminationReason = "This is a test termination reason."; + DateTime now = DateTime.now(); + Date expectedTerminationTime = new Date(now.plusDays(10).getMillis()); + Date markTime = new Date(now.getMillis()); + String fieldName = "fieldName123"; + String fieldValue = "fieldValue456"; + + AWSResource result1 = mkResource(id1, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, false, fieldName, fieldValue); + AWSResource result2 = mkResource(id2, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, true, fieldName, fieldValue); + + ArrayList resources = new ArrayList<>(); + resources.add(result1); + resources.add(result2); + TestRDSJanitorResourceTracker tracker = new TestRDSJanitorResourceTracker(); + when(tracker.getJdbcTemplate().query(Matchers.anyString(), + Matchers.any(Object[].class), + Matchers.any(RowMapper.class))).thenReturn(resources); + + verifyResources(tracker.getResources(resourceType, state, region), + id1, id2, resourceType, state, description, ownerEmail, + region, terminationReason, expectedTerminationTime, markTime, fieldName, fieldValue); + + } + + private void verifyResources(List resources, String id1, String id2, AWSResourceType resourceType, + Resource.CleanupState state, String description, String ownerEmail, String region, + String terminationReason, Date expectedTerminationTime, Date markTime, String fieldName, + String fieldValue) { + + if (id2 == null) { + Assert.assertEquals(resources.size(), 1); + } else { + Assert.assertEquals(resources.size(), 2); + } + + Assert.assertEquals(resources.get(0).getId(), id1); + Assert.assertEquals(resources.get(0).getResourceType(), resourceType); + Assert.assertEquals(resources.get(0).getState(), state); + Assert.assertEquals(resources.get(0).getDescription(), description); + Assert.assertEquals(resources.get(0).getOwnerEmail(), ownerEmail); + Assert.assertEquals(resources.get(0).getRegion(), region); + Assert.assertEquals(resources.get(0).getTerminationReason(), terminationReason); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(0).getExpectedTerminationTime().getTime()), + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(0).getMarkTime().getTime()), + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + Assert.assertEquals(resources.get(0).getAdditionalField(fieldName), fieldValue); + Assert.assertEquals(resources.get(0).isOptOutOfJanitor(), false); + + if (id2 != null) { + Assert.assertEquals(resources.get(1).getId(), id2); + Assert.assertEquals(resources.get(1).getResourceType(), resourceType); + Assert.assertEquals(resources.get(1).getState(), state); + Assert.assertEquals(resources.get(1).getDescription(), description); + Assert.assertEquals(resources.get(1).getOwnerEmail(), ownerEmail); + Assert.assertEquals(resources.get(1).getRegion(), region); + Assert.assertEquals(resources.get(1).getTerminationReason(), terminationReason); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(1).getExpectedTerminationTime().getTime()), + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + Assert.assertEquals( + AWSResource.DATE_FORMATTER.print(resources.get(1).getMarkTime().getTime()), + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + Assert.assertEquals(resources.get(1).isOptOutOfJanitor(), true); + Assert.assertEquals(resources.get(1).getAdditionalField(fieldName), fieldValue); + } + } + + private AWSResource mkResource(String id, AWSResourceType resourceType, Resource.CleanupState state, + String description, String ownerEmail, String region, String terminationReason, + Date expectedTerminationTime, Date markTime, boolean optOut, String fieldName, String fieldValue) { + + Map attrs = new HashMap<>(); + attrs.put(AWSResource.FIELD_RESOURCE_ID, id); + attrs.put(AWSResource.FIELD_RESOURCE_TYPE, resourceType.name()); + attrs.put(AWSResource.FIELD_DESCRIPTION, description); + attrs.put(AWSResource.FIELD_REGION, region); + attrs.put(AWSResource.FIELD_STATE, state.name()); + attrs.put(AWSResource.FIELD_OWNER_EMAIL, ownerEmail); + attrs.put(AWSResource.FIELD_TERMINATION_REASON, terminationReason); + attrs.put(AWSResource.FIELD_EXPECTED_TERMINATION_TIME, + AWSResource.DATE_FORMATTER.print(expectedTerminationTime.getTime())); + attrs.put(AWSResource.FIELD_MARK_TIME, + AWSResource.DATE_FORMATTER.print(markTime.getTime())); + attrs.put(AWSResource.FIELD_OPT_OUT_OF_JANITOR, String.valueOf(optOut)); + attrs.put(fieldName, fieldValue); + + return AWSResource.parseFieldtoValueMap(attrs); + } +} diff --git a/src/test/java/com/netflix/simianarmy/janitor/TestBasicJanitorMonkeyContext.java b/src/test/java/com/netflix/simianarmy/janitor/TestBasicJanitorMonkeyContext.java index 8e6bed03..b9b5cd29 100755 --- a/src/test/java/com/netflix/simianarmy/janitor/TestBasicJanitorMonkeyContext.java +++ b/src/test/java/com/netflix/simianarmy/janitor/TestBasicJanitorMonkeyContext.java @@ -1,12 +1,12 @@ -package com.netflix.simianarmy.basic.janitor; +package com.netflix.simianarmy.janitor; import com.netflix.simianarmy.aws.janitor.rule.generic.UntaggedRule; import com.netflix.simianarmy.basic.TestBasicCalendar; +import com.netflix.simianarmy.basic.janitor.BasicJanitorRuleEngine; import com.netflix.simianarmy.janitor.JanitorRuleEngine; import com.netflix.simianarmy.janitor.Rule; import org.apache.commons.lang.StringUtils; - import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test;