From 67ddaa0b9640519a97d3cf442945342f375bf666 Mon Sep 17 00:00:00 2001 From: David Overton Date: Tue, 26 Mar 2024 11:09:19 +1100 Subject: [PATCH 1/7] write schema dir --- crates/configuration/src/directory.rs | 31 +++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index c1b368fd..c8880217 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -9,9 +9,9 @@ use std::{ use tokio::fs; use tokio_stream::wrappers::ReadDirStream; -use crate::{with_name::WithName, Configuration}; +use crate::{with_name::WithName, Configuration, Schema}; -pub const SCHEMA_FILENAME: &str = "schema"; +pub const SCHEMA_DIRNAME: &str = "schema"; pub const NATIVE_QUERIES_DIRNAME: &str = "native_queries"; pub const CONFIGURATION_EXTENSIONS: [(&str, FileFormat); 3] = @@ -149,14 +149,33 @@ where Ok(value) } -/// Currently only writes `schema.json` -pub async fn write_directory( +pub async fn write_schema_directory( configuration_dir: impl AsRef, - configuration: &Configuration, + schemas: impl IntoIterator, ) -> anyhow::Result<()> { - write_file(configuration_dir, SCHEMA_FILENAME, &configuration.schema).await + let dir_path = configuration_dir.as_ref().join(SCHEMA_DIRNAME); + ensure_directory_exists(&dir_path).await?; + for (filename, schema) in schemas { + write_file(&dir_path, &filename, &schema).await?; + } + Ok(()) } +async fn ensure_directory_exists(dir_path: impl AsRef) -> anyhow::Result<()> { + if !dir_path.as_ref().is_dir() { + fs::create_dir(dir_path).await?; + } + Ok(()) +} + +// /// Currently only writes `schema.json` +// pub async fn write_directory( +// configuration_dir: impl AsRef, +// configuration: &Configuration, +// ) -> anyhow::Result<()> { +// write_file(configuration_dir, SCHEMA_FILENAME, &configuration.schema).await +// } + fn default_file_path(configuration_dir: impl AsRef, basename: &str) -> PathBuf { let dir = configuration_dir.as_ref(); dir.join(format!("{basename}.{DEFAULT_EXTENSION}")) From f02f697525b9bcd8f2ac1d2022e02bda552711ff Mon Sep 17 00:00:00 2001 From: David Overton Date: Tue, 26 Mar 2024 15:23:41 +1100 Subject: [PATCH 2/7] Read and write schemas from schema/ subdirectory --- crates/cli/src/introspection/sampling.rs | 13 +++--- .../cli/src/introspection/type_unification.rs | 20 +-------- .../src/introspection/validation_schema.rs | 42 +++++++++++++------ crates/cli/src/lib.rs | 7 +--- crates/configuration/src/directory.rs | 34 ++++++++------- crates/configuration/src/lib.rs | 2 +- crates/configuration/src/schema/mod.rs | 19 +++++++++ 7 files changed, 77 insertions(+), 60 deletions(-) diff --git a/crates/cli/src/introspection/sampling.rs b/crates/cli/src/introspection/sampling.rs index 1891ba8f..a36608dc 100644 --- a/crates/cli/src/introspection/sampling.rs +++ b/crates/cli/src/introspection/sampling.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use super::type_unification::{ - unify_object_types, unify_schema, unify_type, TypeUnificationContext, TypeUnificationResult, + unify_object_types, unify_type, TypeUnificationContext, TypeUnificationResult, }; use configuration::{ schema::{self, Type}, @@ -22,11 +22,8 @@ type ObjectType = WithName; pub async fn sample_schema_from_db( sample_size: u32, config: &MongoConfig, -) -> anyhow::Result { - let mut schema = Schema { - collections: BTreeMap::new(), - object_types: BTreeMap::new(), - }; +) -> anyhow::Result> { + let mut schemas = BTreeMap::new(); let db = config.client.database(&config.database); let mut collections_cursor = db.list_collections(None, None).await?; @@ -34,9 +31,9 @@ pub async fn sample_schema_from_db( let collection_name = collection_spec.name; let collection_schema = sample_schema_from_collection(&collection_name, sample_size, config).await?; - schema = unify_schema(schema, collection_schema); + schemas.insert(collection_name, collection_schema); } - Ok(schema) + Ok(schemas) } async fn sample_schema_from_collection( diff --git a/crates/cli/src/introspection/type_unification.rs b/crates/cli/src/introspection/type_unification.rs index efcb11e1..97443039 100644 --- a/crates/cli/src/introspection/type_unification.rs +++ b/crates/cli/src/introspection/type_unification.rs @@ -4,7 +4,7 @@ /// use configuration::{ schema::{self, Type}, - Schema, WithName, + WithName, }; use indexmap::IndexMap; use itertools::Itertools as _; @@ -255,24 +255,6 @@ pub fn unify_object_types( Ok(merged_type_map.into_values().collect()) } -/// Unify two schemas. Assumes that the schemas describe mutually exclusive sets of collections. -pub fn unify_schema(schema_a: Schema, schema_b: Schema) -> Schema { - let collections = schema_a - .collections - .into_iter() - .chain(schema_b.collections) - .collect(); - let object_types = schema_a - .object_types - .into_iter() - .chain(schema_b.object_types) - .collect(); - Schema { - collections, - object_types, - } -} - #[cfg(test)] mod tests { use std::collections::{HashMap, HashSet}; diff --git a/crates/cli/src/introspection/validation_schema.rs b/crates/cli/src/introspection/validation_schema.rs index 7b819288..9a276006 100644 --- a/crates/cli/src/introspection/validation_schema.rs +++ b/crates/cli/src/introspection/validation_schema.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use configuration::{ schema::{self, Type}, Schema, WithName, @@ -16,14 +18,14 @@ type ObjectField = WithName; pub async fn get_metadata_from_validation_schema( config: &MongoConfig, -) -> Result { +) -> Result, MongoAgentError> { let db = config.client.database(&config.database); let collections_cursor = db.list_collections(None, None).await?; - let (object_types, collections) = collections_cursor + let schemas: Vec> = collections_cursor .into_stream() .map( - |collection_spec| -> Result<(Vec, Collection), MongoAgentError> { + |collection_spec| -> Result, MongoAgentError> { let collection_spec_value = collection_spec?; let name = &collection_spec_value.name; let schema_bson_option = collection_spec_value @@ -49,16 +51,27 @@ pub async fn get_metadata_from_validation_schema( properties: IndexMap::new(), }), } - .map(|validator_schema| make_collection(name, &validator_schema)) + .map(|validator_schema| make_collection_schema(name, &validator_schema)) }, ) - .try_collect::<(Vec>, Vec)>() + .try_collect::>>() .await?; - Ok(Schema { - collections: WithName::into_map(collections), - object_types: WithName::into_map(object_types.concat()), - }) + Ok(WithName::into_map(schemas)) +} + +fn make_collection_schema( + collection_name: &str, + validator_schema: &ValidatorSchema, +) -> WithName { + let (object_types, collection) = make_collection(collection_name, validator_schema); + WithName::named( + collection.name.clone(), + Schema { + collections: WithName::into_map(vec![collection]), + object_types: WithName::into_map(object_types), + }, + ) } fn make_collection( @@ -100,10 +113,13 @@ fn make_collection( object_type_defs.push(collection_type); - let collection_info = WithName::named(collection_name, schema::Collection { - description: validator_schema.description.clone(), - r#type: collection_name.to_string(), - }); + let collection_info = WithName::named( + collection_name, + schema::Collection { + description: validator_schema.description.clone(), + r#type: collection_name.to_string(), + }, + ); (object_type_defs, collection_info) } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 4843459b..df4c402e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use clap::{Parser, Subcommand}; -use configuration::Configuration; use mongodb_agent_common::interface_types::MongoConfig; #[derive(Debug, Clone, Parser)] @@ -37,15 +36,13 @@ pub async fn run(command: Command, context: &Context) -> anyhow::Result<()> { /// Update the configuration in the current directory by introspecting the database. async fn update(context: &Context, args: &UpdateArgs) -> anyhow::Result<()> { - let schema = match args.sample_size { + let schemas = match args.sample_size { None => introspection::get_metadata_from_validation_schema(&context.mongo_config).await?, Some(sample_size) => { introspection::sample_schema_from_db(sample_size, &context.mongo_config).await? } }; - let configuration = Configuration::from_schema(schema)?; - - configuration::write_directory(&context.path, &configuration).await?; + configuration::write_schema_directory(&context.path, schemas).await?; Ok(()) } diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index c8880217..9ee4a726 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -33,7 +33,8 @@ pub async fn read_directory( ) -> anyhow::Result { let dir = configuration_dir.as_ref(); - let schema = parse_json_or_yaml(dir, SCHEMA_FILENAME).await?; + let schemas = read_subdir_configs(&dir.join(SCHEMA_DIRNAME)).await?.unwrap_or_default(); + let schema = schemas.into_values().fold(Schema::default(), Schema::merge); let native_queries = read_subdir_configs(&dir.join(NATIVE_QUERIES_DIRNAME)) .await? @@ -149,25 +150,30 @@ where Ok(value) } -pub async fn write_schema_directory( - configuration_dir: impl AsRef, - schemas: impl IntoIterator, -) -> anyhow::Result<()> { - let dir_path = configuration_dir.as_ref().join(SCHEMA_DIRNAME); - ensure_directory_exists(&dir_path).await?; - for (filename, schema) in schemas { - write_file(&dir_path, &filename, &schema).await?; +async fn write_subdir_configs(subdir: &Path, configs: impl IntoIterator) -> anyhow::Result<()> +where + T: Serialize, +{ + if !(fs::try_exists(subdir).await?) { + fs::create_dir(subdir).await?; } - Ok(()) -} -async fn ensure_directory_exists(dir_path: impl AsRef) -> anyhow::Result<()> { - if !dir_path.as_ref().is_dir() { - fs::create_dir(dir_path).await?; + for (name, config) in configs { + let with_name: WithName = (name.clone(), config).into(); + write_file(subdir, &name, &with_name).await?; } + Ok(()) } +pub async fn write_schema_directory( + configuration_dir: impl AsRef, + schemas: impl IntoIterator, +) -> anyhow::Result<()> { + let subdir = configuration_dir.as_ref().join(SCHEMA_DIRNAME); + write_subdir_configs(&subdir, schemas).await +} + // /// Currently only writes `schema.json` // pub async fn write_directory( // configuration_dir: impl AsRef, diff --git a/crates/configuration/src/lib.rs b/crates/configuration/src/lib.rs index 91aa4c65..fa22b201 100644 --- a/crates/configuration/src/lib.rs +++ b/crates/configuration/src/lib.rs @@ -6,6 +6,6 @@ mod with_name; pub use crate::configuration::Configuration; pub use crate::directory::read_directory; -pub use crate::directory::write_directory; +pub use crate::directory::write_schema_directory; pub use crate::schema::Schema; pub use crate::with_name::{WithName, WithNameRef}; diff --git a/crates/configuration/src/schema/mod.rs b/crates/configuration/src/schema/mod.rs index bec4bdf2..163b9945 100644 --- a/crates/configuration/src/schema/mod.rs +++ b/crates/configuration/src/schema/mod.rs @@ -42,4 +42,23 @@ impl Schema { .iter() .map(|(name, field)| WithNameRef::named(name, field)) } + + /// Unify two schemas. Assumes that the schemas describe mutually exclusive sets of collections. + pub fn merge(schema_a: Schema, schema_b: Schema) -> Schema { + let collections = schema_a + .collections + .into_iter() + .chain(schema_b.collections) + .collect(); + let object_types = schema_a + .object_types + .into_iter() + .chain(schema_b.object_types) + .collect(); + Schema { + collections, + object_types, + } + } + } From c0369ecde38ff4e41d3e3c43750253feeea63e7e Mon Sep 17 00:00:00 2001 From: David Overton Date: Tue, 26 Mar 2024 16:25:29 +1100 Subject: [PATCH 3/7] Don't sample from collections that already have a schema --- crates/cli/src/introspection/sampling.rs | 11 +++-- crates/cli/src/lib.rs | 8 +++- crates/configuration/src/directory.rs | 59 +++++++++--------------- crates/configuration/src/lib.rs | 1 + 4 files changed, 36 insertions(+), 43 deletions(-) diff --git a/crates/cli/src/introspection/sampling.rs b/crates/cli/src/introspection/sampling.rs index a36608dc..6ffbeb81 100644 --- a/crates/cli/src/introspection/sampling.rs +++ b/crates/cli/src/introspection/sampling.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use super::type_unification::{ unify_object_types, unify_type, TypeUnificationContext, TypeUnificationResult, @@ -22,6 +22,7 @@ type ObjectType = WithName; pub async fn sample_schema_from_db( sample_size: u32, config: &MongoConfig, + existing_schemas: &HashSet, ) -> anyhow::Result> { let mut schemas = BTreeMap::new(); let db = config.client.database(&config.database); @@ -29,9 +30,11 @@ pub async fn sample_schema_from_db( while let Some(collection_spec) = collections_cursor.try_next().await? { let collection_name = collection_spec.name; - let collection_schema = - sample_schema_from_collection(&collection_name, sample_size, config).await?; - schemas.insert(collection_name, collection_schema); + if !existing_schemas.contains(&collection_name) { + let collection_schema = + sample_schema_from_collection(&collection_name, sample_size, config).await?; + schemas.insert(collection_name, collection_schema); + } } Ok(schemas) } diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index df4c402e..e8ce7838 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -39,7 +39,13 @@ async fn update(context: &Context, args: &UpdateArgs) -> anyhow::Result<()> { let schemas = match args.sample_size { None => introspection::get_metadata_from_validation_schema(&context.mongo_config).await?, Some(sample_size) => { - introspection::sample_schema_from_db(sample_size, &context.mongo_config).await? + let existing_schemas = configuration::list_existing_schemas(&context.path).await?; + introspection::sample_schema_from_db( + sample_size, + &context.mongo_config, + &existing_schemas, + ) + .await? } }; configuration::write_schema_directory(&context.path, schemas).await?; diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index 9ee4a726..47d1fb86 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -3,7 +3,7 @@ use futures::stream::TryStreamExt as _; use itertools::Itertools as _; use serde::{Deserialize, Serialize}; use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashSet}, path::{Path, PathBuf}, }; use tokio::fs; @@ -33,7 +33,9 @@ pub async fn read_directory( ) -> anyhow::Result { let dir = configuration_dir.as_ref(); - let schemas = read_subdir_configs(&dir.join(SCHEMA_DIRNAME)).await?.unwrap_or_default(); + let schemas = read_subdir_configs(&dir.join(SCHEMA_DIRNAME)) + .await? + .unwrap_or_default(); let schema = schemas.into_values().fold(Schema::default(), Schema::merge); let native_queries = read_subdir_configs(&dir.join(NATIVE_QUERIES_DIRNAME)) @@ -101,41 +103,6 @@ where } } -/// Given a base name, like "connection", looks for files of the form "connection.json", -/// "connection.yaml", etc; reads the file; and parses it according to its extension. -async fn parse_json_or_yaml(configuration_dir: &Path, basename: &str) -> anyhow::Result -where - for<'a> T: Deserialize<'a>, -{ - let (path, format) = find_file(configuration_dir, basename).await?; - parse_config_file(path, format).await -} - -/// Given a base name, like "connection", looks for files of the form "connection.json", -/// "connection.yaml", etc, and returns the found path with its file format. -async fn find_file( - configuration_dir: &Path, - basename: &str, -) -> anyhow::Result<(PathBuf, FileFormat)> { - for (extension, format) in CONFIGURATION_EXTENSIONS { - let path = configuration_dir.join(format!("{basename}.{extension}")); - if fs::try_exists(&path).await? { - return Ok((path, format)); - } - } - - Err(anyhow!( - "could not find file, {:?}", - configuration_dir.join(format!( - "{basename}.{{{}}}", - CONFIGURATION_EXTENSIONS - .into_iter() - .map(|(ext, _)| ext) - .join(",") - )) - )) -} - async fn parse_config_file(path: impl AsRef, format: FileFormat) -> anyhow::Result where for<'a> T: Deserialize<'a>, @@ -150,7 +117,10 @@ where Ok(value) } -async fn write_subdir_configs(subdir: &Path, configs: impl IntoIterator) -> anyhow::Result<()> +async fn write_subdir_configs( + subdir: &Path, + configs: impl IntoIterator, +) -> anyhow::Result<()> where T: Serialize, { @@ -201,3 +171,16 @@ where .await .with_context(|| format!("error writing {:?}", path)) } + +pub async fn list_existing_schemas( + configuration_dir: impl AsRef, +) -> anyhow::Result> { + let dir = configuration_dir.as_ref(); + + // TODO: we don't really need to read and parse all the schema files here, just get their names. + let schemas = read_subdir_configs::(&dir.join(SCHEMA_DIRNAME)) + .await? + .unwrap_or_default(); + + Ok(schemas.into_keys().collect()) +} diff --git a/crates/configuration/src/lib.rs b/crates/configuration/src/lib.rs index fa22b201..20c2822a 100644 --- a/crates/configuration/src/lib.rs +++ b/crates/configuration/src/lib.rs @@ -5,6 +5,7 @@ pub mod schema; mod with_name; pub use crate::configuration::Configuration; +pub use crate::directory::list_existing_schemas; pub use crate::directory::read_directory; pub use crate::directory::write_schema_directory; pub use crate::schema::Schema; From 9cb422604a491a5fd28fae37bd679340627ee824 Mon Sep 17 00:00:00 2001 From: David Overton Date: Tue, 26 Mar 2024 16:49:46 +1100 Subject: [PATCH 4/7] Add changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c927a9b9..44872af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ This changelog documents the changes between release versions. ## [Unreleased] -Changes to be included in the next upcoming release +- Use separate schema files for each collection +- Don't sample from collections that already have a schema ## [0.0.2] - 2024-03-26 - Rename CLI plugin to ndc-mongodb ([PR #13](https://github.com/hasura/ndc-mongodb/pull/13)) From 1f8a1acb09c554c2e527e70b4ddb2c69877c4a15 Mon Sep 17 00:00:00 2001 From: David Overton Date: Wed, 27 Mar 2024 11:48:37 +1100 Subject: [PATCH 5/7] Remove commented out code --- .../src/introspection/validation_schema.rs | 28 ++++--------------- crates/cli/src/lib.rs | 21 +++++++------- crates/configuration/src/directory.rs | 8 ------ 3 files changed, 16 insertions(+), 41 deletions(-) diff --git a/crates/cli/src/introspection/validation_schema.rs b/crates/cli/src/introspection/validation_schema.rs index 9a276006..c2099dd6 100644 --- a/crates/cli/src/introspection/validation_schema.rs +++ b/crates/cli/src/introspection/validation_schema.rs @@ -24,9 +24,9 @@ pub async fn get_metadata_from_validation_schema( let schemas: Vec> = collections_cursor .into_stream() - .map( - |collection_spec| -> Result, MongoAgentError> { - let collection_spec_value = collection_spec?; + .filter_map( + |collection_spec| -> Option> { + let collection_spec_value = collection_spec.ok()?; let name = &collection_spec_value.name; let schema_bson_option = collection_spec_value .options @@ -34,27 +34,11 @@ pub async fn get_metadata_from_validation_schema( .as_ref() .and_then(|x| x.get("$jsonSchema")); - match schema_bson_option { - Some(schema_bson) => { - from_bson::(schema_bson.clone()).map_err(|err| { - MongoAgentError::BadCollectionSchema( - name.to_owned(), - schema_bson.clone(), - err, - ) - }) - } - None => Ok(ValidatorSchema { - bson_type: BsonType::Object, - description: None, - required: Vec::new(), - properties: IndexMap::new(), - }), - } - .map(|validator_schema| make_collection_schema(name, &validator_schema)) + let validator_schema = schema_bson_option.map(|schema_bson| from_bson::(schema_bson.clone()).ok())??; + Some(make_collection_schema(name, &validator_schema)) }, ) - .try_collect::>>() + .collect::>>() .await?; Ok(WithName::into_map(schemas)) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index e8ce7838..789506f6 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -10,8 +10,8 @@ use mongodb_agent_common::interface_types::MongoConfig; #[derive(Debug, Clone, Parser)] pub struct UpdateArgs { - #[arg(long = "sample-size", value_name = "N")] - sample_size: Option, + #[arg(long = "sample-size", value_name = "N", default_value = "10")] + sample_size: u32, } /// The command invoked by the user. @@ -36,19 +36,18 @@ pub async fn run(command: Command, context: &Context) -> anyhow::Result<()> { /// Update the configuration in the current directory by introspecting the database. async fn update(context: &Context, args: &UpdateArgs) -> anyhow::Result<()> { - let schemas = match args.sample_size { - None => introspection::get_metadata_from_validation_schema(&context.mongo_config).await?, - Some(sample_size) => { - let existing_schemas = configuration::list_existing_schemas(&context.path).await?; + let schemas_from_json_validation = introspection::get_metadata_from_validation_schema(&context.mongo_config).await?; + configuration::write_schema_directory(&context.path, schemas_from_json_validation).await?; + + let existing_schemas = configuration::list_existing_schemas(&context.path).await?; + let schemas_from_sampling = introspection::sample_schema_from_db( - sample_size, + args.sample_size, &context.mongo_config, &existing_schemas, ) - .await? - } - }; - configuration::write_schema_directory(&context.path, schemas).await?; + .await?; + configuration::write_schema_directory(&context.path, schemas_from_sampling).await?; Ok(()) } diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index 47d1fb86..fcac3d6c 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -144,14 +144,6 @@ pub async fn write_schema_directory( write_subdir_configs(&subdir, schemas).await } -// /// Currently only writes `schema.json` -// pub async fn write_directory( -// configuration_dir: impl AsRef, -// configuration: &Configuration, -// ) -> anyhow::Result<()> { -// write_file(configuration_dir, SCHEMA_FILENAME, &configuration.schema).await -// } - fn default_file_path(configuration_dir: impl AsRef, basename: &str) -> PathBuf { let dir = configuration_dir.as_ref(); dir.join(format!("{basename}.{DEFAULT_EXTENSION}")) From d0853329fce5a4c559753f4845db4399a8127ef3 Mon Sep 17 00:00:00 2001 From: David Overton Date: Wed, 27 Mar 2024 12:02:24 +1100 Subject: [PATCH 6/7] Revert "Remove commented out code" This reverts commit 1f8a1acb09c554c2e527e70b4ddb2c69877c4a15. --- .../src/introspection/validation_schema.rs | 28 +++++++++++++++---- crates/cli/src/lib.rs | 21 +++++++------- crates/configuration/src/directory.rs | 8 ++++++ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/crates/cli/src/introspection/validation_schema.rs b/crates/cli/src/introspection/validation_schema.rs index c2099dd6..9a276006 100644 --- a/crates/cli/src/introspection/validation_schema.rs +++ b/crates/cli/src/introspection/validation_schema.rs @@ -24,9 +24,9 @@ pub async fn get_metadata_from_validation_schema( let schemas: Vec> = collections_cursor .into_stream() - .filter_map( - |collection_spec| -> Option> { - let collection_spec_value = collection_spec.ok()?; + .map( + |collection_spec| -> Result, MongoAgentError> { + let collection_spec_value = collection_spec?; let name = &collection_spec_value.name; let schema_bson_option = collection_spec_value .options @@ -34,11 +34,27 @@ pub async fn get_metadata_from_validation_schema( .as_ref() .and_then(|x| x.get("$jsonSchema")); - let validator_schema = schema_bson_option.map(|schema_bson| from_bson::(schema_bson.clone()).ok())??; - Some(make_collection_schema(name, &validator_schema)) + match schema_bson_option { + Some(schema_bson) => { + from_bson::(schema_bson.clone()).map_err(|err| { + MongoAgentError::BadCollectionSchema( + name.to_owned(), + schema_bson.clone(), + err, + ) + }) + } + None => Ok(ValidatorSchema { + bson_type: BsonType::Object, + description: None, + required: Vec::new(), + properties: IndexMap::new(), + }), + } + .map(|validator_schema| make_collection_schema(name, &validator_schema)) }, ) - .collect::>>() + .try_collect::>>() .await?; Ok(WithName::into_map(schemas)) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 789506f6..e8ce7838 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -10,8 +10,8 @@ use mongodb_agent_common::interface_types::MongoConfig; #[derive(Debug, Clone, Parser)] pub struct UpdateArgs { - #[arg(long = "sample-size", value_name = "N", default_value = "10")] - sample_size: u32, + #[arg(long = "sample-size", value_name = "N")] + sample_size: Option, } /// The command invoked by the user. @@ -36,18 +36,19 @@ pub async fn run(command: Command, context: &Context) -> anyhow::Result<()> { /// Update the configuration in the current directory by introspecting the database. async fn update(context: &Context, args: &UpdateArgs) -> anyhow::Result<()> { - let schemas_from_json_validation = introspection::get_metadata_from_validation_schema(&context.mongo_config).await?; - configuration::write_schema_directory(&context.path, schemas_from_json_validation).await?; - - let existing_schemas = configuration::list_existing_schemas(&context.path).await?; - let schemas_from_sampling = + let schemas = match args.sample_size { + None => introspection::get_metadata_from_validation_schema(&context.mongo_config).await?, + Some(sample_size) => { + let existing_schemas = configuration::list_existing_schemas(&context.path).await?; introspection::sample_schema_from_db( - args.sample_size, + sample_size, &context.mongo_config, &existing_schemas, ) - .await?; - configuration::write_schema_directory(&context.path, schemas_from_sampling).await?; + .await? + } + }; + configuration::write_schema_directory(&context.path, schemas).await?; Ok(()) } diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index fcac3d6c..47d1fb86 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -144,6 +144,14 @@ pub async fn write_schema_directory( write_subdir_configs(&subdir, schemas).await } +// /// Currently only writes `schema.json` +// pub async fn write_directory( +// configuration_dir: impl AsRef, +// configuration: &Configuration, +// ) -> anyhow::Result<()> { +// write_file(configuration_dir, SCHEMA_FILENAME, &configuration.schema).await +// } + fn default_file_path(configuration_dir: impl AsRef, basename: &str) -> PathBuf { let dir = configuration_dir.as_ref(); dir.join(format!("{basename}.{DEFAULT_EXTENSION}")) From ddf7c91fb80570b6b332802526d0ebb908dc245d Mon Sep 17 00:00:00 2001 From: David Overton Date: Wed, 27 Mar 2024 12:03:18 +1100 Subject: [PATCH 7/7] Remove commented out code --- crates/configuration/src/directory.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/configuration/src/directory.rs b/crates/configuration/src/directory.rs index 47d1fb86..fcac3d6c 100644 --- a/crates/configuration/src/directory.rs +++ b/crates/configuration/src/directory.rs @@ -144,14 +144,6 @@ pub async fn write_schema_directory( write_subdir_configs(&subdir, schemas).await } -// /// Currently only writes `schema.json` -// pub async fn write_directory( -// configuration_dir: impl AsRef, -// configuration: &Configuration, -// ) -> anyhow::Result<()> { -// write_file(configuration_dir, SCHEMA_FILENAME, &configuration.schema).await -// } - fn default_file_path(configuration_dir: impl AsRef, basename: &str) -> PathBuf { let dir = configuration_dir.as_ref(); dir.join(format!("{basename}.{DEFAULT_EXTENSION}"))