diff --git a/.github/ISSUE_TEMPLATE/native-query.md b/.github/ISSUE_TEMPLATE/native-query.md new file mode 100644 index 00000000..2a425eb5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/native-query.md @@ -0,0 +1,47 @@ +--- +name: Native Query Support +about: Report problems generating native query configurations using the CLI +title: "[Native Query]" +labels: native query +--- + + + +### Connector version + + + +### What form of error did you see? + + + +- [ ] Type inference is not currently implemented for stage / query predicate operator / aggregation operator +- [ ] Cannot infer types for this pipeline +- [ ] Type mismatch +- [ ] Could not read aggregation pipeline +- [ ] other error +- [ ] I have feedback that does not relate to a specific error + +### Error or feedback details + + + +### What did you want to happen? + + + +### Command and pipeline + + + +### Schema + + + + + + + +### Other details + + diff --git a/CHANGELOG.md b/CHANGELOG.md index 448b4abc..d74eb588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This changelog documents the changes between release versions. ## [Unreleased] +### Added + +- Adds CLI command to manage native queries with automatic type inference ([#131](https://github.com/hasura/ndc-mongodb/pull/131)) + ### Changed - Updates MongoDB Rust driver from v2.8 to v3.1.0 ([#124](https://github.com/hasura/ndc-mongodb/pull/124)) @@ -12,6 +16,79 @@ This changelog documents the changes between release versions. - The connector previously used Cloudflare's DNS resolver. Now it uses the locally-configured DNS resolver. ([#125](https://github.com/hasura/ndc-mongodb/pull/125)) +#### Managing native queries with the CLI + +New in this release is a CLI plugin command to create, list, inspect, and delete +native queries. A big advantage of using the command versus writing native query +configurations by hand is that the command will type-check your query's +aggregation pipeline, and will write type declarations automatically. + +This is a BETA feature - it is a work in progress, and will not work for all +cases. It is safe to experiment with since it is limited to managing native +query configuration files, and does not lock you into anything. + +You can run the new command like this: + +```sh +$ ddn connector plugin --connector app/connector/my_connector/connector.yaml -- native-query +``` + +To create a native query create a file with a `.json` extension that contains +the aggregation pipeline for you query. For example this pipeline in +`title_word_frequency.json` outputs frequency counts for words appearing in +movie titles in a given year: + +```json +[ + { + "$match": { + "year": "{{ year }}" + } + }, + { + "$replaceWith": { + "title_words": { "$split": ["$title", " "] } + } + }, + { "$unwind": { "path": "$title_words" } }, + { + "$group": { + "_id": "$title_words", + "count": { "$count": {} } + } + } +] +``` + +In your supergraph directory run a command like this using the path to the pipeline file as an argument, + +```sh +$ ddn connector plugin --connector app/connector/my_connector/connector.yaml -- native-query create title_word_frequency.json --collection movies +``` + +You should see output like this: + +``` +Wrote native query configuration to your-project/connector/native_queries/title_word_frequency.json + +input collection: movies +representation: collection + +## parameters + +year: int! + +## result type + +{ + _id: string!, + count: int! +} +``` + +For more details see the +[documentation page](https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#manage-native-queries-with-the-ddn-cli). + ## [1.4.0] - 2024-11-14 ### Added diff --git a/Cargo.lock b/Cargo.lock index 786ae48f..918cc576 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,6 +1830,7 @@ dependencies = [ "serde", "serde_json", "test-helpers", + "textwrap", "thiserror", "tokio", ] @@ -3106,6 +3107,12 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "smol_str" version = "0.1.24" @@ -3273,6 +3280,17 @@ dependencies = [ "proptest", ] +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -3706,6 +3724,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.23" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index c19d6865..944b2027 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -4,7 +4,8 @@ edition = "2021" version.workspace = true [features] -native-query-subcommand = ["dep:pretty", "dep:nom"] +default = ["native-query-subcommand"] +native-query-subcommand = ["dep:pretty", "dep:nom", "dep:textwrap"] [dependencies] configuration = { path = "../configuration" } @@ -26,6 +27,7 @@ ref-cast = { workspace = true } regex = "^1.11.1" serde = { workspace = true } serde_json = { workspace = true } +textwrap = { version = "^0.16.1", optional = true } thiserror = "1.0.57" tokio = { version = "1.36.0", features = ["full"] } diff --git a/crates/cli/src/native_query/aggregation_expression.rs b/crates/cli/src/native_query/aggregation_expression.rs index 1c83de23..0941249e 100644 --- a/crates/cli/src/native_query/aggregation_expression.rs +++ b/crates/cli/src/native_query/aggregation_expression.rs @@ -347,7 +347,7 @@ pub fn infer_type_from_reference_shorthand( .chain(type_annotation.map(TypeConstraint::from)); context.register_parameter(name.into(), constraints) } - Reference::PipelineVariable { .. } => todo!("pipeline variable"), + Reference::PipelineVariable { name, .. } => Err(Error::Other(format!("Encountered a pipeline variable, $${name}. Pipeline variables are currently not supported.")))?, Reference::InputDocumentField { name, nested_path } => { let doc_type = context.get_input_document_type()?; let path = NonEmpty { diff --git a/crates/cli/src/native_query/error.rs b/crates/cli/src/native_query/error.rs index 62021689..80a02ee9 100644 --- a/crates/cli/src/native_query/error.rs +++ b/crates/cli/src/native_query/error.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use configuration::schema::Type; -use mongodb::bson::{self, Bson, Document}; +use mongodb::bson::{Bson, Document}; use ndc_models::{ArgumentName, FieldName, ObjectTypeName}; use thiserror::Error; @@ -9,6 +9,10 @@ use super::type_constraint::{ObjectTypeConstraint, TypeConstraint, TypeVariable} pub type Result = std::result::Result; +// The URL for native query issues will be visible due to a wrapper around this error message in +// [crate::native_query::create]. +const UNSUPPORTED_FEATURE_MESSAGE: &str = r#"For a list of currently-supported features see https://hasura.io/docs/3.0/connectors/mongodb/native-operations/supported-aggregation-pipeline-features/. Please file a bug report, and declare types for your native query by hand for the time being."#; + #[derive(Clone, Debug, Error, PartialEq)] pub enum Error { #[error("Cannot infer a result type for an empty pipeline")] @@ -41,9 +45,7 @@ pub enum Error { unsolved_variables: Vec, }, - #[error( - "Cannot infer a result document type for pipeline because it does not produce documents" - )] + #[error("Cannot infer a result document type for pipeline because it does not produce documents. You might need to add a --collection flag to your command to specify an input collection for the query.")] IncompletePipeline, #[error("An object representing an expression must have exactly one field: {0}")] @@ -81,13 +83,13 @@ pub enum Error { #[error("Error parsing a string in the aggregation pipeline: {0}")] UnableToParseReferenceShorthand(String), - #[error("Type inference is not currently implemented for the query document operator, {0}. Please file a bug report, and declare types for your native query by hand for the time being.")] + #[error("Type inference is not currently implemented for the query predicate operator, {0}. {UNSUPPORTED_FEATURE_MESSAGE}")] UnknownMatchDocumentOperator(String), - #[error("Type inference is not currently implemented for the aggregation expression operator, {0}. Please file a bug report, and declare types for your native query by hand for the time being.")] + #[error("Type inference is not currently implemented for the aggregation expression operator, {0}. {UNSUPPORTED_FEATURE_MESSAGE}")] UnknownAggregationOperator(String), - #[error("Type inference is not currently implemented for{} stage number {} in your aggregation pipeline. Please file a bug report, and declare types for your native query by hand for the time being.", match stage_name { Some(name) => format!(" {name},"), None => "".to_string() }, stage_index + 1)] + #[error("Type inference is not currently implemented for{} stage number {} in your aggregation pipeline. {UNSUPPORTED_FEATURE_MESSAGE}", match stage_name { Some(name) => format!(" {name},"), None => "".to_string() }, stage_index + 1)] UnknownAggregationStage { stage_index: usize, stage_name: Option<&'static str>, diff --git a/crates/cli/src/native_query/mod.rs b/crates/cli/src/native_query/mod.rs index b5e68373..72c33450 100644 --- a/crates/cli/src/native_query/mod.rs +++ b/crates/cli/src/native_query/mod.rs @@ -36,7 +36,7 @@ use self::error::Result; use self::pipeline::infer_pipeline_types; use self::pretty_printing::pretty_print_native_query_info; -/// Create native queries - custom MongoDB queries that integrate into your data graph +/// [BETA] Create or manage native queries - custom MongoDB queries that integrate into your data graph #[derive(Clone, Debug, Subcommand)] pub enum Command { /// Create a native query from a JSON file containing an aggregation pipeline @@ -105,6 +105,7 @@ async fn delete(context: &Context, native_query_name: &str) -> anyhow::Result<() async fn show(context: &Context, native_query_name: &str) -> anyhow::Result<()> { let (native_query, path) = find_native_query(context, native_query_name).await?; pretty_print_native_query(&mut stdout(context), &native_query, &path).await?; + println!(); // blank line to avoid unterminated output indicator Ok(()) } @@ -145,7 +146,7 @@ async fn create( let pipeline = match read_pipeline(pipeline_path).await { Ok(p) => p, Err(err) => { - eprintln!("Could not read aggregation pipeline.\n\n{err}"); + write_stderr(&format!("Could not read aggregation pipeline.\n\n{err}")); exit(ExitCode::CouldNotReadAggregationPipeline.into()) } }; @@ -153,7 +154,13 @@ async fn create( { Ok(q) => WithName::named(name, q), Err(err) => { - eprintln!("Error interpreting aggregation pipeline. If you are not able to resolve this error you can add the native query by writing the configuration file directly in {}.\n\n{err}", native_query_path.to_string_lossy()); + eprintln!(); + write_stderr(&err.to_string()); + eprintln!(); + write_stderr(&format!("If you are not able to resolve this error you can add the native query by writing the configuration file directly in {}. See https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#write-native-query-configurations-directly", native_query_path.to_string_lossy())); + // eprintln!("See https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#write-native-query-configurations-directly"); + eprintln!(); + write_stderr("If you want to request support for a currently unsupported query feature, report a bug, or get support please file an issue at https://github.com/hasura/ndc-mongodb/issues/new?template=native-query.md"); exit(ExitCode::CouldNotReadAggregationPipeline.into()) } }; @@ -171,7 +178,7 @@ async fn create( ) .await { - eprintln!("Error writing native query configuration: {err}"); + write_stderr(&format!("Error writing native query configuration: {err}")); exit(ExitCode::ErrorWriting.into()) }; eprintln!( @@ -180,6 +187,7 @@ async fn create( ); eprintln!(); pretty_print_native_query_info(&mut stdout(context), &native_query.value).await?; + println!(); // blank line to avoid unterminated output indicator Ok(()) } @@ -193,7 +201,7 @@ async fn read_configuration( { Ok(c) => c, Err(err) => { - eprintln!("Could not read connector configuration - configuration must be initialized before creating native queries.\n\n{err:#}"); + write_stderr(&format!("Could not read connector configuration - configuration must be initialized before creating native queries.\n\n{err:#}")); exit(ExitCode::CouldNotReadConfiguration.into()) } }; @@ -211,7 +219,7 @@ async fn read_native_queries( let native_queries = match read_native_query_directory(&context.path, &[]).await { Ok(native_queries) => native_queries, Err(err) => { - eprintln!("Could not read native queries.\n\n{err}"); + write_stderr(&format!("Could not read native queries.\n\n{err}")); exit(ExitCode::CouldNotReadConfiguration.into()) } }; @@ -292,3 +300,9 @@ fn stdout(context: &Context) -> StandardStream { StandardStream::stdout(ColorChoice::Never) } } + +/// Write a message to sdterr with automatic line wrapping +fn write_stderr(message: &str) { + let wrap_options = 120; + eprintln!("{}", textwrap::fill(message, wrap_options)) +} diff --git a/fixtures/hasura/sample_mflix/connector/native_queries/title_word_frequency.json b/fixtures/hasura/sample_mflix/connector/native_queries/title_word_frequency.json new file mode 100644 index 00000000..397d66ee --- /dev/null +++ b/fixtures/hasura/sample_mflix/connector/native_queries/title_word_frequency.json @@ -0,0 +1,49 @@ +{ + "name": "title_word_frequency", + "representation": "collection", + "inputCollection": "movies", + "arguments": {}, + "resultDocumentType": "title_word_frequency_group", + "objectTypes": { + "title_word_frequency_group": { + "fields": { + "_id": { + "type": { + "scalar": "string" + } + }, + "count": { + "type": { + "scalar": "int" + } + } + } + } + }, + "pipeline": [ + { + "$replaceWith": { + "title": "$title", + "title_words": { + "$split": [ + "$title", + " " + ] + } + } + }, + { + "$unwind": { + "path": "$title_words" + } + }, + { + "$group": { + "_id": "$title_words", + "count": { + "$count": {} + } + } + } + ] +} \ No newline at end of file diff --git a/fixtures/hasura/sample_mflix/connector/native_queries/title_word_requency.json b/fixtures/hasura/sample_mflix/connector/native_queries/title_word_requency.json deleted file mode 100644 index b8306b2d..00000000 --- a/fixtures/hasura/sample_mflix/connector/native_queries/title_word_requency.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "title_word_frequency", - "representation": "collection", - "inputCollection": "movies", - "description": "words appearing in movie titles with counts", - "resultDocumentType": "TitleWordFrequency", - "objectTypes": { - "TitleWordFrequency": { - "fields": { - "_id": { "type": { "scalar": "string" } }, - "count": { "type": { "scalar": "int" } } - } - } - }, - "pipeline": [ - { - "$replaceWith": { - "title_words": { "$split": ["$title", " "] } - } - }, - { "$unwind": { "path": "$title_words" } }, - { - "$group": { - "_id": "$title_words", - "count": { "$count": {} } - } - } - ] -} -