hasura · hallettj · Dec 3, 2024 · Nov 25, 2024 · Nov 25, 2024 · Nov 25, 2024
diff --git a/.github/ISSUE_TEMPLATE/native-query.md b/.github/ISSUE_TEMPLATE/native-query.md
@@ -0,0 +1,47 @@
+---
+name: Native Query Support
+about: Report problems generating native query configurations using the CLI
+title: "[Native Query]"
+labels: native query
+---
+
+<!-- This is the template to use if you ran a command of the form, `ddn connector plugin -- native-query create query.json` and it didn't do what you wanted. -->
+
+### Connector version
+
+<!-- Which MongoDB connector version are you using? This is the version of the Hasura data connector, not the version of your database server. -->
+
+### What form of error did you see?
+
+<!-- Replace `[ ]` with `[x]` next to each applicable list item. -->
+
+- [ ] Type inference is not currently implemented for stage / query predicate operator / aggregation operator
+- [ ] Cannot infer types for this pipeline
+- [ ] Type mismatch
+- [ ] Could not read aggregation pipeline
+- [ ] other error
+- [ ] I have feedback that does not relate to a specific error
+
+### Error or feedback details
+
+<!-- Please paste output from the error or errors. Or if you have feedback that is not related to an error please tell us about it here. -->
+
+### What did you want to happen?
+
+<!-- For example if you got a "cannot infer types" or a "type mismatch" error, what types do you think would be appropriate to infer? If you are here because of a "type inefrence is not currently implemented" error are there details you think would be helpful for us to know about features that would be useful for you for us to support? -->
+
+### Command and pipeline
+
+<!-- Please paste the command that you ran, and your aggregation pipeline (the content of the json file that you provide to the `create` command). -->
+
+### Schema
+
+<!-- If your native query uses an input collection, specified by the `--collection` flag, it is helpful for us to know as much as possible about your connector's schema configuration for that collection. If it does not contain sensitive information please paste or provide a gist link to `app/connector/<connector-name>/schema/<collection-name>.json` -->
+
+<!-- If you are not able to share your schema file please describe configured types of any collection fields referenced in your pipeline, including whether those types are nullable. -->
+
+<!-- If your pipeline includes `$lookup` or `$graphLookup` stages that reference other collections please provide the same information for those collections. -->
+
+### Other details
+
+<!-- If you have any other information, feedback, or questions please let us know. -->
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ This changelog documents the changes between release versions.
 
 ## [Unreleased]
 
+### Added
+
+- Adds CLI command to manage native queries with automatic type inference ([#131](https://github.com/hasura/ndc-mongodb/pull/131))
+
 ### Changed
 
 - Updates MongoDB Rust driver from v2.8 to v3.1.0 ([#124](https://github.com/hasura/ndc-mongodb/pull/124))
@@ -12,6 +16,79 @@ This changelog documents the changes between release versions.
 
 - The connector previously used Cloudflare's DNS resolver. Now it uses the locally-configured DNS resolver. ([#125](https://github.com/hasura/ndc-mongodb/pull/125))
 
+#### Managing native queries with the CLI
+
+New in this release is a CLI plugin command to create, list, inspect, and delete
+native queries. A big advantage of using the command versus writing native query
+configurations by hand is that the command will type-check your query's
+aggregation pipeline, and will write type declarations automatically.
+
+This is a BETA feature - it is a work in progress, and will not work for all
+cases. It is safe to experiment with since it is limited to managing native
+query configuration files, and does not lock you into anything.
+
+You can run the new command like this:
+
+```sh
+$ ddn connector plugin --connector app/connector/my_connector/connector.yaml -- native-query
+```
+
+To create a native query create a file with a `.json` extension that contains
+the aggregation pipeline for you query. For example this pipeline in
+`title_word_frequency.json` outputs frequency counts for words appearing in
+movie titles in a given year:
+
+```json
+[
+  {
+    "$match": {
+      "year": "{{ year }}"
+    }
+  },
+  { 
+    "$replaceWith": {
+      "title_words": { "$split": ["$title", " "] }
+    }
+  },
+  { "$unwind": { "path": "$title_words" } },
+  { 
+    "$group": {
+      "_id": "$title_words",
+      "count": { "$count": {} }
+    }
+  }
+]
+```
+
+In your supergraph directory run a command like this using the path to the pipeline file as an argument,
+
+```sh
+$ ddn connector plugin --connector app/connector/my_connector/connector.yaml -- native-query create title_word_frequency.json --collection movies
+```
+
+You should see output like this:
+
+```
+Wrote native query configuration to your-project/connector/native_queries/title_word_frequency.json
+
+input collection: movies
+representation: collection
+
+## parameters
+
+year: int!
+
+## result type
+
+{
+  _id: string!,
+  count: int!
+}
+```
+
+For more details see the
+[documentation page](https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#manage-native-queries-with-the-ddn-cli).
+
 ## [1.4.0] - 2024-11-14
 
 ### Added

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
@@ -4,7 +4,8 @@ edition = "2021"
 version.workspace = true
 
 [features]
-native-query-subcommand = ["dep:pretty", "dep:nom"]
+default = ["native-query-subcommand"]
+native-query-subcommand = ["dep:pretty", "dep:nom", "dep:textwrap"]
 
 [dependencies]
 configuration = { path = "../configuration" }
@@ -26,6 +27,7 @@ ref-cast = { workspace = true }
 regex = "^1.11.1"
 serde = { workspace = true }
 serde_json = { workspace = true }
+textwrap = { version = "^0.16.1", optional = true }
 thiserror = "1.0.57"
 tokio = { version = "1.36.0", features = ["full"] }
 

diff --git a/crates/cli/src/native_query/aggregation_expression.rs b/crates/cli/src/native_query/aggregation_expression.rs
@@ -347,7 +347,7 @@ pub fn infer_type_from_reference_shorthand(
                 .chain(type_annotation.map(TypeConstraint::from));
             context.register_parameter(name.into(), constraints)
         }
-        Reference::PipelineVariable { .. } => todo!("pipeline variable"),
+        Reference::PipelineVariable { name, .. } => Err(Error::Other(format!("Encountered a pipeline variable, $${name}. Pipeline variables are currently not supported.")))?,
         Reference::InputDocumentField { name, nested_path } => {
             let doc_type = context.get_input_document_type()?;
             let path = NonEmpty {

diff --git a/crates/cli/src/native_query/error.rs b/crates/cli/src/native_query/error.rs
@@ -1,14 +1,18 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 
 use configuration::schema::Type;
-use mongodb::bson::{self, Bson, Document};
+use mongodb::bson::{Bson, Document};
 use ndc_models::{ArgumentName, FieldName, ObjectTypeName};
 use thiserror::Error;
 
 use super::type_constraint::{ObjectTypeConstraint, TypeConstraint, TypeVariable};
 
 pub type Result<T> = std::result::Result<T, Error>;
 
+// The URL for native query issues will be visible due to a wrapper around this error message in
+// [crate::native_query::create].
+const UNSUPPORTED_FEATURE_MESSAGE: &str = r#"For a list of currently-supported features see https://hasura.io/docs/3.0/connectors/mongodb/native-operations/supported-aggregation-pipeline-features/. Please file a bug report, and declare types for your native query by hand for the time being."#;
+
 #[derive(Clone, Debug, Error, PartialEq)]
 pub enum Error {
     #[error("Cannot infer a result type for an empty pipeline")]
@@ -41,9 +45,7 @@ pub enum Error {
         unsolved_variables: Vec<TypeVariable>,
     },
 
-    #[error(
-        "Cannot infer a result document type for pipeline because it does not produce documents"
-    )]
+    #[error("Cannot infer a result document type for pipeline because it does not produce documents. You might need to add a --collection flag to your command to specify an input collection for the query.")]
     IncompletePipeline,
 
     #[error("An object representing an expression must have exactly one field: {0}")]
@@ -81,13 +83,13 @@ pub enum Error {
     #[error("Error parsing a string in the aggregation pipeline: {0}")]
     UnableToParseReferenceShorthand(String),
 
-    #[error("Type inference is not currently implemented for the query document operator, {0}. Please file a bug report, and declare types for your native query by hand for the time being.")]
+    #[error("Type inference is not currently implemented for the query predicate operator, {0}. {UNSUPPORTED_FEATURE_MESSAGE}")]
     UnknownMatchDocumentOperator(String),
 
-    #[error("Type inference is not currently implemented for the aggregation expression operator, {0}. Please file a bug report, and declare types for your native query by hand for the time being.")]
+    #[error("Type inference is not currently implemented for the aggregation expression operator, {0}. {UNSUPPORTED_FEATURE_MESSAGE}")]
     UnknownAggregationOperator(String),
 
-    #[error("Type inference is not currently implemented for{} stage number {} in your aggregation pipeline. Please file a bug report, and declare types for your native query by hand for the time being.", match stage_name { Some(name) => format!(" {name},"), None => "".to_string() }, stage_index + 1)]
+    #[error("Type inference is not currently implemented for{} stage number {} in your aggregation pipeline. {UNSUPPORTED_FEATURE_MESSAGE}", match stage_name { Some(name) => format!(" {name},"), None => "".to_string() }, stage_index + 1)]
     UnknownAggregationStage {
         stage_index: usize,
         stage_name: Option<&'static str>,

diff --git a/crates/cli/src/native_query/mod.rs b/crates/cli/src/native_query/mod.rs
@@ -36,7 +36,7 @@ use self::error::Result;
 use self::pipeline::infer_pipeline_types;
 use self::pretty_printing::pretty_print_native_query_info;
 
-/// Create native queries - custom MongoDB queries that integrate into your data graph
+/// [BETA] Create or manage native queries - custom MongoDB queries that integrate into your data graph
 #[derive(Clone, Debug, Subcommand)]
 pub enum Command {
     /// Create a native query from a JSON file containing an aggregation pipeline
@@ -105,6 +105,7 @@ async fn delete(context: &Context, native_query_name: &str) -> anyhow::Result<()
 async fn show(context: &Context, native_query_name: &str) -> anyhow::Result<()> {
     let (native_query, path) = find_native_query(context, native_query_name).await?;
     pretty_print_native_query(&mut stdout(context), &native_query, &path).await?;
+    println!(); // blank line to avoid unterminated output indicator
     Ok(())
 }
 
@@ -145,15 +146,21 @@ async fn create(
     let pipeline = match read_pipeline(pipeline_path).await {
         Ok(p) => p,
         Err(err) => {
-            eprintln!("Could not read aggregation pipeline.\n\n{err}");
+            write_stderr(&format!("Could not read aggregation pipeline.\n\n{err}"));
             exit(ExitCode::CouldNotReadAggregationPipeline.into())
         }
     };
     let native_query = match native_query_from_pipeline(&configuration, &name, collection, pipeline)
     {
         Ok(q) => WithName::named(name, q),
         Err(err) => {
-            eprintln!("Error interpreting aggregation pipeline. If you are not able to resolve this error you can add the native query by writing the configuration file directly in {}.\n\n{err}", native_query_path.to_string_lossy());
+            eprintln!();
+            write_stderr(&err.to_string());
+            eprintln!();
+            write_stderr(&format!("If you are not able to resolve this error you can add the native query by writing the configuration file directly in {}. See https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#write-native-query-configurations-directly", native_query_path.to_string_lossy()));
+            // eprintln!("See https://hasura.io/docs/3.0/connectors/mongodb/native-operations/native-queries/#write-native-query-configurations-directly");
+            eprintln!();
+            write_stderr("If you want to request support for a currently unsupported query feature, report a bug, or get support please file an issue at https://github.com/hasura/ndc-mongodb/issues/new?template=native-query.md");
             exit(ExitCode::CouldNotReadAggregationPipeline.into())
         }
     };
@@ -171,7 +178,7 @@ async fn create(
     )
     .await
     {
-        eprintln!("Error writing native query configuration: {err}");
+        write_stderr(&format!("Error writing native query configuration: {err}"));
         exit(ExitCode::ErrorWriting.into())
     };
     eprintln!(
@@ -180,6 +187,7 @@ async fn create(
     );
     eprintln!();
     pretty_print_native_query_info(&mut stdout(context), &native_query.value).await?;
+    println!(); // blank line to avoid unterminated output indicator
     Ok(())
 }
 
@@ -193,7 +201,7 @@ async fn read_configuration(
     {
         Ok(c) => c,
         Err(err) => {
-            eprintln!("Could not read connector configuration - configuration must be initialized before creating native queries.\n\n{err:#}");
+            write_stderr(&format!("Could not read connector configuration - configuration must be initialized before creating native queries.\n\n{err:#}"));
             exit(ExitCode::CouldNotReadConfiguration.into())
         }
     };
@@ -211,7 +219,7 @@ async fn read_native_queries(
     let native_queries = match read_native_query_directory(&context.path, &[]).await {
         Ok(native_queries) => native_queries,
         Err(err) => {
-            eprintln!("Could not read native queries.\n\n{err}");
+            write_stderr(&format!("Could not read native queries.\n\n{err}"));
             exit(ExitCode::CouldNotReadConfiguration.into())
         }
     };
@@ -292,3 +300,9 @@ fn stdout(context: &Context) -> StandardStream {
         StandardStream::stdout(ColorChoice::Never)
     }
 }
+
+/// Write a message to sdterr with automatic line wrapping
+fn write_stderr(message: &str) {
+    let wrap_options = 120;
+    eprintln!("{}", textwrap::fill(message, wrap_options))
+}