From bfe867ae9e26e7cccfc1841ac338c9d1c4e5ac3c Mon Sep 17 00:00:00 2001 From: Jesse Hallett Date: Thu, 15 Aug 2024 17:15:48 -0700 Subject: [PATCH] development experience improvements with more logging, and easier setup --- .envrc | 7 ++ README.md | 120 ++++++++++++++++++++------- arion-compose/services/connector.nix | 2 +- arion-compose/services/engine.nix | 3 +- justfile | 24 +++++- 5 files changed, 120 insertions(+), 36 deletions(-) diff --git a/.envrc b/.envrc index a8ff4b71..7a32a50f 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,10 @@ # this line sources your `.envrc.local` file source_env_if_exists .envrc.local + +# Install nix-direnv which provides significantly faster Nix integration +if ! has nix_direnv_version || ! nix_direnv_version 3.0.5; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.5/direnvrc" "sha256-RuwIS+QKFj/T9M2TFXScjBsLR6V3A17YVoEW/Q6AZ1w=" +fi + +# Apply the devShell configured in flake.nix use flake diff --git a/README.md b/README.md index a437d162..b3deac50 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,63 @@ # Hasura MongoDB Connector -## Requirements +This repo provides a service that connects [Hasura v3][] to MongoDB databases. +Supports MongoDB 6 or later. + +[Hasura v3]: https://hasura.io/ + +## Docker Images + +The MongoDB connector is available from the [Hasura connectors directory][]. +There are also Docker images available at: + +https://github.com/hasura/ndc-mongodb/pkgs/container/ndc-mongodb + +The published Docker images are multi-arch, supporting amd64 and arm64 Linux. + +[Hasura connectors directory]: https://hasura.io/connectors/mongodb + +## Build Requirements + +The easiest way to set up build and development dependencies for this project is +to use Nix. If you don't already have Nix we recommend the [Determinate Systems +Nix Installer][] which automatically applies settings required by this project. + +[Determinate Systems Nix Installer]: https://github.com/DeterminateSystems/nix-installer/blob/main/README.md + +If you prefer to manage dependencies yourself you will need, * Rust via Rustup * MongoDB `>= 6` * OpenSSL development files -or get dependencies automatically with Nix +## Quickstart + +To run everything you need run this command to start services in Docker +containers: + +```sh +$ just up +``` -Some of the build instructions require Nix. To set that up [install Nix][], and -configure it to [enable flakes][]. +Next access the GraphQL interface at http://localhost:7100/ -[install Nix]: https://nixos.org/download.html -[enable flakes]: https://nixos.wiki/wiki/Flakes +If you are using the development shell (see below) the `just` command will be +provided automatically. -## Build & Run +Run the above command again to restart after making code changes. -To build a statically-linked binary run, +## Build + +To build the MongoDB connector run, ```sh $ nix build --print-build-logs && cp result/bin/mongodb-connector ``` -To cross-compile a statically-linked ARM build for Linux run, +To cross-compile statically-linked binaries for x86_64 or ARM for Linux run, ```sh +$ nix build .#mongo-connector-x86_64-linux --print-build-logs && cp result/bin/mongodb-connector $ nix build .#mongo-connector-aarch64-linux --print-build-logs && cp result/bin/mongodb-connector ``` @@ -54,36 +87,58 @@ nixpkgs#skopeo -- --insecure-policy copy docker-archive:result docker-daemon:mon ## Developing -This project uses a devShell configuration in `flake.nix` that automatically -loads specific version of Rust, mongosh, and other utilities. The easiest way to -make use of the devShell is to install nix, direnv and nix-direnv. See -https://github.com/nix-community/nix-direnv +### The development shell + +This project uses a development shell configured in `flake.nix` that automatically +loads specific version of Rust along with all other project dependencies. The +simplest way to start a development shell is with this command: -Direnv will source `.envrc`, install the appropriate Nix packages automatically -(isolated from the rest of your system packages), and configure your shell to -use the project dependencies when you cd into the project directory. All shell -modifications are reversed when you navigate to another directory. +```sh +$ nix develop +``` + +If you are going to be doing a lot of work on this project it can be more +convenient to set up [direnv][] which automatically links project dependencies +in your shell when you cd to the project directory, and automatically reverses +all shell modifications when you navigate to another directory. You can also set +up direnv integration in your editor to get your editor LSP to use the same +version of Rust that the project uses. + +[direnv]: https://direnv.net/ ### Running the Connector During Development -If you have set up nix and direnv then you can use arion to run the agent with -all of the services that it needs to function. Arion is a frontend for -docker-compose that adds a layer of convenience where it can easily load agent -code changes. It is automatically included with the project's devShell. +There is a `justfile` for getting started quickly. You can use its recipes to +run relevant services locally including the MongoDB connector itself, a MongoDB +database server, and the Hasura GraphQL Engine. Use these commands: + +```sh +just up # start services; run this again to restart after making code changes +just down # stop services +just down-volumes # stop services, and remove MongoDB database volume +just logs # see service logs +just test # run unit and integration tests +just # list available recipes +``` + +Integration tests run in an independent set of ephemeral docker containers. + +The `just` command is provided automatically if you are using the development +shell. Or you can install it yourself. + +The `justfile` delegates to arion which is a frontend for docker-compose that +adds a layer of convenience where it can easily load agent code changes. If you +are using the devShell you can run `arion` commands directly. They mostly work +just like `docker-compose` commands: To start all services run: $ arion up -d -To recompile and restart the agent after code changes run: +To recompile and restart the connector after code changes run: $ arion up -d connector -Arion delegates to docker-compose so it uses the same subcommands with the same -flags. Note that the PostgreSQL and MongoDB services use persistent volumes so -if you want to completely reset the state of those services you will need to -remove volumes using the `docker volume rm` command. - The arion configuration runs these services: - connector: the MongoDB data connector agent defined in this repo (port 7130) @@ -99,13 +154,14 @@ Instead of a `docker-compose.yaml` configuration is found in `arion-compose.nix` ### Working with Test Data The arion configuration in the previous section preloads MongoDB with test data. -There is corresponding OpenDDN configuration in the `fixtures/` directory. +There is corresponding OpenDDN configuration in the `fixtures/hasura/` +directory. -The preloaded data is in the form of scripts in `fixtures/mongodb/`. Any `.js` +Preloaded databases are populated by scripts in `fixtures/mongodb/`. Any `.js` or `.sh` scripts added to this directory will be run when the mongodb service is run from a fresh state. Note that you will have to remove any existing docker volume to get to a fresh state. Using arion you can remove volumes by running -`arion down`. +`arion down --volumes`. ### Running with a different MongoDB version @@ -113,11 +169,11 @@ Override the MongoDB version that arion runs by assigning a Docker image name to the environment variable `MONGODB_IMAGE`. For example, $ arion down --volumes # delete potentially-incompatible MongoDB data - $ MONGODB_IMAGE=mongo:4 arion up -d + $ MONGODB_IMAGE=mongo:6 arion up -d Or run integration tests against a specific MongoDB version, - $ MONGODB_IMAGE=mongo:4 just test-integration + $ MONGODB_IMAGE=mongo:6 just test-integration ## License diff --git a/arion-compose/services/connector.nix b/arion-compose/services/connector.nix index a65e2c7e..f542619d 100644 --- a/arion-compose/services/connector.nix +++ b/arion-compose/services/connector.nix @@ -37,7 +37,7 @@ let MONGODB_DATABASE_URI = database-uri; OTEL_SERVICE_NAME = "mongodb-connector"; OTEL_EXPORTER_OTLP_ENDPOINT = otlp-endpoint; - RUST_LOG = "mongodb-connector=debug,dc_api=debug"; + RUST_LOG = "configuration=debug,mongodb_agent_common=debug,mongodb_connector=debug,mongodb_support=debug,ndc_query_plan=debug"; }; volumes = [ "${configuration-dir}:/configuration:ro" diff --git a/arion-compose/services/engine.nix b/arion-compose/services/engine.nix index 34f2f004..4050e0a1 100644 --- a/arion-compose/services/engine.nix +++ b/arion-compose/services/engine.nix @@ -88,6 +88,7 @@ in "--port=${port}" "--metadata-path=${metadata}" "--authn-config-path=${auth-config}" + "--expose-internal-errors" ] ++ (pkgs.lib.optionals (otlp-endpoint != null) [ "--otlp-endpoint=${otlp-endpoint}" ]); @@ -95,7 +96,7 @@ in "${hostPort}:${port}" ]; environment = { - RUST_LOG = "engine=debug,hasura-authn-core=debug"; + RUST_LOG = "engine=debug,hasura_authn_core=debug,hasura_authn_jwt=debug,hasura_authn_noauth=debug,hasura_authn_webhook=debug,lang_graphql=debug,open_dds=debug,schema=debug,metadata-resolve=debug"; }; healthcheck = { test = [ "CMD" "curl" "-f" "http://localhost:${port}/" ]; diff --git a/justfile b/justfile index 7c41f4e6..1092590d 100644 --- a/justfile +++ b/justfile @@ -1,9 +1,29 @@ -# Most of these tests assume that you are running in a nix develop shell. You -# can do that by running `$ nix develop`, or by setting up nix-direnv. +# Run commands in a nix develop shell by default which provides commands like +# `arion`. +set shell := ["nix", "--experimental-features", "nix-command flakes", "develop", "--command", "bash", "-c"] +# Display available recipes default: @just --list +# Run a local development environment using docker. This makes the GraphQL +# Engine available on https://localhost:7100/ with two connected MongoDB +# connector instances. +up: + arion up -d + +# Stop the local development environment docker containers. +down: + arion down + +# Stop the local development environment docker containers, and remove volumes. +down-volumes: + arion down --volumes + +# Output logs from local development environment services. +logs: + arion logs + test: test-unit test-integration test-unit: