From da78a77fb2dd685388d9e8e77c6093320250109e Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 11:32:11 -0500 Subject: [PATCH 1/8] In tests/dev.sh: upgrade ruamel to support python 3.7. Closes #3312 Tested on python 3.5 and 3.7 We make light use of pyenv to set an appropriate python version if installed. We could easily install a correct version too if we wanted but that seemed invasive. The newer ruamel was an annoying upgrade but also offers some improvements that exposed some test suite issues (fixed later). --- .gitignore | 1 + scripts/dev.sh | 46 ++++++++++++++++++++- server/CONTRIBUTING.md | 1 + server/tests-py/context.py | 5 ++- server/tests-py/requirements-top-level.txt | 4 +- server/tests-py/requirements.txt | 48 +++++++++++----------- server/tests-py/validate.py | 43 +++++++++++++++---- 7 files changed, 111 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 28736796769e6..df0feb75c363d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .hasura-dev-python-venv +server/tests-py/.devsh_version npm-debug.log *.temp *.DS_Store diff --git a/scripts/dev.sh b/scripts/dev.sh index e7d493d7c170d..cfbb6727694c8 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -10,6 +10,12 @@ set -euo pipefail echo_pretty() { echo ">>> $(tput setaf 2)$1$(tput sgr0)" } +echo_error() { + echo ">>> $(tput setaf 1)$1$(tput sgr0)" +} +echo_warn() { + echo ">>> $(tput setaf 3)$1$(tput sgr0)" +} die_usage() { cat </dev/null 2>&1 && pwd )" # ... https://stackoverflow.com/a/246128/176841 cd "$PROJECT_ROOT" +# Use pyenv if available to set an appropriate python version that will work with pytests etc. +if command -v pyenv ; then + # For now I guess use the greatest python3 >= 3.5 + v=$(pyenv versions --bare | (grep '^ *3' || true) | awk '{if($1>=3.5)print$1}' | tail -n1) + if [ -z "$v" ]; then + echo_error 'Please `pyenv install` a version of python >= 3.5 so we can use it' + exit 2 + fi + echo_pretty "Pyenv found. Using python version: $v" + export PYENV_VERSION=$v + python3 --version +else + echo_warn "Pyenv not installed. Proceeding with system python version: $(python3 --version)" +fi #################################### ### Shared environment stuff ### @@ -299,14 +323,33 @@ elif [ "$MODE" = "test" ]; then done echo " Ok" + ### Check for and install dependencies in venv cd "$PROJECT_ROOT/server/tests-py" PY_VENV=.hasura-dev-python-venv + DEVSH_VERSION_FILE=.devsh_version + # Do we need to force reinstall? + if [ "$DEVSH_VERSION" = "$(cat $DEVSH_VERSION_FILE 2>/dev/null || true)" ]; then + true # ok + else + echo_warn 'dev.sh version was bumped. Forcing reinstallation of dependencies.' + rm -r "$PY_VENV" + echo "$DEVSH_VERSION" > "$DEVSH_VERSION_FILE" + fi set +u # for venv activate if [ ! -d "$PY_VENV" ]; then python3 -m venv "$PY_VENV" source "$PY_VENV/bin/activate" pip3 install wheel - pip3 install -r requirements.txt + # If the maintainer of this script or pytests needs to change dependencies: + # - alter requirements-top-level.txt as needed + # - delete requirements.txt + # - run this script, then check in the new frozen requirements.txt + if [ -f requirements.txt ]; then + pip3 install -r requirements.txt + else + pip3 install -r requirements-top-level.txt + pip3 freeze > requirements.txt + fi else echo_pretty "It looks like python dependencies have been installed already. Skipping." echo_pretty "If things fail please run this and try again" @@ -315,6 +358,7 @@ elif [ "$MODE" = "test" ]; then source "$PY_VENV/bin/activate" fi + # TODO MAYBE: fix deprecation warnings, make them an error if pytest -W ignore::DeprecationWarning --hge-urls http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT --pg-urls "$DB_URL" $PYTEST_ARGS; then PASSED=true diff --git a/server/CONTRIBUTING.md b/server/CONTRIBUTING.md index cab2daeff2daa..6d383e58beab1 100644 --- a/server/CONTRIBUTING.md +++ b/server/CONTRIBUTING.md @@ -11,6 +11,7 @@ own machine and how to contribute. - brotli - libpq-dev - python >= 3.5 with pip3 +- [pyenv](https://github.com/pyenv/pyenv) (Recommended, not required) The Brotli can be installed from source using `git`, `cmake` and `pkgconf` on Debian with: diff --git a/server/tests-py/context.py b/server/tests-py/context.py index e18c510e5ee15..c4d269807f4a2 100644 --- a/server/tests-py/context.py +++ b/server/tests-py/context.py @@ -309,8 +309,9 @@ def v1q(self, q, headers = {}): def v1q_f(self, fn): with open(fn) as f: - # NOTE: preserve ordering with RoundTripLoader: - return self.v1q(yaml.load(f, yaml.RoundTripLoader)) + # NOTE: preserve ordering with ruamel + yml = yaml.YAML() + return self.v1q(yml.load(f)) def teardown(self): self.http.close() diff --git a/server/tests-py/requirements-top-level.txt b/server/tests-py/requirements-top-level.txt index 41a40eeb6cec5..fb243ea616abc 100644 --- a/server/tests-py/requirements-top-level.txt +++ b/server/tests-py/requirements-top-level.txt @@ -1,6 +1,6 @@ psycopg2-binary sqlalchemy -pytest +pytest==4.5.0 pytest-xdist requests pyyaml @@ -10,5 +10,5 @@ jsondiff cryptography graphene brotlipy -ruamel.yaml < 0.15 +ruamel.yaml > 0.15 graphql-core diff --git a/server/tests-py/requirements.txt b/server/tests-py/requirements.txt index 0c8791214cfa1..28f78a0ad4fde 100644 --- a/server/tests-py/requirements.txt +++ b/server/tests-py/requirements.txt @@ -1,37 +1,37 @@ -aniso8601==3.0.2 +aniso8601==7.0.0 apipkg==1.5 -asn1crypto==0.24.0 atomicwrites==1.3.0 -attrs==19.1.0 -certifi==2019.3.9 -cffi==1.12.3 +attrs==19.3.0 +brotlipy==0.7.0 +certifi==2019.9.11 +cffi==1.13.2 chardet==3.0.4 -cryptography==2.7 -execnet==1.6.0 -graphene==2.1.3 -graphql-core==2.1 -graphql-relay==0.4.5 +cryptography==2.8 +execnet==1.7.1 +graphene==2.1.8 +graphql-core==2.2.1 +graphql-relay==2.0.0 idna==2.8 -importlib-metadata==0.17 -jsondiff==1.1.2 -more-itertools==7.0.0 -pluggy==0.12.0 +importlib-metadata==0.23 +jsondiff==1.2.0 +more-itertools==7.2.0 +pluggy==0.13.0 promise==2.2.1 -psycopg2-binary==2.8.2 +psycopg2-binary==2.8.4 py==1.8.0 pycparser==2.19 PyJWT==1.7.1 pytest==4.5.0 -pytest-forked==1.0.2 -pytest-xdist==1.28.0 -PyYAML==5.1 +pytest-forked==1.1.3 +pytest-xdist==1.30.0 +PyYAML==5.1.2 requests==2.22.0 +ruamel.yaml==0.16.5 +ruamel.yaml.clib==0.2.0 Rx==1.6.1 -six==1.12.0 -SQLAlchemy==1.3.4 -urllib3==1.25.3 +six==1.13.0 +SQLAlchemy==1.3.11 +urllib3==1.25.7 wcwidth==0.1.7 websocket-client==0.56.0 -zipp==0.5.1 -brotlipy==0.7.0 -ruamel.yaml==0.14.12 +zipp==0.6.0 diff --git a/server/tests-py/validate.py b/server/tests-py/validate.py index 0e25fb7707352..151be1c6f42b2 100644 --- a/server/tests-py/validate.py +++ b/server/tests-py/validate.py @@ -2,6 +2,8 @@ import pytest import ruamel.yaml as yaml +from ruamel.yaml.compat import ordereddict, StringIO +from ruamel.yaml.comments import CommentedMap import json import copy import graphql @@ -250,12 +252,15 @@ def assert_graphql_resp_expected(resp_orig, exp_response_orig, query): # consideration only the ordering that we care about: resp = collapse_order_not_selset(resp_orig, query) exp_response = collapse_order_not_selset(exp_response_orig, query) - matched = resp == exp_response + matched = equal_CommentedMap(resp, exp_response) if pytest.config.getoption("--accept"): print('skipping assertion since we chose to --accept new output') else: - assert matched, '\n' + yaml.dump({ + yml = yaml.YAML() + # https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string : + dump_str = StringIO() + yml.dump({ # Keep strict received order when displaying errors: 'response': resp_orig, 'expected': exp_response_orig, @@ -263,9 +268,30 @@ def assert_graphql_resp_expected(resp_orig, exp_response_orig, query): (lambda diff: "(results differ only in their order of keys)" if diff == {} else diff) (stringify_keys(jsondiff.diff(exp_response, resp))) - }, Dumper=yaml.RoundTripDumper ) + }, stream=dump_str) + assert matched, dump_str.getvalue() return resp, matched # matched always True unless --accept +# This really sucks; newer ruamel made __eq__ ignore ordering: +# https://bitbucket.org/ruamel/yaml/issues/326/commentedmap-equality-no-longer-takes-into +def equal_CommentedMap(m1, m2): + if isinstance(m1, list) and isinstance(m2, list): + return (len(m1) == len(m2) and all(equal_CommentedMap(x,y) for x,y in zip(m1,m2))) + elif isinstance(m1, dict) and isinstance(m2, dict): + # (see collapse_order_not_selset): + if isinstance(m1, (ordereddict, CommentedMap)) and \ + isinstance(m2, (ordereddict, CommentedMap)): + m1_l = list(m1.items()) + m2_l = list(m2.items()) + else: + m1_l = sorted(list(m1.items())) + m2_l = sorted(list(m2.items())) + return (len(m1_l) == len(m2_l) and + all(k1 == k2 and equal_CommentedMap(v1,v2) + for (k1,v1),(k2,v2) in zip(m1_l,m2_l))) + # else this is a scalar: + else: + return m1 == m2 def check_query_f(hge_ctx, f, transport='http', add_auth=True): print("Test file: " + f) @@ -275,10 +301,11 @@ def check_query_f(hge_ctx, f, transport='http', add_auth=True): # For `--accept`: should_write_back = False - # ruamel RoundTripLoader will preserve order so that we can test the - # JSON ordering property conforms to YAML spec. - # It also lets us write back the yaml nicely when we --accept. - conf = yaml.load(c, yaml.RoundTripLoader) + # ruamel will preserve order so that we can test the JSON ordering + # property conforms to YAML spec. It also lets us write back the yaml + # nicely when we `--accept.` + yml = yaml.YAML() + conf = yml.load(c) if isinstance(conf, list): for ix, sconf in enumerate(conf): actual_resp, matched = check_query(hge_ctx, sconf, transport, add_auth) @@ -305,7 +332,7 @@ def check_query_f(hge_ctx, f, transport='http', add_auth=True): "\n NOTE: if this case was marked 'xfail' this won't be correct!" ) c.seek(0) - c.write(yaml.dump(conf, Dumper=yaml.RoundTripDumper)) + c.write(yml.dump(conf)) c.truncate() From 92b1f9c93ddd8e1edb34cc588f399df9e8f0e9e2 Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 11:35:43 -0500 Subject: [PATCH 2/8] In tests: fix extraneous extra keys in yaml --- .../v1/relationships/object_relationship_foreign_key.yaml | 2 -- .../queries/v1/relationships/object_relationship_manual.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/server/tests-py/queries/v1/relationships/object_relationship_foreign_key.yaml b/server/tests-py/queries/v1/relationships/object_relationship_foreign_key.yaml index 91ac523790a28..3d34235809e93 100644 --- a/server/tests-py/queries/v1/relationships/object_relationship_foreign_key.yaml +++ b/server/tests-py/queries/v1/relationships/object_relationship_foreign_key.yaml @@ -103,8 +103,6 @@ status: 200 response: message: success - query: - query: query: type: drop_relationship args: diff --git a/server/tests-py/queries/v1/relationships/object_relationship_manual.yaml b/server/tests-py/queries/v1/relationships/object_relationship_manual.yaml index e4ccacb541a43..5559514d566db 100644 --- a/server/tests-py/queries/v1/relationships/object_relationship_manual.yaml +++ b/server/tests-py/queries/v1/relationships/object_relationship_manual.yaml @@ -106,8 +106,6 @@ status: 200 response: message: success - query: - query: query: type: drop_relationship args: From a1da8bcfdde2e8022d129db0e1ee6fdd9615956d Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 11:36:30 -0500 Subject: [PATCH 3/8] Fix product_mul_price test The intention was to make this two cases, using a top-level YAML list. The result was one test with duplicate keys (effectively only running the second test). This is an error that's now flagged by newer ruamel. Both tests needed to be "corrected" to pass and need review. --- .../v1/update/basic/product_mul_price.yaml | 89 +++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/server/tests-py/queries/v1/update/basic/product_mul_price.yaml b/server/tests-py/queries/v1/update/basic/product_mul_price.yaml index f4b1b01ed2832..4a4706a8a2543 100644 --- a/server/tests-py/queries/v1/update/basic/product_mul_price.yaml +++ b/server/tests-py/queries/v1/update/basic/product_mul_price.yaml @@ -1,47 +1,46 @@ -description: Updated product price using _mul operator -url: /v1/query -status: 200 -response: - affected_rows: 2 - returning: - - product_id: 1 - name: Product 1 - price: 15 - - product_id: 2 - name: Product 2 - price: 45 -query: - type: update - args: - table: products - $mul: - price: 3 - where: - product_id: - $gte: 1 +- description: Updated product price using _mul operator + url: /v1/query + status: 200 + response: + affected_rows: 2 returning: - - product_id - - name - - price - -description: Updated product price using _mul operator -url: /v1/query -status: 200 -response: - affected_rows: 1 - returning: - - product_id: 2 - name: Product 2 - price: '16.5000' -query: - type: update - args: - table: products - $mul: - price: 1.1 - where: - product_id: 2 + - product_id: 1 + name: Product 1 + price: '15' + - product_id: 2 + name: Product 2 + price: '45' + query: + type: update + args: + table: products + $mul: + price: 3 + where: + product_id: + $gte: 1 + returning: + - product_id + - name + - price +- description: Updated product price using _mul operator + url: /v1/query + status: 200 + response: + affected_rows: 1 returning: - - product_id - - name - - price + - product_id: 2 + name: Product 2 + price: '49.5000' + query: + type: update + args: + table: products + $mul: + price: 1.1 + where: + product_id: 2 + returning: + - product_id + - name + - price From 9e0e42444b1dc97404c81b85d778c4f22476309e Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 11:41:41 -0500 Subject: [PATCH 4/8] Add 'venv' to top gitignore, as standard name convention lsfor python venv's in this project --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index df0feb75c363d..0f1abbc6abdbb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,8 @@ test-server-flags-output .vscode .idea +# Standardize on 'venv' as the name for python venv directories +venv + # Test artifacts *.tix From 5827d652f6f04df27a767b67247be77a94531385 Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 12:12:45 -0500 Subject: [PATCH 5/8] In dev.sh tests: use postgres:11.5-alpine-postgis image which boots 3 sec faster --- scripts/dev.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/dev.sh b/scripts/dev.sh index cfbb6727694c8..0c276473c9dd6 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -225,9 +225,8 @@ EOF CONF_FLAGS=$(echo "$CONF" | sed -e 's/^/-c /' | tr '\n' ' ') -# The unofficial 'mdillon/postgis' comes with postgis installed, needed for tests: docker run --name "$PG_CONTAINER_NAME" -p 127.0.0.1:"$PG_PORT":$PG_PORT --expose="$PG_PORT" \ - -e POSTGRES_PASSWORD="$PGPASSWORD" -d mdillon/postgis:11 \ + -e POSTGRES_PASSWORD="$PGPASSWORD" -d circleci/postgres:11.5-alpine-postgis \ $CONF_FLAGS @@ -263,8 +262,9 @@ function cleanup { # Since scripts here are tailored to the env we've just launched: rm -r "$DEV_SHIM_PATH" - echo_pretty "Removing $PG_CONTAINER_NAME and its volumes in 5 seconds! PRESS CTRL-C TO ABORT." - sleep 5 + echo_pretty "Removing $PG_CONTAINER_NAME and its volumes in 5 seconds!" + echo_pretty " PRESS CTRL-C TO ABORT removal, or ENTER to clean up right away" + read -t5 || true docker stop "$PG_CONTAINER_NAME" docker rm -v "$PG_CONTAINER_NAME" ;; From 256be84dba7ef293354ccf5f943dfe550e8aace9 Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 13:03:33 -0500 Subject: [PATCH 6/8] dev.sh script: build before waiting for PG --- scripts/dev.sh | 92 ++++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/scripts/dev.sh b/scripts/dev.sh index 0c276473c9dd6..b132917ac1014 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -79,7 +79,7 @@ PROJECT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd cd "$PROJECT_ROOT" # Use pyenv if available to set an appropriate python version that will work with pytests etc. -if command -v pyenv ; then +if command -v pyenv >/dev/null; then # For now I guess use the greatest python3 >= 3.5 v=$(pyenv versions --bare | (grep '^ *3' || true) | awk '{if($1>=3.5)print$1}' | tail -n1) if [ -z "$v" ]; then @@ -136,7 +136,7 @@ if [ "$MODE" = "graphql-engine" ]; then export HASURA_GRAPHQL_SERVER_PORT=${HASURA_GRAPHQL_SERVER_PORT-8181} # Prettify JSON output if possible: - if command -v jq ; then + if command -v jq >/dev/null; then PIPE_JQ="| jq --unbuffered -R -r '. as \$line | try fromjson catch \$line'" fi @@ -144,7 +144,6 @@ if [ "$MODE" = "graphql-engine" ]; then echo_pretty "If you haven't yet, please launch a postgres container in a separate terminal with:" echo_pretty " $ $0 postgres" echo_pretty "or press CTRL-C and invoke graphql-engine manually" - wait_docker_postgres RUN_INVOCATION="stack exec graphql-engine -- --database-url='$DB_URL' serve --enable-console --console-assets-dir \'$PROJECT_ROOT/console/static/dist\' +RTS -N -T -RTS ${PIPE_JQ-}" @@ -164,6 +163,7 @@ if [ "$MODE" = "graphql-engine" ]; then else $BUILD_INVOCATION fi + wait_docker_postgres # Print helpful info after startup logs so it's visible: { @@ -205,7 +205,6 @@ fi ### Postgres Container ### ################################# - # Useful development defaults for postgres (no spaces here, please): # # setting 'port' in container is a workaround for the pg_dump endpoint (see tests) @@ -224,18 +223,18 @@ EOF # log lines above as -c flag arguments we pass to postgres CONF_FLAGS=$(echo "$CONF" | sed -e 's/^/-c /' | tr '\n' ' ') - -docker run --name "$PG_CONTAINER_NAME" -p 127.0.0.1:"$PG_PORT":$PG_PORT --expose="$PG_PORT" \ - -e POSTGRES_PASSWORD="$PGPASSWORD" -d circleci/postgres:11.5-alpine-postgis \ - $CONF_FLAGS - - -# graphql-engine calls the pg_dump executable. To avoid a version mismatch (and -# the dependency entirely) we create a shim that executes the pg_dump in the -# postgres container. Note output to file won't work. -DEV_SHIM_PATH="/tmp/hasura-dev-shims-$PG_PORT" -mkdir -p "$DEV_SHIM_PATH" -cat >"$DEV_SHIM_PATH/pg_dump" <"$DEV_SHIM_PATH/pg_dump" </dev/null || true - fi + if [ ! -z "${GRAPHQL_ENGINE_PID-}" ]; then + # This may already have been killed: + kill "$GRAPHQL_ENGINE_PID" &>/dev/null || true + fi - case "$MODE" in - test|postgres) - # Since scripts here are tailored to the env we've just launched: - rm -r "$DEV_SHIM_PATH" - - echo_pretty "Removing $PG_CONTAINER_NAME and its volumes in 5 seconds!" - echo_pretty " PRESS CTRL-C TO ABORT removal, or ENTER to clean up right away" - read -t5 || true - docker stop "$PG_CONTAINER_NAME" - docker rm -v "$PG_CONTAINER_NAME" - ;; - graphql-engine) - ;; - esac - - echo_pretty "Done" -} -trap cleanup EXIT + case "$MODE" in + test|postgres) + # Since scripts here are tailored to the env we've just launched: + rm -r "$DEV_SHIM_PATH" -wait_docker_postgres + echo_pretty "Removing $PG_CONTAINER_NAME and its volumes in 5 seconds!" + echo_pretty " PRESS CTRL-C TO ABORT removal, or ENTER to clean up right away" + read -t5 || true + docker stop "$PG_CONTAINER_NAME" + docker rm -v "$PG_CONTAINER_NAME" + ;; + graphql-engine) + ;; + esac + + echo_pretty "Done" + } + trap cleanup EXIT +} if [ "$MODE" = "postgres" ]; then + launch_postgres_container + wait_docker_postgres echo_pretty "Postgres logs will start to show up in realtime here. Press CTRL-C to exit and " echo_pretty "shutdown this container." echo_pretty "" @@ -308,6 +308,10 @@ elif [ "$MODE" = "test" ]; then echo_pretty "Rebuilding for code coverage" $BUILD_INVOCATION --coverage + # It's better UX to build first (possibly failing) before trying to launch PG: + launch_postgres_container + wait_docker_postgres + echo_pretty "Running Haskell test suite" HASURA_GRAPHQL_DATABASE_URL="$DB_URL" $TEST_INVOCATION --coverage From 505ea4deef3ec8726ce68602b6e811813476ab43 Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 13:33:30 -0500 Subject: [PATCH 7/8] dev.sh: support running only integration tests or haskell/unit tests individually --- scripts/dev.sh | 156 ++++++++++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 66 deletions(-) diff --git a/scripts/dev.sh b/scripts/dev.sh index b132917ac1014..cb10d9c15dcf5 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -34,10 +34,12 @@ Available COMMANDs: Launch a postgres container suitable for use with graphql-engine, watch its logs, clean up nicely after - test [pytest_args...] - Run the integration tests, handling spinning up all dependencies. This will force - a recompile. A code coverage report will be generated. All arguments after 'test' - will be passed to the 'pytest' invocation. + test [--integration [pytest_args...] | --unit] + Run the unit and integration tests, handling spinning up all dependencies. + This will force a recompile. A code coverage report will be generated. + Either integration or unit tests can be run individually with their + respective flags. With '--integration' any arguments that follow will be + passed to the pytest invocation EOL exit 1 @@ -64,7 +66,24 @@ case "${1-}" in postgres) ;; test) - PYTEST_ARGS="${@:2}" + case "${2-}" in + --unit) + RUN_INTEGRATION_TESTS=false + RUN_UNIT_TESTS=true + ;; + --integration) + PYTEST_ARGS="${@:3}" + RUN_INTEGRATION_TESTS=true + RUN_UNIT_TESTS=false + ;; + "") + RUN_INTEGRATION_TESTS=true + RUN_UNIT_TESTS=true + ;; + *) + die_usage + ;; + esac ;; *) die_usage @@ -312,74 +331,79 @@ elif [ "$MODE" = "test" ]; then launch_postgres_container wait_docker_postgres - echo_pretty "Running Haskell test suite" - HASURA_GRAPHQL_DATABASE_URL="$DB_URL" $TEST_INVOCATION --coverage - - echo_pretty "Starting graphql-engine" - GRAPHQL_ENGINE_TEST_LOG=/tmp/hasura-dev-test-engine.log - export HASURA_GRAPHQL_SERVER_PORT=8088 - # stopped in cleanup() - stack exec graphql-engine -- --database-url="$DB_URL" serve --enable-console --stringify-numeric-types \ - --console-assets-dir ../console/static/dist &> $GRAPHQL_ENGINE_TEST_LOG & GRAPHQL_ENGINE_PID=$! - echo -n "Waiting for graphql-engine" - until curl -s "http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT/v1/query" &>/dev/null; do - echo -n '.' && sleep 0.2 - done - echo " Ok" - - ### Check for and install dependencies in venv - cd "$PROJECT_ROOT/server/tests-py" - PY_VENV=.hasura-dev-python-venv - DEVSH_VERSION_FILE=.devsh_version - # Do we need to force reinstall? - if [ "$DEVSH_VERSION" = "$(cat $DEVSH_VERSION_FILE 2>/dev/null || true)" ]; then - true # ok - else - echo_warn 'dev.sh version was bumped. Forcing reinstallation of dependencies.' - rm -r "$PY_VENV" - echo "$DEVSH_VERSION" > "$DEVSH_VERSION_FILE" + # These also depend on a running DB: + if [ "$RUN_UNIT_TESTS" = true ]; then + echo_pretty "Running Haskell test suite" + HASURA_GRAPHQL_DATABASE_URL="$DB_URL" $TEST_INVOCATION --coverage fi - set +u # for venv activate - if [ ! -d "$PY_VENV" ]; then - python3 -m venv "$PY_VENV" - source "$PY_VENV/bin/activate" - pip3 install wheel - # If the maintainer of this script or pytests needs to change dependencies: - # - alter requirements-top-level.txt as needed - # - delete requirements.txt - # - run this script, then check in the new frozen requirements.txt - if [ -f requirements.txt ]; then - pip3 install -r requirements.txt + + if [ "$RUN_INTEGRATION_TESTS" = true ]; then + echo_pretty "Starting graphql-engine" + GRAPHQL_ENGINE_TEST_LOG=/tmp/hasura-dev-test-engine.log + export HASURA_GRAPHQL_SERVER_PORT=8088 + # stopped in cleanup() + stack exec graphql-engine -- --database-url="$DB_URL" serve --enable-console --stringify-numeric-types \ + --console-assets-dir ../console/static/dist &> $GRAPHQL_ENGINE_TEST_LOG & GRAPHQL_ENGINE_PID=$! + echo -n "Waiting for graphql-engine" + until curl -s "http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT/v1/query" &>/dev/null; do + echo -n '.' && sleep 0.2 + done + echo " Ok" + + ### Check for and install dependencies in venv + cd "$PROJECT_ROOT/server/tests-py" + PY_VENV=.hasura-dev-python-venv + DEVSH_VERSION_FILE=.devsh_version + # Do we need to force reinstall? + if [ "$DEVSH_VERSION" = "$(cat $DEVSH_VERSION_FILE 2>/dev/null || true)" ]; then + true # ok else - pip3 install -r requirements-top-level.txt - pip3 freeze > requirements.txt + echo_warn 'dev.sh version was bumped. Forcing reinstallation of dependencies.' + rm -r "$PY_VENV" + echo "$DEVSH_VERSION" > "$DEVSH_VERSION_FILE" fi - else - echo_pretty "It looks like python dependencies have been installed already. Skipping." - echo_pretty "If things fail please run this and try again" - echo_pretty " $ rm -r \"$PROJECT_ROOT/server/tests-py/$PY_VENV\"" + set +u # for venv activate + if [ ! -d "$PY_VENV" ]; then + python3 -m venv "$PY_VENV" + source "$PY_VENV/bin/activate" + pip3 install wheel + # If the maintainer of this script or pytests needs to change dependencies: + # - alter requirements-top-level.txt as needed + # - delete requirements.txt + # - run this script, then check in the new frozen requirements.txt + if [ -f requirements.txt ]; then + pip3 install -r requirements.txt + else + pip3 install -r requirements-top-level.txt + pip3 freeze > requirements.txt + fi + else + echo_pretty "It looks like python dependencies have been installed already. Skipping." + echo_pretty "If things fail please run this and try again" + echo_pretty " $ rm -r \"$PROJECT_ROOT/server/tests-py/$PY_VENV\"" - source "$PY_VENV/bin/activate" - fi + source "$PY_VENV/bin/activate" + fi - # TODO MAYBE: fix deprecation warnings, make them an error - if pytest -W ignore::DeprecationWarning --hge-urls http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT --pg-urls "$DB_URL" $PYTEST_ARGS; then - PASSED=true - else - PASSED=false - echo_pretty "^^^ graphql-engine logs from failed test run can be inspected at: $GRAPHQL_ENGINE_TEST_LOG" - fi - deactivate # python venv - set -u + # TODO MAYBE: fix deprecation warnings, make them an error + if pytest -W ignore::DeprecationWarning --hge-urls http://127.0.0.1:$HASURA_GRAPHQL_SERVER_PORT --pg-urls "$DB_URL" $PYTEST_ARGS; then + PASSED=true + else + PASSED=false + echo_pretty "^^^ graphql-engine logs from failed test run can be inspected at: $GRAPHQL_ENGINE_TEST_LOG" + fi + deactivate # python venv + set -u - cd "$PROJECT_ROOT/server" - # INT so we get hpc report - kill -INT "$GRAPHQL_ENGINE_PID" - wait "$GRAPHQL_ENGINE_PID" || true - echo - stack hpc report graphql-engine.tix - rm graphql-engine.tix + cd "$PROJECT_ROOT/server" + # INT so we get hpc report + kill -INT "$GRAPHQL_ENGINE_PID" + wait "$GRAPHQL_ENGINE_PID" || true + echo + stack hpc report graphql-engine.tix + rm graphql-engine.tix + fi else echo "impossible; fix script." From 84cfe18b3cbe0c31d3c280353190cc58808251d7 Mon Sep 17 00:00:00 2001 From: Brandon Simmons Date: Thu, 14 Nov 2019 14:32:27 -0500 Subject: [PATCH 8/8] dev.sh: combine tix from unit/haskell tests and integration tests in final report --- scripts/dev.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/dev.sh b/scripts/dev.sh index cb10d9c15dcf5..473b346427758 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -36,7 +36,8 @@ Available COMMANDs: test [--integration [pytest_args...] | --unit] Run the unit and integration tests, handling spinning up all dependencies. - This will force a recompile. A code coverage report will be generated. + This will force a recompile. A combined code coverage report will be + generated for all test suites. Either integration or unit tests can be run individually with their respective flags. With '--integration' any arguments that follow will be passed to the pytest invocation @@ -401,9 +402,15 @@ elif [ "$MODE" = "test" ]; then kill -INT "$GRAPHQL_ENGINE_PID" wait "$GRAPHQL_ENGINE_PID" || true echo - stack hpc report graphql-engine.tix - rm graphql-engine.tix + # Combine any tix from haskell/unit tests: + if [ "$RUN_UNIT_TESTS" = true ]; then + stack exec hpc -- combine "$(stack path --local-hpc-root)/graphql-engine/graphql-engine-tests/graphql-engine-tests.tix" graphql-engine.tix --union > graphql-engine-combined.tix + stack hpc report graphql-engine-combined.tix + else + stack hpc report graphql-engine.tix + fi fi + rm -f graphql-engine.tix graphql-engine-combined.tix else echo "impossible; fix script."