这是indexloc提供的服务，不要输入任何密码
Skip to content

Fix catalog version for v1.1.1 (close #4354) #4355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 10, 2020
Merged

Fix catalog version for v1.1.1 (close #4354) #4355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b8cd080
Fix catalog version for v1.1.1
nizar-m Apr 9, 2020
dcd78d6
Remove entries of removed tables from hdb_catalog
nizar-m Apr 9, 2020
a70409c
Test downgrade command along with upgrade tests
nizar-m Apr 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ jobs:
apt install -y curl procps
- run:
name: upgrade_test
command: .circleci/server-upgrade/run.sh
command: .circleci/server-upgrade-downgrade/run.sh
environment:
HASURA_GRAPHQL_DATABASE_URL: postgresql://gql_test:@localhost:5432/gql_test
- store_artifacts:
Expand Down
0 .circleci/server-upgrade/.gitignore → ...cleci/server-upgrade-downgrade/.gitignore
View file
Open in desktop
File renamed without changes.
3 changes: 2 additions & 1 deletion .circleci/server-upgrade/run-dev.sh → ...cleci/server-upgrade-downgrade/run-dev.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ SERVER_DIR="$ROOT/../../server"
cd $SERVER_DIR
set -x
cabal new-build --project-file=cabal.project.dev-sh exe:graphql-engine
export SERVER_BINARY=$(cabal new-exec which graphql-engine)
export SERVER_BINARY="$(cabal new-exec --verbose=0 which graphql-engine)"
echo "server binary: $SERVER_BINARY"
cd -
set +x

Expand Down
169 changes: 110 additions & 59 deletions .circleci/server-upgrade/run.sh → .circleci/server-upgrade-downgrade/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,27 +110,42 @@ cur_server_version(){
log "Run pytests with server upgrade"

WORKTREE_DIR="$(mktemp -d)"
RELEASE_PYTEST_DIR="${WORKTREE_DIR}/server/tests-py"
RELEASE_VERSION="$( $LATEST_SERVER_BINARY version | cut -d':' -f2 | awk '{print $1}' )"
rm_worktree(){
rm -rf "$WORKTREE_DIR"
}
trap rm_worktree ERR

make_latest_release_worktree() {
version="$( $LATEST_SERVER_BINARY version | cut -d':' -f2 | awk '{print $1}' )"
git worktree add --detach "$WORKTREE_DIR" "$version"
git worktree add --detach "$WORKTREE_DIR" "$RELEASE_VERSION"
}

cleanup_hasura_metadata() {
set -x
psql "$HASURA_GRAPHQL_DATABASE_URL" -c 'drop schema if exists hdb_catalog cascade;
drop schema if exists hdb_views cascade' >/dev/null 2>/dev/null
set +x
cleanup_hasura_metadata_if_present() {
set -x
psql "$HASURA_GRAPHQL_DATABASE_URL" -c 'drop schema if exists hdb_catalog cascade;
drop schema if exists hdb_views cascade' >/dev/null 2>/dev/null
set +x
}

get_tables_of_interest() {
psql $HASURA_GRAPHQL_DATABASE_URL -P pager=off -c "
select table_schema as schema, table_name as name
from information_schema.tables
where table_schema not in ('hdb_catalog','hdb_views', 'pg_catalog', 'information_schema','topology', 'tiger')
and (table_schema <> 'public'
or table_name not in ('geography_columns','geometry_columns','spatial_ref_sys','raster_columns','raster_overviews')
);
"
}

get_current_catalog_version() {
psql $HASURA_GRAPHQL_DATABASE_URL -P pager=off -c "SELECT version FROM hdb_catalog.hdb_version"
}

args=("$@")
get_server_upgrade_tests() {
cd $PYTEST_DIR
cd $RELEASE_PYTEST_DIR
tmpfile="$(mktemp --dry-run)"
set -x
python3 -m pytest -q --collect-only --collect-upgrade-tests-to-file "$tmpfile" -m 'allow_server_upgrade_test and not skip_server_upgrade_test' "${args[@]}" 1>/dev/null 2>/dev/null
Expand All @@ -140,107 +155,143 @@ get_server_upgrade_tests() {
rm "$tmpfile"
}

# TODO: Fix me
# The right way to run the server-upgrade tests is get the worktree of the latest release, and run the pytests in it.
# This will not work for us for this release cycle (Since the old pytest do not have the options needed to run server-upgrade tests)
# So for now we are running pytests in the current build, but filtering out only the the tests which are also present in the previous release
# From the next release onwards, we should run tests corresponding to the latest release (and not pytests in the current build)
run_server_upgrade_pytest() {
local RELEASE_PYTEST_DIR="${WORKTREE_DIR}/server/tests-py"
local HGE_URL="http://localhost:${HASURA_GRAPHQL_SERVER_PORT}"
collect_common_tests() {
cd "$RELEASE_PYTEST_DIR"
set -x
pytest --hge-urls "${HGE_URL}" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --collect-only -q "$1" 2>/dev/null | grep '::'
cd - > /dev/null
HGE_PID=""
cleanup_hge(){
kill $HGE_PID || true
wait $HGE_PID || true
# cleanup_hasura_metadata_if_present
rm_worktree
}
trap cleanup_hge ERR
local HGE_URL="http://localhost:${HASURA_GRAPHQL_SERVER_PORT}"
local tests_to_run="$1"

[ -n "$1" ] || ( echo "Got no test as input" && false )
[ -n "$tests_to_run" ] || ( echo "Got no test as input" && false )

local tests_to_run="$(collect_common_tests $1)"
[ -z "$tests_to_run" ] && log "could not collect any tests for $1" && return
run_pytest(){
cd $RELEASE_PYTEST_DIR
set -x

log "Removing schemas with hasura metadata"
# This is required since we do not have a dowgrade command, yet.
cleanup_hasura_metadata
# With --avoid-error-message-checks, we are only going to throw warnings if the error message has changed between releases
pytest --hge-urls "${HGE_URL}" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" \
--avoid-error-message-checks "$@" \
-m 'allow_server_upgrade_test and not skip_server_upgrade_test' -v $tests_to_run
set +x
cd -
}

# Start the old GraphQL Engine
############## Tests for latest release GraphQL engine #########################

# Start the old (latest release) GraphQL Engine
log "starting latest graphql engine release"
$LATEST_SERVER_BINARY serve > $LATEST_SERVER_LOG 2>&1 &
HGE_PID=$!
cleanup_hge(){
kill $HGE_PID || true
wait $HGE_PID || true
cleanup_hasura_metadata || true
rm_worktree
}
trap cleanup_hge ERR


# Wait for server start
wait_for_port $HASURA_GRAPHQL_SERVER_PORT $HGE_PID $LATEST_SERVER_LOG

log "Catalog version for $(cur_server_version)"
get_current_catalog_version

log "Run pytest for latest graphql-engine release $(cur_server_version) while skipping schema teardown"
cd $PYTEST_DIR
set -x
# We are only going to throw warnings if the error message has changed between releases
pytest --hge-urls "${HGE_URL}" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --skip-schema-teardown --avoid-error-message-checks -m 'allow_server_upgrade_test and not skip_server_upgrade_test' -v $tests_to_run
set +x
cd -
run_pytest --skip-schema-teardown

log "kill the api server $(cur_server_version)"
kill $HGE_PID || true
wait $HGE_PID || true

log "the tables of interest in the database are: "
psql $HASURA_GRAPHQL_DATABASE_URL -P pager=off -c "
select table_schema as schema, table_name as name
from information_schema.tables
where table_schema not in ('hdb_catalog','hdb_views', 'pg_catalog', 'information_schema','topology', 'tiger')
and (table_schema <> 'public'
or table_name not in ('geography_columns','geometry_columns','spatial_ref_sys','raster_columns','raster_overviews')
);
"
get_tables_of_interest

############## Tests for the current build GraphQL engine #########################

if [[ "$1" =~ "test_schema_stitching" ]] ; then
# Hasura metadata would have GraphQL servers defined as remote.
# In such a case we need to have remote GraphQL server running
# for the graphql-engine to start
cd $PYTEST_DIR
# In this case, Hasura metadata will have GraphQL servers defined as remote.
# We need to have remote GraphQL server running for the graphql-engine to avoid
# inconsistent metadata error
cd $RELEASE_PYTEST_DIR
python3 graphql_server.py & REMOTE_GQL_PID=$!
wait_for_port 5000
cd -
fi

log "start the current build"
set -x
rm -f graphql-engine.tix
$SERVER_BINARY serve > $CURRENT_SERVER_LOG 2>&1 &
HGE_PID=$!
set +x

# Wait for server start
wait_for_port $HASURA_GRAPHQL_SERVER_PORT $HGE_PID $CURRENT_SERVER_LOG

log "Catalog version for $(cur_server_version)"
get_current_catalog_version

if [[ "$1" =~ "test_schema_stitching" ]] ; then
kill $REMOTE_GQL_PID || true
wait $REMOTE_GQL_PID || true
fi

log "Run pytest for the current build $(cur_server_version) while skipping schema setup "
cd $PYTEST_DIR
set -x
pytest -vv --hge-urls "$HGE_URL" --pg-urls "$HASURA_GRAPHQL_DATABASE_URL" --skip-schema-setup -v $tests_to_run
set +x
cd -
log "Run pytest for the current build $(cur_server_version) without modifying schema"
run_pytest --skip-schema-setup --skip-schema-teardown

log "kill the api server $(cur_server_version)"
kill $HGE_PID || true
wait $HGE_PID || true


#################### Downgrade to release version ##########################

log "Downgrade graphql-engine to $RELEASE_VERSION"
$SERVER_BINARY downgrade "--to-$RELEASE_VERSION"


############## Tests for latest release GraphQL engine once more after downgrade #########################

if [[ "$1" =~ "test_schema_stitching" ]] ; then
cd $RELEASE_PYTEST_DIR
python3 graphql_server.py & REMOTE_GQL_PID=$!
wait_for_port 5000
cd -
fi

# Start the old (latest release) GraphQL Engine
log "starting latest graphql engine release"
$LATEST_SERVER_BINARY serve > $LATEST_SERVER_LOG 2>&1 &
HGE_PID=$!

# Wait for server start
wait_for_port $HASURA_GRAPHQL_SERVER_PORT $HGE_PID $LATEST_SERVER_LOG

log "Catalog version for $(cur_server_version)"
get_current_catalog_version

if [[ "$1" =~ "test_schema_stitching" ]] ; then
kill $REMOTE_GQL_PID || true
wait $REMOTE_GQL_PID || true
fi

log "Run pytest for latest graphql-engine release $(cur_server_version) (once more) while skipping schema setup"
run_pytest --skip-schema-setup

log "kill the api server $(cur_server_version)"
kill $HGE_PID || true
wait $HGE_PID || true

}

make_latest_release_worktree

cleanup_hasura_metadata_if_present

for pytest in $(get_server_upgrade_tests) ; do
log "Running pytest $pytest"
run_server_upgrade_pytest "$pytest"
done
cleanup_hasura_metadata

cleanup_hasura_metadata_if_present

exit 0
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ The order and collapsed state of columns is now persisted across page navigation
- server: reserved keywords in column references break parser (fix #3597) #3927
- server: fix postgres specific error message that exposed database type on invalid query parameters (#4294)
- server: fix an edge case where some events wouldn't be processed because of internal erorrs (#4213)
- server: fix downgrade not working to version v1.1.1 (#4354)

## `v1.2.0-beta.3`

Expand Down
2 changes: 1 addition & 1 deletion server/src-rsr/catalog_versions.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ v1.1.0-beta.3 31
v1.1.0 31
v1.2.0-beta.1 32
v1.2.0-beta.2 32
v1.1.1 34
v1.1.1 31
2 changes: 2 additions & 0 deletions server/src-rsr/migrations/32_to_31.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ DROP TABLE hdb_catalog.hdb_action_permission;
DROP TABLE hdb_catalog.hdb_action;
DROP TABLE hdb_catalog.hdb_custom_types;
DROP TABLE hdb_catalog.hdb_action_log;
DELETE FROM hdb_catalog.hdb_relationship where table_schema = 'hdb_catalog' and table_name in ('hdb_action', 'hdb_action_permission', 'hdb_action_log', 'hdb_custom_types', 'hdb_role');
DELETE FROM hdb_catalog.hdb_table where table_schema = 'hdb_catalog' and table_name in ('hdb_action', 'hdb_action_permission', 'hdb_action_log', 'hdb_custom_types', 'hdb_role');
19 changes: 0 additions & 19 deletions server/tests-py/test_graphql_mutations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,6 @@ def test_user_err_unexpected_constraint(self, hge_ctx, transport):
def test_role_has_no_permissions_err(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/address_permission_error.yaml")

# This test captures a bug in the previous release
# Avoiding this test for server upgrades
@pytest.mark.skip_server_upgrade_test
def test_author_user_role_insert_check_perm_success(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/author_user_role_insert_check_perm_success.yaml")

Expand Down Expand Up @@ -252,11 +249,9 @@ def dir(cls):
# Those tests capture bugs in the previous release
class TestGraphqlNestedInserts:

@pytest.mark.skip_server_upgrade_test
def test_author_with_articles(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/author_with_articles.yaml")

@pytest.mark.skip_server_upgrade_test
def test_author_with_articles_empty(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/author_with_articles_empty.yaml")

Expand All @@ -266,7 +261,6 @@ def test_author_with_articles_null(self, hge_ctx, transport):
def test_author_with_articles_author_id_fail(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/author_with_articles_author_id_fail.yaml")

@pytest.mark.skip_server_upgrade_test
def test_articles_with_author(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/articles_with_author.yaml")

Expand All @@ -279,7 +273,6 @@ def test_author_upsert_articles_fail(self, hge_ctx, transport):
def test_articles_author_upsert_fail(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/articles_author_upsert_fail.yaml")

@pytest.mark.skip_server_upgrade_test
def test_articles_with_author_returning(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/articles_with_author_returning.yaml")

Expand Down Expand Up @@ -310,9 +303,6 @@ def test_insert_view_author_simple(self, hge_ctx, transport):
def test_insert_view_author_complex_fail(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/insert_view_author_complex_fail.yaml")

# This test captures a bug in the previous release
# Avoiding this test for server upgrades
@pytest.mark.skip_server_upgrade_test
def test_nested_insert_article_author_simple_view(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/nested_insert_article_author_simple_view.yaml")

Expand All @@ -328,9 +318,6 @@ def dir(cls):
@use_mutation_fixtures
class TestGraphqlUpdateBasic:

# This test captures a bug in the previous release
# Avoiding this test for server upgrades
@pytest.mark.skip_server_upgrade_test
def test_set_author_name(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/author_set_name.yaml")

Expand Down Expand Up @@ -395,9 +382,6 @@ def dir(cls):
@use_mutation_fixtures
class TestGraphqlUpdatePermissions:

# This test captures a bug in the previous release
# Avoiding this test for server upgrades
@pytest.mark.skip_server_upgrade_test
def test_user_update_author(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + "/user_update_author.yaml")

Expand Down Expand Up @@ -505,9 +489,6 @@ class TestGraphqlMutationCustomSchema:
def test_insert_author(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + '/insert_author.yaml', transport)

# This test captures a bug in the previous release
# Avoiding this test for server upgrades
@pytest.mark.skip_server_upgrade_test
def test_insert_article_author(self, hge_ctx, transport):
check_query_f(hge_ctx, self.dir() + '/insert_article_author.yaml', transport)

Expand Down