From d0f1d3b5f5e499e547aa01ffea1a294e82f69df3 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Mon, 22 Jun 2020 16:09:15 +0530 Subject: [PATCH 01/14] assume empty files for metdata files which are not found --- cli/migrate/database/hasuradb/metadata.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/migrate/database/hasuradb/metadata.go b/cli/migrate/database/hasuradb/metadata.go index 0b7d92b3ecf8b..688ec093d67db 100644 --- a/cli/migrate/database/hasuradb/metadata.go +++ b/cli/migrate/database/hasuradb/metadata.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" gyaml "github.com/ghodss/yaml" "github.com/hasura/graphql-engine/cli/metadata/types" @@ -151,6 +152,10 @@ func (h *HasuraDB) BuildMetadata() (yaml.MapSlice, error) { for _, plg := range h.config.Plugins { err := plg.Build(&tmpMeta) if err != nil { + if os.IsNotExist(errors.Cause(err)) { + h.logger.Debugf("metadata file for %s was not found, assuming an empty files", plg.Name()) + continue + } return tmpMeta, errors.Wrap(err, fmt.Sprintf("cannot build %s from metadata", plg.Name())) } } From b2074cf94fe4003a6cf18efc71dbf0b63d26fa64 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Mon, 22 Jun 2020 17:28:08 +0530 Subject: [PATCH 02/14] add tests --- cli/go.sum | 2 + cli/integration_test/cli_test.go | 50 +++++++++++ .../incomplete_metadata/t1/from-server.golden | 4 + .../t1/metadata/actions.graphql | 18 ++++ .../t1/metadata/allow_list.yaml | 1 + .../t1/metadata/functions.yaml | 1 + .../t1/metadata/query_collections.yaml | 1 + .../t1/metadata/remote_schemas.yaml | 1 + .../t1/metadata/tables.yaml | 1 + .../t1/metadata/version.yaml | 1 + .../incomplete_metadata/t2/from-server.golden | 49 +++++++++++ .../t2/metadata/actions.graphql | 18 ++++ .../t2/metadata/actions.yaml | 12 +++ .../t2/metadata/allow_list.yaml | 1 + .../t2/metadata/cron_triggers.yaml | 1 + .../t2/metadata/functions.yaml | 1 + .../t2/metadata/query_collections.yaml | 1 + .../t2/metadata/remote_schemas.yaml | 1 + .../t2/metadata/tables.yaml | 1 + .../t2/metadata/version.yaml | 1 + cli/integration_test/v2/metadata.go | 87 +++++++++++++++++++ 21 files changed, 253 insertions(+) create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/from-server.golden create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/actions.graphql create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/allow_list.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/functions.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/query_collections.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/remote_schemas.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/tables.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t1/metadata/version.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/from-server.golden create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.graphql create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/allow_list.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/cron_triggers.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/functions.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/query_collections.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/remote_schemas.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/tables.yaml create mode 100644 cli/integration_test/testdata/incomplete_metadata/t2/metadata/version.yaml diff --git a/cli/go.sum b/cli/go.sum index 95096812b13b2..bb92c2512161d 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -47,8 +47,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= diff --git a/cli/integration_test/cli_test.go b/cli/integration_test/cli_test.go index fe9c6c02e437a..8e3b23562e730 100644 --- a/cli/integration_test/cli_test.go +++ b/cli/integration_test/cli_test.go @@ -143,6 +143,56 @@ func TestCommands(t *testing.T) { v2.TestSeedsApplyCmd(t, ec) }) }) + t.Run("config=v2/incomplete_migration_dir", func(t *testing.T) { + ec := cli.NewExecutionContext() + ec.Config = &cli.Config{} + logger, _ := test.NewNullLogger() + ec.Logger = logger + ec.Spinner = spinner.New(spinner.CharSets[7], 100*time.Millisecond) + ec.Spinner.Writer = ioutil.Discard + ec.Viper = viper.New() + + initDir := filepath.Join(os.TempDir(), "hasura-cli-test-"+strconv.Itoa(rand.Intn(1000))) + defer os.RemoveAll(initDir) + + // This will prepare the execution context, so no need to run ec.Prepare() on all the other tests + t.Run("prepare", func(t *testing.T) { + integrationtest.TestPrepare(t, ec) + }) + + skip(t) + // This will init the project dir + t.Run("init command", func(t *testing.T) { + v2.TestInitCmd(t, ec, initDir) + }) + + skip(t) + // This will validate the project dir + t.Run("validate", func(t *testing.T) { + integrationtest.TestValidate(t, ec) + }) + + skip(t) + if cliExtManifestFilePath := os.Getenv("HASURA_GRAPHQL_TEST_CLI_EXT_MANIFEST_FILE_PATH"); cliExtManifestFilePath != "" { + t.Run("cli-ext-plugin-install", func(t *testing.T) { + installOpts := &commands.PluginInstallOptions{ + EC: ec, + Name: cli.CLIExtPluginName, + ManifestFile: cliExtManifestFilePath, + } + err := installOpts.Run() + if err != nil { + t.Fatalf("unable to install %s plugin, got %v", cli.CLIExtPluginName, err) + } + }) + } + + skip(t) + t.Run("metadata apply", func(t *testing.T) { + v2.TestIncompleteMetadataDir(t, ec) + }) + + }) } func skip(t *testing.T) { diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/from-server.golden b/cli/integration_test/testdata/incomplete_metadata/t1/from-server.golden new file mode 100644 index 0000000000000..631b9391b8032 --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/from-server.golden @@ -0,0 +1,4 @@ +{ + "version": 2, + "tables": [] +} \ No newline at end of file diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/actions.graphql b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/actions.graphql new file mode 100644 index 0000000000000..662291d327329 --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/actions.graphql @@ -0,0 +1,18 @@ +type Mutation { + actionName ( + arg1: SampleInput! + ): SampleOutput +} + + + + +input SampleInput { + username : String! + password : String! +} + +type SampleOutput { + accessToken : String! +} + diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/allow_list.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/allow_list.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/allow_list.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/functions.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/functions.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/functions.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/query_collections.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/query_collections.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/query_collections.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/remote_schemas.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/remote_schemas.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/remote_schemas.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/tables.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/tables.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/tables.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t1/metadata/version.yaml b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/version.yaml new file mode 100644 index 0000000000000..22817d2a9c7fc --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t1/metadata/version.yaml @@ -0,0 +1 @@ +version: 2 diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/from-server.golden b/cli/integration_test/testdata/incomplete_metadata/t2/from-server.golden new file mode 100644 index 0000000000000..d831ad0c8716d --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/from-server.golden @@ -0,0 +1,49 @@ +{ + "version": 2, + "tables": [], + "actions": [ + { + "name": "actionName", + "definition": { + "handler": "http://localhost:3000", + "output_type": "SampleOutput", + "arguments": [ + { + "name": "arg1", + "type": "SampleInput!" + } + ], + "type": "mutation", + "kind": "synchronous" + } + } + ], + "custom_types": { + "input_objects": [ + { + "name": "SampleInput", + "fields": [ + { + "name": "username", + "type": "String!" + }, + { + "name": "password", + "type": "String!" + } + ] + } + ], + "objects": [ + { + "name": "SampleOutput", + "fields": [ + { + "name": "accessToken", + "type": "String!" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.graphql b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.graphql new file mode 100644 index 0000000000000..662291d327329 --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.graphql @@ -0,0 +1,18 @@ +type Mutation { + actionName ( + arg1: SampleInput! + ): SampleOutput +} + + + + +input SampleInput { + username : String! + password : String! +} + +type SampleOutput { + accessToken : String! +} + diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.yaml new file mode 100644 index 0000000000000..e874c678929c9 --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/actions.yaml @@ -0,0 +1,12 @@ +actions: +- name: actionName + definition: + kind: synchronous + handler: http://localhost:3000 +custom_types: + enums: [] + input_objects: + - name: SampleInput + objects: + - name: SampleOutput + scalars: [] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/allow_list.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/allow_list.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/allow_list.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/cron_triggers.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/cron_triggers.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/cron_triggers.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/functions.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/functions.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/functions.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/query_collections.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/query_collections.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/query_collections.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/remote_schemas.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/remote_schemas.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/remote_schemas.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/tables.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/tables.yaml new file mode 100644 index 0000000000000..fe51488c7066f --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/tables.yaml @@ -0,0 +1 @@ +[] diff --git a/cli/integration_test/testdata/incomplete_metadata/t2/metadata/version.yaml b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/version.yaml new file mode 100644 index 0000000000000..22817d2a9c7fc --- /dev/null +++ b/cli/integration_test/testdata/incomplete_metadata/t2/metadata/version.yaml @@ -0,0 +1 @@ +version: 2 diff --git a/cli/integration_test/v2/metadata.go b/cli/integration_test/v2/metadata.go index 5b0f16d717796..d66a8ffe30ce1 100644 --- a/cli/integration_test/v2/metadata.go +++ b/cli/integration_test/v2/metadata.go @@ -2,7 +2,10 @@ package v2 import ( "bytes" + "encoding/json" + "fmt" "io/ioutil" + "net/http" "os" "path/filepath" "testing" @@ -138,3 +141,87 @@ func TestMetadataCmd(t *testing.T, ec *cli.ExecutionContext) { }) } } + +func TestIncompleteMetadataDir(t *testing.T, ec *cli.ExecutionContext) { + currDir, _ := os.Getwd() + tt := []struct { + name string + opts metadataInterface + err error + copyMetadataFolder string + goldenFile string + }{ + { + "t1", + &commands.MetadataExportOptions{ + EC: ec, + ActionType: "apply", + }, + nil, + filepath.Join(currDir, "testdata/incomplete_metadata/t1/metadata"), + filepath.Join(currDir, "testdata/incomplete_metadata/t1/from-server.golden"), + }, + { + "t2", + &commands.MetadataExportOptions{ + EC: ec, + ActionType: "apply", + }, + nil, + filepath.Join(currDir, "testdata/incomplete_metadata/t2/metadata"), + filepath.Join(currDir, "testdata/incomplete_metadata/t2/from-server.golden"), + }, + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.copyMetadataFolder != "" { + err := os.RemoveAll(ec.MetadataDir) + if err != nil { + t.Fatalf("%s: unable to remove metadata directory, got %v", tc.name, err) + } + err = util.CopyDir(tc.copyMetadataFolder, ec.MetadataDir) + if err != nil { + t.Fatalf("%s: unable to copy metadata file, got %v", tc.name, err) + } + } + err := tc.opts.Run() + if err != tc.err { + t.Fatalf("%s: expected %v, got %v", tc.name, tc.err, err) + } + // get metadata from server + testServerUrl := os.Getenv("HASURA_GRAPHQL_TEST_ENDPOINT") + if testServerUrl == "" { + testServerUrl = "http://localhost:8080" + } + url := fmt.Sprintf("%s/%s", testServerUrl, "v1/query") + var body = []byte(` +{ + "type" : "export_metadata", + "args": {} +} +`) + resp, err := http.Post(url, "application/json", bytes.NewReader(body)) + defer resp.Body.Close() + assert.Equal(t, resp.StatusCode, http.StatusOK) + got, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err) + + want, err := ioutil.ReadFile(tc.goldenFile) + assert.NoError(t, err) + var wantM, gotM map[string]interface{} + + err = json.Unmarshal(want, &wantM) + assert.NoError(t, err) + err = json.Unmarshal(got, &gotM) + assert.NoError(t, err) + + wantB, err := json.Marshal(wantM) + assert.NoError(t, err) + gotB, err := json.Marshal(gotM) + assert.NoError(t, err) + + assert.Equal(t, string(wantB), string(gotB)) + }) + } +} From ec4a4991ccd5f29f0d081bd00956e1f528fa6703 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Mon, 22 Jun 2020 17:47:20 +0530 Subject: [PATCH 03/14] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e1e08b1d9b8..f8880f9809406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - console: allow manual edit of column types and handle array data types (close #2544, #3335, #2583) (#4546) - console: add the ability to delete a role in permissions summary page (close #3353) (#4987) +- cli: fix bug with metadata apply on upgrading from a previous version (close #5163) ## `v1.3.0-beta.2` From 65690048932cafe3220b209d9ee87fe9f397a633 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Mon, 22 Jun 2020 19:10:53 +0530 Subject: [PATCH 04/14] handle admin secret --- cli/integration_test/v2/metadata.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/integration_test/v2/metadata.go b/cli/integration_test/v2/metadata.go index d66a8ffe30ce1..8859317fb6c75 100644 --- a/cli/integration_test/v2/metadata.go +++ b/cli/integration_test/v2/metadata.go @@ -201,7 +201,14 @@ func TestIncompleteMetadataDir(t *testing.T, ec *cli.ExecutionContext) { "args": {} } `) - resp, err := http.Post(url, "application/json", bytes.NewReader(body)) + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + assert.NoError(t, err) + if ec.Config.AdminSecret != "" { + req.Header.Set(cli.XHasuraAdminSecret, ec.Config.AdminSecret) + } + c := http.Client{} + resp, err := c.Do(req) + defer resp.Body.Close() assert.Equal(t, resp.StatusCode, http.StatusOK) got, err := ioutil.ReadAll(resp.Body) From 490aaa6ec3b147102d7dcb4a1aaaf0771e03ca73 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 11:20:43 +0530 Subject: [PATCH 05/14] add test to check upgrade from prev stable version --- .../test/test-upgrade-from-latest-release.sh | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh new file mode 100644 index 0000000000000..dfba847aea0e0 --- /dev/null +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -0,0 +1,49 @@ + +#!/usr/bin/env bash + +set -evo pipefail +IFS=$'\n\t' +ROOT="$(readlink -f ${BASH_SOURCE[0]%/*}/../../)" + +wait_for_server() { + echo "waiting for server" + for _ in $(seq 1 60); + do + docker run --network container:graphql-engine appropriate/curl http://127.0.0.1:8080/v1/version && return + echo -n . + sleep 1 + done + echo "Failed waiting for server" && exit 1 +} + +# get previous stable version docker-compose file +curl -L https://raw.githubusercontent.com/hasura/graphql-engine/stable/install-manifests/docker-compose/docker-compose.yaml -o docker-compose-latest.yaml +sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2/' docker-compose-latest.yaml +# start postgres +docker-compose -f docker-compose-latest.yaml up --no-start graphql-engine +# copy migrations directory to /hasura-migrations +docker cp migrations/. graphql-engine:/hasura-migrations +# copy metadata directory to /hasura-metadata +docker cp metadata/. graphql-engine:/hasura-metadata +# start graphql-engine +docker-compose up -d --no-recreate graphql-engine +wait_for_server +# export metadata and run diff with validation/metadata.json +docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json - +# get list of migrations applied from graphql-engine server +docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json - +# delete postgres and graphql-engine +docker-compose -f docker-compose-latest.yaml down + +# start with current build +# start postgres +docker-compose up --no-start graphql-engine +# start graphql-engine +docker-compose up -d --no-recreate graphql-engine +wait_for_server +# export metadata and run diff with validation/metadata.json +docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json - +# get list of migrations applied from graphql-engine server +docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json - +# delete postgres and graphql-engine +docker-compose down -v From ce3c60e1c16e081aeb72ea1aaf1671485ac4e065 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 11:22:08 +0530 Subject: [PATCH 06/14] fix script --- .../cli-migrations/v2/test/test-upgrade-from-latest-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh index dfba847aea0e0..6e52aef68d806 100644 --- a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -26,7 +26,7 @@ docker cp migrations/. graphql-engine:/hasura-migrations # copy metadata directory to /hasura-metadata docker cp metadata/. graphql-engine:/hasura-metadata # start graphql-engine -docker-compose up -d --no-recreate graphql-engine +docker-compose -f docker-compose-latest.yaml up -d --no-recreate graphql-engine wait_for_server # export metadata and run diff with validation/metadata.json docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json - From e3a654c723fb1e36bd2484614386c467fdf8a6a2 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 11:50:15 +0530 Subject: [PATCH 07/14] update test script --- scripts/cli-migrations/Makefile | 1 + .../cli-migrations/v2/test/test-upgrade-from-latest-release.sh | 0 2 files changed, 1 insertion(+) mode change 100644 => 100755 scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh diff --git a/scripts/cli-migrations/Makefile b/scripts/cli-migrations/Makefile index 6aac2cded6018..c71ff68e06768 100644 --- a/scripts/cli-migrations/Makefile +++ b/scripts/cli-migrations/Makefile @@ -40,6 +40,7 @@ build-cli-migrations-v2: test-cli-migrations-v2: cd v2/test ./test.sh + ./test-upgrade-from-latest-release.sh .PHONY: all all: load-server-image build-cli-migrations-v1 build-cli-migrations-v2 test-cli-migrations-v1 test-cli-migrations-v2 \ No newline at end of file diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh old mode 100644 new mode 100755 From 4b42cc31fdddf03c4a0c696ce6af11e51577407e Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 12:14:11 +0530 Subject: [PATCH 08/14] fix shellbang --- .../cli-migrations/v2/test/test-upgrade-from-latest-release.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh index 6e52aef68d806..d9669b60f54c6 100755 --- a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash set -evo pipefail From d91cff97ececd6804d09e221b09e9189c535d9b7 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 13:06:49 +0530 Subject: [PATCH 09/14] add container name to latest docker-compose file --- .../cli-migrations/v2/test/test-upgrade-from-latest-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh index d9669b60f54c6..f8e2bc4f46f41 100755 --- a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -17,7 +17,7 @@ wait_for_server() { # get previous stable version docker-compose file curl -L https://raw.githubusercontent.com/hasura/graphql-engine/stable/install-manifests/docker-compose/docker-compose.yaml -o docker-compose-latest.yaml -sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2/' docker-compose-latest.yaml +sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2\n container_name: graphql-engine/' docker-compose-latest.yaml # start postgres docker-compose -f docker-compose-latest.yaml up --no-start graphql-engine # copy migrations directory to /hasura-migrations From d639e48fedb6354adb6d4f08e0f72bdc57cb6d70 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 13:33:12 +0530 Subject: [PATCH 10/14] try to emulate a natural update --- .../v2/test/test-upgrade-from-latest-release.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh index f8e2bc4f46f41..216e5d5475fed 100755 --- a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -15,15 +15,12 @@ wait_for_server() { echo "Failed waiting for server" && exit 1 } + # get previous stable version docker-compose file curl -L https://raw.githubusercontent.com/hasura/graphql-engine/stable/install-manifests/docker-compose/docker-compose.yaml -o docker-compose-latest.yaml sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2\n container_name: graphql-engine/' docker-compose-latest.yaml # start postgres docker-compose -f docker-compose-latest.yaml up --no-start graphql-engine -# copy migrations directory to /hasura-migrations -docker cp migrations/. graphql-engine:/hasura-migrations -# copy metadata directory to /hasura-metadata -docker cp metadata/. graphql-engine:/hasura-metadata # start graphql-engine docker-compose -f docker-compose-latest.yaml up -d --no-recreate graphql-engine wait_for_server @@ -31,14 +28,9 @@ wait_for_server docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json - # get list of migrations applied from graphql-engine server docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "run_sql", "args" : {"sql": "select * from hdb_catalog.schema_migrations"} }' localhost:8080/v1/query | jq -j '.' | diff validation/schema_migrations.json - -# delete postgres and graphql-engine -docker-compose -f docker-compose-latest.yaml down -# start with current build -# start postgres -docker-compose up --no-start graphql-engine -# start graphql-engine -docker-compose up -d --no-recreate graphql-engine +# use the current build to start container +docker-compose up -d wait_for_server # export metadata and run diff with validation/metadata.json docker run --network container:graphql-engine appropriate/curl -s -f -d'{"type" : "export_metadata", "args" : {} }' localhost:8080/v1/query | jq -j '.' | diff validation/metadata.json - From 161a690c9fe180fbe5f0ad5d22305ae73a688b83 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Tue, 23 Jun 2020 14:18:50 +0530 Subject: [PATCH 11/14] attempt to fix test --- .../v2/test/test-upgrade-from-latest-release.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh index 216e5d5475fed..57a7088f0baf2 100755 --- a/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh +++ b/scripts/cli-migrations/v2/test/test-upgrade-from-latest-release.sh @@ -21,6 +21,10 @@ curl -L https://raw.githubusercontent.com/hasura/graphql-engine/stable/install-m sed -i '/hasura\/graphql-engine:/ s/$/.cli-migrations-v2\n container_name: graphql-engine/' docker-compose-latest.yaml # start postgres docker-compose -f docker-compose-latest.yaml up --no-start graphql-engine +# copy migrations directory to /hasura-migrations +docker cp migrations/. graphql-engine:/hasura-migrations +# copy metadata directory to /hasura-metadata +docker cp metadata/. graphql-engine:/hasura-metadata # start graphql-engine docker-compose -f docker-compose-latest.yaml up -d --no-recreate graphql-engine wait_for_server From dcd8835b4e483b8dfcbe6b34267270d95cbbd658 Mon Sep 17 00:00:00 2001 From: Aravind Date: Tue, 23 Jun 2020 14:41:14 +0530 Subject: [PATCH 12/14] Update CHANGELOG.md Co-authored-by: Tirumarai Selvan --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8880f9809406..d6967a7547cd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - console: allow manual edit of column types and handle array data types (close #2544, #3335, #2583) (#4546) - console: add the ability to delete a role in permissions summary page (close #3353) (#4987) -- cli: fix bug with metadata apply on upgrading from a previous version (close #5163) +- cli: handle missing files during metadata apply (close #5163) (#5170) ## `v1.3.0-beta.2` From 864af26195e6803933f4a4aa0e34536a87fd21c5 Mon Sep 17 00:00:00 2001 From: Aravind Date: Tue, 23 Jun 2020 20:02:48 +0530 Subject: [PATCH 13/14] Update cli/migrate/database/hasuradb/metadata.go Co-authored-by: jflambert --- cli/migrate/database/hasuradb/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/migrate/database/hasuradb/metadata.go b/cli/migrate/database/hasuradb/metadata.go index 688ec093d67db..21c6ce20d79e1 100644 --- a/cli/migrate/database/hasuradb/metadata.go +++ b/cli/migrate/database/hasuradb/metadata.go @@ -153,7 +153,7 @@ func (h *HasuraDB) BuildMetadata() (yaml.MapSlice, error) { err := plg.Build(&tmpMeta) if err != nil { if os.IsNotExist(errors.Cause(err)) { - h.logger.Debugf("metadata file for %s was not found, assuming an empty files", plg.Name()) + h.logger.Debugf("metadata file for %s was not found, assuming an empty file", plg.Name()) continue } return tmpMeta, errors.Wrap(err, fmt.Sprintf("cannot build %s from metadata", plg.Name())) From 8228e937bd8ddd980156a5484511248c13c39507 Mon Sep 17 00:00:00 2001 From: scriptonist Date: Wed, 24 Jun 2020 08:47:48 +0530 Subject: [PATCH 14/14] Sync remote, fix wording --- cli/integration_test/cli_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/integration_test/cli_test.go b/cli/integration_test/cli_test.go index 8e3b23562e730..5fe3a77b601c9 100644 --- a/cli/integration_test/cli_test.go +++ b/cli/integration_test/cli_test.go @@ -143,7 +143,7 @@ func TestCommands(t *testing.T) { v2.TestSeedsApplyCmd(t, ec) }) }) - t.Run("config=v2/incomplete_migration_dir", func(t *testing.T) { + t.Run("config=v2/incomplete_metadata_dir", func(t *testing.T) { ec := cli.NewExecutionContext() ec.Config = &cli.Config{} logger, _ := test.NewNullLogger()