这是indexloc提供的服务，不要输入任何密码
Skip to content

deterministic ordering of objects in exported metadata (close #3125) #3230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Nov 6, 2019
Merged

deterministic ordering of objects in exported metadata (close #3125) #3230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b940c9d
deterministic ordering of objects in exported metadata, close #3125
rakeshkky Oct 24, 2019
76f6727
remove orderedDict check
rakeshkky Oct 25, 2019
1724e40
refactor 'Hasura.RQL.DDL.Metadata'
rakeshkky Oct 31, 2019
58db57e
Merge branch 'master' into issue-3125-deterministic-export
lexi-lambda Nov 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
166 changes: 96 additions & 70 deletions server/src-lib/Hasura/RQL/DDL/Metadata.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,49 +30,50 @@ import Data.Aeson.Casing
import Data.Aeson.TH
import Language.Haskell.TH.Syntax (Lift)

import qualified Data.HashMap.Strict as M
import qualified Data.HashMap.Strict as HM
import qualified Data.HashMap.Strict.InsOrd as HMIns
import qualified Data.HashSet as HS
import qualified Data.List as L
import qualified Data.Text as T

import Hasura.EncJSON
import Hasura.Prelude
import Hasura.RQL.DDL.ComputedField (dropComputedFieldFromCatalog)
import Hasura.RQL.DDL.EventTrigger (delEventTriggerFromCatalog,
subTableP2)
import Hasura.RQL.DDL.Permission.Internal (dropPermFromCatalog)
import Hasura.RQL.DDL.RemoteSchema (addRemoteSchemaP2,
buildGCtxMap,
removeRemoteSchemaFromCatalog)
import Hasura.RQL.Types
import Hasura.SQL.Types

import qualified Database.PG.Query as Q
import qualified Hasura.RQL.DDL.ComputedField as DCC
import qualified Hasura.RQL.DDL.EventTrigger as DE
import qualified Hasura.RQL.DDL.Permission as DP
import qualified Hasura.RQL.DDL.Permission.Internal as DP
import qualified Hasura.RQL.DDL.QueryCollection as DQC
import qualified Hasura.RQL.DDL.Relationship as DR
import qualified Hasura.RQL.DDL.RemoteSchema as DRS
import qualified Hasura.RQL.DDL.Schema as DS
import qualified Hasura.RQL.Types.EventTrigger as DTS
import qualified Hasura.RQL.Types.RemoteSchema as TRS
import qualified Hasura.RQL.DDL.Permission as Permission
import qualified Hasura.RQL.DDL.QueryCollection as Collection
import qualified Hasura.RQL.DDL.Relationship as Relationship
import qualified Hasura.RQL.DDL.Schema as Schema


data TableMeta
= TableMeta
{ _tmTable :: !QualifiedTable
, _tmIsEnum :: !Bool
, _tmConfiguration :: !(TableConfig)
, _tmObjectRelationships :: ![DR.ObjRelDef]
, _tmArrayRelationships :: ![DR.ArrRelDef]
, _tmInsertPermissions :: ![DP.InsPermDef]
, _tmSelectPermissions :: ![DP.SelPermDef]
, _tmUpdatePermissions :: ![DP.UpdPermDef]
, _tmDeletePermissions :: ![DP.DelPermDef]
, _tmEventTriggers :: ![DTS.EventTriggerConf]
, _tmConfiguration :: !TableConfig
, _tmObjectRelationships :: ![Relationship.ObjRelDef]
, _tmArrayRelationships :: ![Relationship.ArrRelDef]
, _tmInsertPermissions :: ![Permission.InsPermDef]
, _tmSelectPermissions :: ![Permission.SelPermDef]
, _tmUpdatePermissions :: ![Permission.UpdPermDef]
, _tmDeletePermissions :: ![Permission.DelPermDef]
, _tmEventTriggers :: ![EventTriggerConf]
} deriving (Show, Eq, Lift)
$(makeLenses ''TableMeta)

mkTableMeta :: QualifiedTable -> Bool -> TableConfig -> TableMeta
mkTableMeta qt isEnum config =
TableMeta qt isEnum config [] [] [] [] [] [] []

makeLenses ''TableMeta

instance FromJSON TableMeta where
parseJSON (Object o) = do
unless (null unexpectedKeys) $
Expand Down Expand Up @@ -104,7 +105,7 @@ instance FromJSON TableMeta where
etKey = "event_triggers"

unexpectedKeys =
HS.fromList (M.keys o) `HS.difference` expectedKeySet
HS.fromList (HM.keys o) `HS.difference` expectedKeySet

expectedKeySet =
HS.fromList [ tableKey, isEnumKey, configKey, orKey
Expand Down Expand Up @@ -141,16 +142,16 @@ runClearMetadata
runClearMetadata _ = do
adminOnly
liftTx clearMetadata
DS.buildSchemaCacheStrict
Schema.buildSchemaCacheStrict
return successMsg

data ReplaceMetadata
= ReplaceMetadata
{ aqTables :: ![TableMeta]
, aqFunctions :: !(Maybe [QualifiedFunction])
, aqRemoteSchemas :: !(Maybe [TRS.AddRemoteSchemaQuery])
, aqQueryCollections :: !(Maybe [DQC.CreateCollection])
, aqAllowlist :: !(Maybe [DQC.CollectionReq])
, aqRemoteSchemas :: !(Maybe [AddRemoteSchemaQuery])
, aqQueryCollections :: !(Maybe [Collection.CreateCollection])
, aqAllowlist :: !(Maybe [Collection.CollectionReq])
} deriving (Show, Eq, Lift)

$(deriveJSON (aesonDrop 2 snakeCase){omitNothingFields=True} ''ReplaceMetadata)
Expand All @@ -168,14 +169,14 @@ applyQP1 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) =

-- process each table
void $ indexedForM tables $ \table -> withTableName (table ^. tmTable) $ do
let allRels = map DR.rdName (table ^. tmObjectRelationships) <>
map DR.rdName (table ^. tmArrayRelationships)
let allRels = map Relationship.rdName (table ^. tmObjectRelationships) <>
map Relationship.rdName (table ^. tmArrayRelationships)

insPerms = map DP.pdRole $ table ^. tmInsertPermissions
selPerms = map DP.pdRole $ table ^. tmSelectPermissions
updPerms = map DP.pdRole $ table ^. tmUpdatePermissions
delPerms = map DP.pdRole $ table ^. tmDeletePermissions
eventTriggers = map DTS.etcName $ table ^. tmEventTriggers
insPerms = map Permission.pdRole $ table ^. tmInsertPermissions
selPerms = map Permission.pdRole $ table ^. tmSelectPermissions
updPerms = map Permission.pdRole $ table ^. tmUpdatePermissions
delPerms = map Permission.pdRole $ table ^. tmDeletePermissions
eventTriggers = map etcName $ table ^. tmEventTriggers

checkMultipleDecls "relationships" allRels
checkMultipleDecls "insert permissions" insPerms
Expand All @@ -189,15 +190,15 @@ applyQP1 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) =

onJust mSchemas $ \schemas ->
withPathK "remote_schemas" $
checkMultipleDecls "remote schemas" $ map TRS._arsqName schemas
checkMultipleDecls "remote schemas" $ map _arsqName schemas

onJust mCollections $ \collections ->
withPathK "query_collections" $
checkMultipleDecls "query collections" $ map DQC._ccName collections
checkMultipleDecls "query collections" $ map Collection._ccName collections

onJust mAllowlist $ \allowlist ->
withPathK "allowlist" $
checkMultipleDecls "allow list" $ map DQC._crCollection allowlist
checkMultipleDecls "allow list" $ map Collection._crCollection allowlist

where
withTableName qt = withPathK (qualObjectToText qt)
Expand Down Expand Up @@ -226,7 +227,7 @@ applyQP2
applyQP2 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) = do

liftTx clearMetadata
DS.buildSchemaCacheStrict
Schema.buildSchemaCacheStrict

systemDefined <- askSystemDefined
withPathK "tables" $ do
Expand All @@ -235,16 +236,16 @@ applyQP2 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) =
let tableName = tableMeta ^. tmTable
isEnum = tableMeta ^. tmIsEnum
config = tableMeta ^. tmConfiguration
void $ DS.trackExistingTableOrViewP2 tableName systemDefined isEnum config
void $ Schema.trackExistingTableOrViewP2 tableName systemDefined isEnum config

-- Relationships
indexedForM_ tables $ \table -> do
withPathK "object_relationships" $
indexedForM_ (table ^. tmObjectRelationships) $ \objRel ->
DR.objRelP2 (table ^. tmTable) objRel
Relationship.objRelP2 (table ^. tmTable) objRel
withPathK "array_relationships" $
indexedForM_ (table ^. tmArrayRelationships) $ \arrRel ->
DR.arrRelP2 (table ^. tmTable) arrRel
Relationship.arrRelP2 (table ^. tmTable) arrRel

-- Permissions
indexedForM_ tables $ \table -> do
Expand All @@ -262,31 +263,31 @@ applyQP2 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) =
indexedForM_ tables $ \table ->
withPathK "event_triggers" $
indexedForM_ (table ^. tmEventTriggers) $ \etc ->
DE.subTableP2 (table ^. tmTable) False etc
subTableP2 (table ^. tmTable) False etc

-- sql functions
withPathK "functions" $
indexedMapM_ (void . DS.trackFunctionP2) functions
indexedMapM_ (void . Schema.trackFunctionP2) functions

-- query collections
withPathK "query_collections" $
indexedForM_ collections $ \c -> do
liftTx $ DQC.addCollectionToCatalog c systemDefined
indexedForM_ collections $ \c ->
liftTx $ Collection.addCollectionToCatalog c systemDefined

-- allow list
withPathK "allowlist" $ do
indexedForM_ allowlist $ \(DQC.CollectionReq name) -> do
liftTx $ DQC.addCollectionToAllowlistCatalog name
indexedForM_ allowlist $ \(Collection.CollectionReq name) ->
liftTx $ Collection.addCollectionToAllowlistCatalog name
-- add to cache
DQC.refreshAllowlist
Collection.refreshAllowlist

-- remote schemas
onJust mSchemas $ \schemas ->
withPathK "remote_schemas" $
indexedMapM_ (void . DRS.addRemoteSchemaP2) schemas
indexedMapM_ (void . addRemoteSchemaP2) schemas

-- build GraphQL Context with Remote schemas
DRS.buildGCtxMap
buildGCtxMap

return successMsg

Expand All @@ -296,8 +297,8 @@ applyQP2 (ReplaceMetadata tables mFunctions mSchemas mCollections mAllowlist) =
allowlist = fromMaybe [] mAllowlist
processPerms tabInfo perms =
indexedForM_ perms $ \permDef -> do
permInfo <- DP.addPermP1 tabInfo permDef
DP.addPermP2 (_tiName tabInfo) permDef permInfo
permInfo <- Permission.addPermP1 tabInfo permDef
Permission.addPermP2 (_tiName tabInfo) permDef permInfo

runReplaceMetadata
:: ( QErrM m, UserInfoM m, CacheRWM m, MonadTx m
Expand All @@ -321,7 +322,7 @@ $(deriveToJSON defaultOptions ''ExportMetadata)
fetchMetadata :: Q.TxE QErr ReplaceMetadata
fetchMetadata = do
tables <- Q.catchE defaultTxErrorHandler fetchTables
let tableMetaMap = M.fromList . flip map tables $
let tableMetaMap = HMIns.fromList . flip map tables $
\(schema, name, isEnum, maybeConfig) ->
let qualifiedName = QualifiedObject schema name
configuration = maybe emptyTableConfig Q.getAltJ maybeConfig
Expand Down Expand Up @@ -360,16 +361,16 @@ fetchMetadata = do
Q.catchE defaultTxErrorHandler fetchFunctions

-- fetch all custom resolvers
schemas <- DRS.fetchRemoteSchemas
remoteSchemas <- fetchRemoteSchemas

-- fetch all collections
collections <- fetchCollections

-- fetch allow list
allowlist <- map DQC.CollectionReq <$> DQC.fetchAllowlist
allowlist <- map Collection.CollectionReq <$> fetchAllowlists

return $ ReplaceMetadata (M.elems postRelMap) (Just functions)
(Just schemas) (Just collections) (Just allowlist)
return $ ReplaceMetadata (HMIns.elems postRelMap) (Just functions)
(Just remoteSchemas) (Just collections) (Just allowlist)

where

Expand All @@ -381,13 +382,13 @@ fetchMetadata = do

permRowToDef (sn, tn, rn, _, Q.AltJ pDef, mComment) = do
perm <- decodeValue pDef
return (QualifiedObject sn tn, DP.PermDef rn perm mComment)
return (QualifiedObject sn tn, Permission.PermDef rn perm mComment)

mkRelDefs rt = mapM relRowToDef . filter (\rr -> rr ^. _4 == rt)

relRowToDef (sn, tn, rn, _, Q.AltJ rDef, mComment) = do
using <- decodeValue rDef
return (QualifiedObject sn tn, DR.RelDef rn using mComment)
return (QualifiedObject sn tn, Relationship.RelDef rn using mComment)

mkTriggerMetaDefs = mapM trigRowToDef

Expand All @@ -400,43 +401,68 @@ fetchMetadata = do
SELECT table_schema, table_name, is_enum, configuration::json
FROM hdb_catalog.hdb_table
WHERE is_system_defined = 'false'
ORDER BY table_schema ASC, table_name ASC
|] () False

fetchRelationships =
Q.listQ [Q.sql|
SELECT table_schema, table_name, rel_name, rel_type, rel_def::json, comment
FROM hdb_catalog.hdb_relationship
WHERE is_system_defined = 'false'
ORDER BY table_schema ASC, table_name ASC, rel_name ASC
|] () False

fetchPermissions =
Q.listQ [Q.sql|
SELECT table_schema, table_name, role_name, perm_type, perm_def::json, comment
FROM hdb_catalog.hdb_permission
WHERE is_system_defined = 'false'
ORDER BY table_schema ASC, table_name ASC, role_name ASC, perm_type ASC
|] () False

fetchEventTriggers =
Q.listQ [Q.sql|
SELECT e.schema_name, e.table_name, e.configuration::json
FROM hdb_catalog.event_triggers e
ORDER BY e.schema_name ASC, e.table_name ASC, e.name ASC
|] () False

fetchFunctions =
Q.listQ [Q.sql|
SELECT function_schema, function_name
FROM hdb_catalog.hdb_function
WHERE is_system_defined = 'false'
ORDER BY function_schema ASC, function_name ASC
|] () False

fetchCollections = do
r <- Q.listQE defaultTxErrorHandler [Q.sql|
fetchCollections =
map fromRow <$> Q.listQE defaultTxErrorHandler [Q.sql|
SELECT collection_name, collection_defn::json, comment
FROM hdb_catalog.hdb_query_collection
WHERE is_system_defined = 'false'
ORDER BY collection_name ASC
|] () False
return $ flip map r $ \(name, Q.AltJ defn, mComment)
-> DQC.CreateCollection name defn mComment
where
fromRow (name, Q.AltJ defn, mComment) =
Collection.CreateCollection name defn mComment

fetchAllowlists = map runIdentity <$>
Q.listQE defaultTxErrorHandler [Q.sql|
SELECT collection_name
FROM hdb_catalog.hdb_allowlist
ORDER BY collection_name ASC
|] () False

fetchRemoteSchemas =
map fromRow <$> Q.listQE defaultTxErrorHandler
[Q.sql|
SELECT name, definition, comment
FROM hdb_catalog.remote_schemas
ORDER BY name ASC
|] () True
where
fromRow (name, Q.AltJ def, comment) =
AddRemoteSchemaQuery name def comment

runExportMetadata
:: (QErrM m, UserInfoM m, MonadTx m)
Expand All @@ -459,7 +485,7 @@ runReloadMetadata
=> ReloadMetadata -> m EncJSON
runReloadMetadata _ = do
adminOnly
DS.buildSchemaCache
Schema.buildSchemaCache
return successMsg

data DumpInternalState
Expand Down Expand Up @@ -521,10 +547,10 @@ runDropInconsistentMetadata _ = do

purgeMetadataObj :: MonadTx m => MetadataObjId -> m ()
purgeMetadataObj = liftTx . \case
(MOTable qt) -> DS.deleteTableFromCatalog qt
(MOFunction qf) -> DS.delFunctionFromCatalog qf
(MORemoteSchema rsn) -> DRS.removeRemoteSchemaFromCatalog rsn
(MOTableObj qt (MTORel rn _)) -> DR.delRelFromCatalog qt rn
(MOTableObj qt (MTOPerm rn pt)) -> DP.dropPermFromCatalog qt rn pt
(MOTableObj _ (MTOTrigger trn)) -> DE.delEventTriggerFromCatalog trn
(MOTableObj qt (MTOComputedField ccn)) -> DCC.dropComputedFieldFromCatalog qt ccn
(MOTable qt) -> Schema.deleteTableFromCatalog qt
(MOFunction qf) -> Schema.delFunctionFromCatalog qf
(MORemoteSchema rsn) -> removeRemoteSchemaFromCatalog rsn
(MOTableObj qt (MTORel rn _)) -> Relationship.delRelFromCatalog qt rn
(MOTableObj qt (MTOPerm rn pt)) -> dropPermFromCatalog qt rn pt
(MOTableObj _ (MTOTrigger trn)) -> delEventTriggerFromCatalog trn
(MOTableObj qt (MTOComputedField ccn)) -> dropComputedFieldFromCatalog qt ccn
12 changes: 0 additions & 12 deletions server/src-lib/Hasura/RQL/DDL/RemoteSchema.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ module Hasura.RQL.DDL.RemoteSchema
, removeRemoteSchemaFromCatalog
, runReloadRemoteSchema
, buildGCtxMap
, fetchRemoteSchemas
, addRemoteSchemaP1
, addRemoteSchemaP2Setup
, addRemoteSchemaP2
Expand Down Expand Up @@ -145,14 +144,3 @@ removeRemoteSchemaFromCatalog name =
DELETE FROM hdb_catalog.remote_schemas
WHERE name = $1
|] (Identity name) True


fetchRemoteSchemas :: Q.TxE QErr [AddRemoteSchemaQuery]
fetchRemoteSchemas =
map fromRow <$> Q.listQE defaultTxErrorHandler
[Q.sql|
SELECT name, definition, comment
FROM hdb_catalog.remote_schemas
|] () True
where
fromRow (n, Q.AltJ def, comm) = AddRemoteSchemaQuery n def comm
Loading