From 2e4fdf3a746edb2cc4d7259facc5029083aa07f0 Mon Sep 17 00:00:00 2001 From: rakeshkky Date: Tue, 30 Apr 2019 13:12:21 +0530 Subject: [PATCH 1/5] build schema cache function without db setup --- server/src-lib/Hasura/RQL/DDL/Schema/Table.hs | 29 +++++++++++++------ server/src-lib/Hasura/Server/SchemaUpdate.hs | 4 +-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs index bdb6278f6caa5..8fa5ed08174fa 100644 --- a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs +++ b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs @@ -327,9 +327,19 @@ buildSchemaCacheStrict = do buildSchemaCache :: (MonadTx m, CacheRWM m, MonadIO m, HasHttpManager m, HasSQLGenCtx m) => m () -buildSchemaCache = do +buildSchemaCache = buildSchemaCacheG True + +buildSCWithoutSetup + :: (MonadTx m, CacheRWM m, MonadIO m, HasHttpManager m, HasSQLGenCtx m) + => m () +buildSCWithoutSetup = buildSchemaCacheG False + +buildSchemaCacheG + :: (MonadTx m, CacheRWM m, MonadIO m, HasHttpManager m, HasSQLGenCtx m) + => Bool -> m () +buildSchemaCacheG withSetup = do -- clean hdb_views - liftTx $ Q.catchE defaultTxErrorHandler clearHdbViews + when withSetup $ liftTx $ Q.catchE defaultTxErrorHandler clearHdbViews -- reset the current schemacache writeSchemaCache emptySchemaCache hMgr <- askHttpManager @@ -376,10 +386,10 @@ buildSchemaCache = do modifyErr (\e -> "table " <> tn <<> "; role " <> rn <<> "; " <> e) $ handleInconsistentObj mkInconsObj $ case pt of - PTInsert -> permHelper sqlGenCtx sn tn rn pDef PAInsert - PTSelect -> permHelper sqlGenCtx sn tn rn pDef PASelect - PTUpdate -> permHelper sqlGenCtx sn tn rn pDef PAUpdate - PTDelete -> permHelper sqlGenCtx sn tn rn pDef PADelete + PTInsert -> permHelper withSetup sqlGenCtx sn tn rn pDef PAInsert + PTSelect -> permHelper withSetup sqlGenCtx sn tn rn pDef PASelect + PTUpdate -> permHelper withSetup sqlGenCtx sn tn rn pDef PAUpdate + PTDelete -> permHelper withSetup sqlGenCtx sn tn rn pDef PADelete -- Fetch all the query templates qtemplates <- liftTx $ Q.catchE defaultTxErrorHandler fetchQTemplates @@ -404,7 +414,8 @@ buildSchemaCache = do etc <- decodeValue configuration subTableP2Setup qt etc allCols <- getCols . tiFieldInfoMap <$> askTabInfo qt - liftTx $ mkTriggerQ trn qt allCols (stringifyNum sqlGenCtx) (etcDefinition etc) + when withSetup $ liftTx $ + mkTriggerQ trn qt allCols (stringifyNum sqlGenCtx) (etcDefinition etc) functions <- liftTx $ Q.catchE defaultTxErrorHandler fetchFunctions forM_ functions $ \(sn, fn) -> do @@ -425,14 +436,14 @@ buildSchemaCache = do forM_ remoteSchemas $ resolveSingleRemoteSchema hMgr where - permHelper sqlGenCtx sn tn rn pDef pa = do + permHelper setup sqlGenCtx sn tn rn pDef pa = do qCtx <- mkAdminQCtx sqlGenCtx <$> askSchemaCache perm <- decodeValue pDef let qt = QualifiedObject sn tn permDef = PermDef rn perm Nothing createPerm = WithTable qt permDef (permInfo, deps) <- liftP1WithQCtx qCtx $ createPermP1 createPerm - addPermP2Setup qt permDef permInfo + when setup $ addPermP2Setup qt permDef permInfo addPermToCache qt rn pa permInfo deps -- p2F qt rn p1Res diff --git a/server/src-lib/Hasura/Server/SchemaUpdate.hs b/server/src-lib/Hasura/Server/SchemaUpdate.hs index 682437e55b86a..92df214dcb511 100644 --- a/server/src-lib/Hasura/Server/SchemaUpdate.hs +++ b/server/src-lib/Hasura/Server/SchemaUpdate.hs @@ -5,7 +5,7 @@ where import Hasura.Prelude import Hasura.Logging -import Hasura.RQL.DDL.Schema.Table (buildSchemaCache) +import Hasura.RQL.DDL.Schema.Table (buildSCWithoutSetup) import Hasura.RQL.Types import Hasura.Server.App (SchemaCacheRef (..), withSCUpdate) import Hasura.Server.Init (InstanceId (..)) @@ -194,7 +194,7 @@ refreshSchemaCache sqlGenCtx pool logger httpManager cacheRef threadType msg = d -- Reload schema cache from catalog resE <- liftIO $ runExceptT $ withSCUpdate cacheRef logger $ peelRun emptySchemaCache adminUserInfo - httpManager sqlGenCtx (PGExecCtx pool PG.Serializable) buildSchemaCache + httpManager sqlGenCtx (PGExecCtx pool PG.Serializable) buildSCWithoutSetup case resE of Left e -> logError logger threadType $ TEQueryError e Right _ -> From 3a730732427c24e9d5b16d426bc66dc6f7770f09 Mon Sep 17 00:00:00 2001 From: rakeshkky Date: Tue, 30 Apr 2019 13:59:53 +0530 Subject: [PATCH 2/5] avoid clearing views and routines in hdb_views schema in SQL loop --- server/src-lib/Hasura/RQL/DDL/Permission.hs | 1 - server/src-lib/Hasura/RQL/DDL/Utils.hs | 59 ++++++++++++--------- server/src-lib/Hasura/SQL/Types.hs | 3 ++ 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/server/src-lib/Hasura/RQL/DDL/Permission.hs b/server/src-lib/Hasura/RQL/DDL/Permission.hs index 29ed54ac2c3b0..07a1f30c8486f 100644 --- a/server/src-lib/Hasura/RQL/DDL/Permission.hs +++ b/server/src-lib/Hasura/RQL/DDL/Permission.hs @@ -86,7 +86,6 @@ buildViewName (QualifiedObject sn tn) (RoleName rTxt) pt = QualifiedObject hdbViewsSchema $ TableName (rTxt <> "__" <> T.pack (show pt) <> "__" <> snTxt <> "__" <> tnTxt) where - hdbViewsSchema = SchemaName "hdb_views" snTxt = getSchemaTxt sn tnTxt = getTableTxt tn diff --git a/server/src-lib/Hasura/RQL/DDL/Utils.hs b/server/src-lib/Hasura/RQL/DDL/Utils.hs index 13483cfe8cc89..2f83374a4fb44 100644 --- a/server/src-lib/Hasura/RQL/DDL/Utils.hs +++ b/server/src-lib/Hasura/RQL/DDL/Utils.hs @@ -2,35 +2,42 @@ module Hasura.RQL.DDL.Utils ( clearHdbViews ) where -import qualified Data.Text as T import qualified Database.PG.Query as Q -import Hasura.Prelude ((<>)) + +import Hasura.Prelude +import Hasura.SQL.Types clearHdbViews :: Q.Tx () -clearHdbViews = Q.multiQ (Q.fromText (clearHdbOnlyViews <> clearHdbViewsFunc)) +clearHdbViews = do + -- clear views + views <- fetchAllHdbViews + mapM_ dropView views + -- clear routines + routines <- fetchAllRoutines + mapM_ dropRoutine routines + where + fetchAllHdbViews = + map runIdentity <$> Q.listQ [Q.sql| + SELECT viewname + FROM pg_catalog.pg_views + WHERE schemaname = $1 + |] (Identity hdbViewsSchema) True -clearHdbOnlyViews :: T.Text -clearHdbOnlyViews = - "DO $$ DECLARE \ - \ r RECORD; \ - \ BEGIN \ - \ FOR r IN (SELECT viewname FROM pg_views WHERE schemaname = 'hdb_views') LOOP \ - \ EXECUTE 'DROP VIEW IF EXISTS hdb_views.' || quote_ident(r.viewname) || ' CASCADE'; \ - \ END LOOP; \ - \ END $$; " + fetchAllRoutines = + map runIdentity <$> Q.listQ [Q.sql| + SELECT routine_name + FROM information_schema.routines + WHERE specific_schema = $1 + |] (Identity hdbViewsSchema) True + dropView :: TableName -> Q.Tx () + dropView vn = do + let qvn = QualifiedObject hdbViewsSchema vn + q = "DROP VIEW IF EXISTS " <> toSQLTxt qvn <> " CASCADE" + Q.unitQ (Q.fromText q) () True -clearHdbViewsFunc :: T.Text -clearHdbViewsFunc = - "DO $$ DECLARE \ - \ _sql text; \ - \ BEGIN \ - \ SELECT INTO _sql \ - \ string_agg('DROP FUNCTION hdb_views.' || quote_ident(r.routine_name) || '() CASCADE;' \ - \ , E'\n') \ - \ FROM information_schema.routines r \ - \ WHERE r.specific_schema = 'hdb_views'; \ - \ IF _sql IS NOT NULL THEN \ - \ EXECUTE _sql; \ - \ END IF; \ - \ END $$; " + dropRoutine :: FunctionName -> Q.Tx () + dropRoutine fn = do + let qfn = QualifiedObject hdbViewsSchema fn + q = "DROP FUNCTION " <> toSQLTxt qfn <> "() CASCADE" + Q.unitQ (Q.fromText q) () True diff --git a/server/src-lib/Hasura/SQL/Types.hs b/server/src-lib/Hasura/SQL/Types.hs index e33a8afe5806f..ad9e3d3d47536 100644 --- a/server/src-lib/Hasura/SQL/Types.hs +++ b/server/src-lib/Hasura/SQL/Types.hs @@ -164,6 +164,9 @@ newtype SchemaName publicSchema :: SchemaName publicSchema = SchemaName "public" +hdbViewsSchema :: SchemaName +hdbViewsSchema = SchemaName "hdb_views" + instance IsIden SchemaName where toIden (SchemaName t) = Iden t From adf32fc78eddbc94743c3ee55c850c558847260b Mon Sep 17 00:00:00 2001 From: rakeshkky Date: Thu, 2 May 2019 20:15:54 +0530 Subject: [PATCH 3/5] use SQL loop to drop hdb_views schema views and routines with ordering --- server/src-lib/Hasura/RQL/DDL/Utils.hs | 91 ++++++++++++++++---------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/server/src-lib/Hasura/RQL/DDL/Utils.hs b/server/src-lib/Hasura/RQL/DDL/Utils.hs index 2f83374a4fb44..578858406e3fa 100644 --- a/server/src-lib/Hasura/RQL/DDL/Utils.hs +++ b/server/src-lib/Hasura/RQL/DDL/Utils.hs @@ -5,39 +5,62 @@ module Hasura.RQL.DDL.Utils import qualified Database.PG.Query as Q import Hasura.Prelude -import Hasura.SQL.Types + +-- clearHdbViews :: Q.Tx () +-- clearHdbViews = do +-- -- clear views +-- views <- fetchAllHdbViews +-- mapM_ dropView views +-- -- clear routines +-- routines <- fetchAllRoutines +-- mapM_ dropRoutine routines +-- where +-- fetchAllHdbViews = +-- map runIdentity <$> Q.listQ [Q.sql| +-- SELECT viewname +-- FROM pg_catalog.pg_views +-- WHERE schemaname = $1 +-- |] (Identity hdbViewsSchema) True + +-- fetchAllRoutines = +-- map runIdentity <$> Q.listQ [Q.sql| +-- SELECT routine_name +-- FROM information_schema.routines +-- WHERE specific_schema = $1 +-- |] (Identity hdbViewsSchema) True + +-- dropView :: TableName -> Q.Tx () +-- dropView vn = do +-- let qvn = QualifiedObject hdbViewsSchema vn +-- q = "DROP VIEW IF EXISTS " <> toSQLTxt qvn <> " CASCADE" +-- Q.unitQ (Q.fromText q) () True + +-- dropRoutine :: FunctionName -> Q.Tx () +-- dropRoutine fn = do +-- let qfn = QualifiedObject hdbViewsSchema fn +-- q = "DROP FUNCTION " <> toSQLTxt qfn <> "() CASCADE" +-- Q.unitQ (Q.fromText q) () True clearHdbViews :: Q.Tx () -clearHdbViews = do - -- clear views - views <- fetchAllHdbViews - mapM_ dropView views - -- clear routines - routines <- fetchAllRoutines - mapM_ dropRoutine routines - where - fetchAllHdbViews = - map runIdentity <$> Q.listQ [Q.sql| - SELECT viewname - FROM pg_catalog.pg_views - WHERE schemaname = $1 - |] (Identity hdbViewsSchema) True - - fetchAllRoutines = - map runIdentity <$> Q.listQ [Q.sql| - SELECT routine_name - FROM information_schema.routines - WHERE specific_schema = $1 - |] (Identity hdbViewsSchema) True - - dropView :: TableName -> Q.Tx () - dropView vn = do - let qvn = QualifiedObject hdbViewsSchema vn - q = "DROP VIEW IF EXISTS " <> toSQLTxt qvn <> " CASCADE" - Q.unitQ (Q.fromText q) () True - - dropRoutine :: FunctionName -> Q.Tx () - dropRoutine fn = do - let qfn = QualifiedObject hdbViewsSchema fn - q = "DROP FUNCTION " <> toSQLTxt qfn <> "() CASCADE" - Q.unitQ (Q.fromText q) () True +clearHdbViews = Q.multiQ (Q.fromText (clearHdbOnlyViews <> clearHdbViewsFunc)) + +clearHdbOnlyViews :: Text +clearHdbOnlyViews = + "DO $$ DECLARE \ + \ r RECORD; \ + \ BEGIN \ + \ FOR r IN (SELECT viewname FROM pg_views WHERE schemaname = 'hdb_views' ORDER BY viewname) LOOP \ + \ EXECUTE 'DROP VIEW IF EXISTS hdb_views.' || quote_ident(r.viewname) || ' CASCADE'; \ + \ END LOOP; \ + \ END $$; " + + +clearHdbViewsFunc :: Text +clearHdbViewsFunc = + "DO $$ DECLARE \ + \ r RECORD; \ + \ BEGIN \ + \ FOR r IN (SELECT routine_name FROM information_schema.routines WHERE specific_schema = 'hdb_views' ORDER BY routine_name) LOOP \ + \ EXECUTE 'DROP FUNCTION hdb_views.' || quote_ident(r.routine_name) || '() CASCADE'; \ + \ END LOOP; \ + \ END $$; " From cbce986a317063a267fa93f1aeeab1916706a0c7 Mon Sep 17 00:00:00 2001 From: rakeshkky Date: Thu, 2 May 2019 21:45:15 +0530 Subject: [PATCH 4/5] remove commented code in ..DDL/Utils.hs --- server/src-lib/Hasura/RQL/DDL/Utils.hs | 35 -------------------------- 1 file changed, 35 deletions(-) diff --git a/server/src-lib/Hasura/RQL/DDL/Utils.hs b/server/src-lib/Hasura/RQL/DDL/Utils.hs index 578858406e3fa..1110c23b64bbf 100644 --- a/server/src-lib/Hasura/RQL/DDL/Utils.hs +++ b/server/src-lib/Hasura/RQL/DDL/Utils.hs @@ -6,41 +6,6 @@ import qualified Database.PG.Query as Q import Hasura.Prelude --- clearHdbViews :: Q.Tx () --- clearHdbViews = do --- -- clear views --- views <- fetchAllHdbViews --- mapM_ dropView views --- -- clear routines --- routines <- fetchAllRoutines --- mapM_ dropRoutine routines --- where --- fetchAllHdbViews = --- map runIdentity <$> Q.listQ [Q.sql| --- SELECT viewname --- FROM pg_catalog.pg_views --- WHERE schemaname = $1 --- |] (Identity hdbViewsSchema) True - --- fetchAllRoutines = --- map runIdentity <$> Q.listQ [Q.sql| --- SELECT routine_name --- FROM information_schema.routines --- WHERE specific_schema = $1 --- |] (Identity hdbViewsSchema) True - --- dropView :: TableName -> Q.Tx () --- dropView vn = do --- let qvn = QualifiedObject hdbViewsSchema vn --- q = "DROP VIEW IF EXISTS " <> toSQLTxt qvn <> " CASCADE" --- Q.unitQ (Q.fromText q) () True - --- dropRoutine :: FunctionName -> Q.Tx () --- dropRoutine fn = do --- let qfn = QualifiedObject hdbViewsSchema fn --- q = "DROP FUNCTION " <> toSQLTxt qfn <> "() CASCADE" --- Q.unitQ (Q.fromText q) () True - clearHdbViews :: Q.Tx () clearHdbViews = Q.multiQ (Q.fromText (clearHdbOnlyViews <> clearHdbViewsFunc)) From c689f2455cfd2163392cace2091702b105b94587 Mon Sep 17 00:00:00 2001 From: Vamshi Surabhi Date: Fri, 3 May 2019 13:33:55 +0530 Subject: [PATCH 5/5] schema sync now only processes the latest event This becomes useful when a lot of schema change events happen while we are still processing an earlier event. --- server/src-lib/Hasura/Server/SchemaUpdate.hs | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/server/src-lib/Hasura/Server/SchemaUpdate.hs b/server/src-lib/Hasura/Server/SchemaUpdate.hs index 92df214dcb511..fca76fc9ec8d9 100644 --- a/server/src-lib/Hasura/Server/SchemaUpdate.hs +++ b/server/src-lib/Hasura/Server/SchemaUpdate.hs @@ -80,17 +80,18 @@ startSchemaSync -> InstanceId -> Maybe UTC.UTCTime -> IO () startSchemaSync sqlGenCtx pool logger httpMgr cacheRef instanceId cacheInitTime = do - -- Init events queue - eventsQueue <- STM.newTQueueIO + -- only the latest event is recorded here + -- we don't want to store and process all the events, only the latest event + updateEventRef <- STM.newTVarIO Nothing -- Start listener thread lTId <- C.forkIO $ listener sqlGenCtx pool - logger httpMgr eventsQueue cacheRef instanceId cacheInitTime + logger httpMgr updateEventRef cacheRef instanceId cacheInitTime logThreadStarted TTListener lTId -- Start processor thread pTId <- C.forkIO $ processor sqlGenCtx pool - logger httpMgr eventsQueue cacheRef instanceId + logger httpMgr updateEventRef cacheRef instanceId logThreadStarted TTProcessor pTId where @@ -109,11 +110,11 @@ listener -> PG.PGPool -> Logger -> HTTP.Manager - -> STM.TQueue EventPayload + -> STM.TVar (Maybe EventPayload) -> SchemaCacheRef -> InstanceId -> Maybe UTC.UTCTime -> IO () -listener sqlGenCtx pool logger httpMgr eventsQueue +listener sqlGenCtx pool logger httpMgr updateEventRef cacheRef instanceId cacheInitTime = -- Never exits forever $ do @@ -150,7 +151,7 @@ listener sqlGenCtx pool logger httpMgr eventsQueue Right payload -> do logInfo logger threadType $ object ["received_event" .= payload] -- Push a notify event to Queue - STM.atomically $ STM.writeTQueue eventsQueue payload + STM.atomically $ STM.writeTVar updateEventRef $ Just payload onError = logError logger threadType . TEQueryError logWarn = unLogger logger $ @@ -164,19 +165,28 @@ processor -> PG.PGPool -> Logger -> HTTP.Manager - -> STM.TQueue EventPayload + -> STM.TVar (Maybe EventPayload) -> SchemaCacheRef -> InstanceId -> IO () -processor sqlGenCtx pool logger httpMgr eventsQueue +processor sqlGenCtx pool logger httpMgr updateEventRef cacheRef instanceId = -- Never exits forever $ do - event <- STM.atomically $ STM.readTQueue eventsQueue + event <- STM.atomically getLatestEvent logInfo logger threadType $ object ["processed_event" .= event] when (shouldReload event) $ refreshSchemaCache sqlGenCtx pool logger httpMgr cacheRef threadType "schema cache reloaded" where + -- checks if there is an event + -- and replaces it with Nothing + getLatestEvent = do + eventM <- STM.readTVar updateEventRef + case eventM of + Just event -> do + STM.writeTVar updateEventRef Nothing + return event + Nothing -> STM.retry threadType = TTProcessor -- If event is from another server