From a3aca4cfd7881ff27acd53324a6f530b333b476a Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan A Date: Fri, 2 Nov 2018 16:11:27 +0530 Subject: [PATCH 1/9] 1) Put all event trigger conf into one column 2) Accept webhook_from_env --- server/src-lib/Hasura/Events/Lib.hs | 2 +- server/src-lib/Hasura/RQL/DDL/Metadata.hs | 12 +- server/src-lib/Hasura/RQL/DDL/Schema/Table.hs | 14 ++- server/src-lib/Hasura/RQL/DDL/Subscribe.hs | 58 ++++++---- .../src-lib/Hasura/RQL/Types/SchemaCache.hs | 22 ++-- server/src-lib/Hasura/RQL/Types/Subscribe.hs | 108 +++++++++++++----- server/src-rsr/initialise.sql | 9 +- 7 files changed, 142 insertions(+), 83 deletions(-) diff --git a/server/src-lib/Hasura/Events/Lib.hs b/server/src-lib/Hasura/Events/Lib.hs index f4cc88c6878ef..f9e1e3400b376 100644 --- a/server/src-lib/Hasura/Events/Lib.hs +++ b/server/src-lib/Hasura/Events/Lib.hs @@ -279,7 +279,7 @@ tryWebhook logenv pool e = do case meti of Nothing -> return $ Left $ HOther "table or event-trigger not found" Just eti -> do - let webhook = etiWebhook eti + let webhook = wciCachedValue $ etiWebhookInfo eti createdAt = eCreatedAt e eventId = eId e headerInfos = etiHeaders eti diff --git a/server/src-lib/Hasura/RQL/DDL/Metadata.hs b/server/src-lib/Hasura/RQL/DDL/Metadata.hs index 7083dfe9d3a38..a2fa596723b8c 100644 --- a/server/src-lib/Hasura/RQL/DDL/Metadata.hs +++ b/server/src-lib/Hasura/RQL/DDL/Metadata.hs @@ -63,7 +63,7 @@ data TableMeta , _tmSelectPermissions :: ![DP.SelPermDef] , _tmUpdatePermissions :: ![DP.UpdPermDef] , _tmDeletePermissions :: ![DP.DelPermDef] - , _tmEventTriggers :: ![DTS.EventTriggerDef] + , _tmEventTriggers :: ![DTS.EventTriggerConf] } deriving (Show, Eq, Lift) mkTableMeta :: QualifiedTable -> TableMeta @@ -165,7 +165,7 @@ applyQP1 (ReplaceMetadata tables templates) = do selPerms = map DP.pdRole $ table ^. tmSelectPermissions updPerms = map DP.pdRole $ table ^. tmUpdatePermissions delPerms = map DP.pdRole $ table ^. tmDeletePermissions - eventTriggers = map DTS.etdName $ table ^. tmEventTriggers + eventTriggers = map DTS.etcName $ table ^. tmEventTriggers checkMultipleDecls "relationships" allRels checkMultipleDecls "insert permissions" insPerms @@ -325,9 +325,9 @@ fetchMetadata = do mkTriggerMetaDefs = mapM trigRowToDef - trigRowToDef (sn, tn, trn, Q.AltJ tDefVal, webhook, nr, rint, Q.AltJ mheaders) = do - tDef <- decodeValue tDefVal - return (QualifiedTable sn tn, DTS.EventTriggerDef trn tDef webhook (RetryConf nr rint) mheaders) + trigRowToDef (sn, tn, Q.AltJ configuration) = do + conf <- decodeValue configuration + return (QualifiedTable sn tn, conf::EventTriggerConf) fetchTables = Q.listQ [Q.sql| @@ -357,7 +357,7 @@ fetchMetadata = do |] () False fetchEventTriggers = Q.listQ [Q.sql| - SELECT e.schema_name, e.table_name, e.name, e.definition::json, e.webhook, e.num_retries, e.retry_interval, e.headers::json + SELECT e.schema_name, e.table_name, e.configuration::json FROM hdb_catalog.event_triggers e |] () False diff --git a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs index 048ebb74f6dfc..953104c047c54 100644 --- a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs +++ b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs @@ -376,14 +376,16 @@ buildSchemaCache = flip execStateT emptySchemaCache $ do addQTemplateToCache qti eventTriggers <- lift $ Q.catchE defaultTxErrorHandler fetchEventTriggers - forM_ eventTriggers $ \(sn, tn, trid, trn, Q.AltJ tDefVal, webhook, nr, rint, Q.AltJ mheaders) -> do - let headerConfs = fromMaybe [] mheaders + forM_ eventTriggers $ \(sn, tn, trid, trn, Q.AltJ configuration) -> do + conf <- decodeValue configuration + let EventTriggerConf _ opsdef webhookConf retryConf mheaders = conf + headerConfs = fromMaybe [] mheaders qt = QualifiedTable sn tn allCols <- getCols . tiFieldInfoMap <$> askTabInfo qt + webhookInfo <- getWebhookInfoFromConf webhookConf headers <- getHeaderInfosFromConf headerConfs - tDef <- decodeValue tDefVal - addEventTriggerToCache (QualifiedTable sn tn) trid trn tDef (RetryConf nr rint) webhook headers - liftTx $ mkTriggerQ trid trn qt allCols tDef + addEventTriggerToCache (QualifiedTable sn tn) trid trn opsdef retryConf webhookInfo headers + liftTx $ mkTriggerQ trid trn qt allCols opsdef where permHelper sn tn rn pDef pa = do qCtx <- mkAdminQCtx <$> get @@ -421,7 +423,7 @@ buildSchemaCache = flip execStateT emptySchemaCache $ do fetchEventTriggers = Q.listQ [Q.sql| - SELECT e.schema_name, e.table_name, e.id, e.name, e.definition::json, e.webhook, e.num_retries, e.retry_interval, e.headers::json + SELECT e.schema_name, e.table_name, e.id, e.name, e.configuration::json FROM hdb_catalog.event_triggers e |] () False diff --git a/server/src-lib/Hasura/RQL/DDL/Subscribe.hs b/server/src-lib/Hasura/RQL/DDL/Subscribe.hs index 6f70ffecd1934..d97682bf9d9b2 100644 --- a/server/src-lib/Hasura/RQL/DDL/Subscribe.hs +++ b/server/src-lib/Hasura/RQL/DDL/Subscribe.hs @@ -122,17 +122,17 @@ mkTriggerQ trid trn qt allCols (TriggerOpsDef insert update delete) = do addEventTriggerToCatalog :: QualifiedTable -> [PGColInfo] - -> EventTriggerDef + -> EventTriggerConf -> Q.TxE QErr TriggerId -addEventTriggerToCatalog qt@(QualifiedTable sn tn) allCols (EventTriggerDef name def webhook rconf mheaders) = do +addEventTriggerToCatalog qt@(QualifiedTable sn tn) allCols eventTriggerConf@(EventTriggerConf name opsdef _ _ _) = do ids <- map runIdentity <$> Q.listQE defaultTxErrorHandler [Q.sql| - INSERT into hdb_catalog.event_triggers (name, type, schema_name, table_name, definition, webhook, num_retries, retry_interval, headers) - VALUES ($1, 'table', $2, $3, $4, $5, $6, $7, $8) + INSERT into hdb_catalog.event_triggers (name, type, schema_name, table_name, configuration) + VALUES ($1, 'table', $2, $3, $4) RETURNING id - |] (name, sn, tn, Q.AltJ $ toJSON def, webhook, toInt64 $ rcNumRetries rconf, toInt64 $ rcIntervalSec rconf, Q.AltJ $ toJSON mheaders) True + |] (name, sn, tn, Q.AltJ $ toJSON eventTriggerConf) True trid <- getTrid ids - mkTriggerQ trid name qt allCols def + mkTriggerQ trid name qt allCols opsdef return trid where getTrid [] = throw500 "could not create event-trigger" @@ -153,22 +153,18 @@ delEventTriggerFromCatalog trn = do updateEventTriggerToCatalog :: QualifiedTable -> [PGColInfo] - -> EventTriggerDef + -> EventTriggerConf -> Q.TxE QErr TriggerId -updateEventTriggerToCatalog qt allCols (EventTriggerDef name def webhook rconf mheaders) = do +updateEventTriggerToCatalog qt allCols eventTriggerConf@(EventTriggerConf name opsdef _ _ _) = do ids <- map runIdentity <$> Q.listQE defaultTxErrorHandler [Q.sql| UPDATE hdb_catalog.event_triggers SET - definition = $1, - webhook = $2, - num_retries = $3, - retry_interval = $4, - headers = $5 - WHERE name = $6 + configuration = $1 + WHERE name = $2 RETURNING id - |] (Q.AltJ $ toJSON def, webhook, toInt64 $ rcNumRetries rconf, toInt64 $ rcIntervalSec rconf, Q.AltJ $ toJSON mheaders, name) True + |] (Q.AltJ $ toJSON eventTriggerConf, name) True trid <- getTrid ids - mkTriggerQ trid name qt allCols def + mkTriggerQ trid name qt allCols opsdef return trid where getTrid [] = throw500 "could not update event-trigger" @@ -223,8 +219,8 @@ markForDelivery eid = WHERE id = $1 |] (Identity eid) True -subTableP1 :: (P1C m) => CreateEventTriggerQuery -> m (QualifiedTable, Bool, EventTriggerDef) -subTableP1 (CreateEventTriggerQuery name qt insert update delete retryConf webhook mheaders replace) = do +subTableP1 :: (P1C m) => CreateEventTriggerQuery -> m (QualifiedTable, Bool, EventTriggerConf) +subTableP1 (CreateEventTriggerQuery name qt insert update delete retryConf webhook webhookFromEnv mheaders replace) = do adminOnly ti <- askTabInfo qt -- can only replace for same table @@ -236,8 +232,12 @@ subTableP1 (CreateEventTriggerQuery name qt insert update delete retryConf webho assertCols ti update assertCols ti delete + webhookConf <- case (webhook, webhookFromEnv) of + (Just w, Nothing) -> return $ WCValue w + (Nothing, Just wEnv) -> return $ WCEnv wEnv + _ -> throw500 "expected webhook or webhook_from_env" let rconf = fromMaybe (RetryConf defaultNumRetries defaultRetryInterval) retryConf - return (qt, replace, EventTriggerDef name (TriggerOpsDef insert update delete) webhook rconf mheaders) + return (qt, replace, EventTriggerConf name (TriggerOpsDef insert update delete) webhookConf rconf mheaders) where assertCols _ Nothing = return () assertCols ti (Just sos) = do @@ -246,8 +246,8 @@ subTableP1 (CreateEventTriggerQuery name qt insert update delete retryConf webho SubCStar -> return () SubCArray pgcols -> forM_ pgcols (assertPGCol (tiFieldInfoMap ti) "") -subTableP2 :: (P2C m) => QualifiedTable -> Bool -> EventTriggerDef -> m () -subTableP2 qt replace q@(EventTriggerDef name def webhook rconf mheaders) = do +subTableP2 :: (P2C m) => QualifiedTable -> Bool -> EventTriggerConf -> m () +subTableP2 qt replace q@(EventTriggerConf name def webhookConf rconf mheaders) = do allCols <- getCols . tiFieldInfoMap <$> askTabInfo qt trid <- if replace then do @@ -256,16 +256,17 @@ subTableP2 qt replace q@(EventTriggerDef name def webhook rconf mheaders) = do else liftTx $ addEventTriggerToCatalog qt allCols q let headerConfs = fromMaybe [] mheaders + webhookInfo <- getWebhookInfoFromConf webhookConf headerInfos <- getHeaderInfosFromConf headerConfs - addEventTriggerToCache qt trid name def rconf webhook headerInfos + addEventTriggerToCache qt trid name def rconf webhookInfo headerInfos -subTableP2shim :: (P2C m) => (QualifiedTable, Bool, EventTriggerDef) -> m RespBody +subTableP2shim :: (P2C m) => (QualifiedTable, Bool, EventTriggerConf) -> m RespBody subTableP2shim (qt, replace, etdef) = do subTableP2 qt replace etdef return successMsg instance HDBQuery CreateEventTriggerQuery where - type Phase1Res CreateEventTriggerQuery = (QualifiedTable, Bool, EventTriggerDef) + type Phase1Res CreateEventTriggerQuery = (QualifiedTable, Bool, EventTriggerConf ) phaseOne = subTableP1 phaseTwo _ = subTableP2shim schemaCachePolicy = SCPReload @@ -312,5 +313,14 @@ getHeaderInfosFromConf = mapM getHeader Nothing -> throw400 NotFound $ "environment variable '" <> val <> "' not set" Just envval -> return $ EventHeaderInfo hconf (T.pack envval) +getWebhookInfoFromConf :: (P2C m) => WebhookConf -> m WebhookConfInfo +getWebhookInfoFromConf wc = case wc of + WCValue w -> return $ WebhookConfInfo wc w + WCEnv we -> do + mEnv <- liftIO $ lookupEnv (T.unpack we) + case mEnv of + Nothing -> throw400 NotFound $ "environment variable '" <> we <> "' not set" + Just envval -> return $ WebhookConfInfo wc (T.pack envval) + toInt64 :: (Integral a) => a -> Int64 toInt64 = fromIntegral diff --git a/server/src-lib/Hasura/RQL/Types/SchemaCache.hs b/server/src-lib/Hasura/RQL/Types/SchemaCache.hs index acb94a88c35f0..4ba218bd61a42 100644 --- a/server/src-lib/Hasura/RQL/Types/SchemaCache.hs +++ b/server/src-lib/Hasura/RQL/Types/SchemaCache.hs @@ -369,14 +369,14 @@ instance CachedSchemaObj OpTriggerInfo where data EventTriggerInfo = EventTriggerInfo - { etiId :: !TriggerId - , etiName :: !TriggerName - , etiInsert :: !(Maybe OpTriggerInfo) - , etiUpdate :: !(Maybe OpTriggerInfo) - , etiDelete :: !(Maybe OpTriggerInfo) - , etiRetryConf :: !RetryConf - , etiWebhook :: !T.Text - , etiHeaders :: ![EventHeaderInfo] + { etiId :: !TriggerId + , etiName :: !TriggerName + , etiInsert :: !(Maybe OpTriggerInfo) + , etiUpdate :: !(Maybe OpTriggerInfo) + , etiDelete :: !(Maybe OpTriggerInfo) + , etiRetryConf :: !RetryConf + , etiWebhookInfo :: !WebhookConfInfo + , etiHeaders :: ![EventHeaderInfo] } deriving (Show, Eq) $(deriveToJSON (aesonDrop 3 snakeCase) ''EventTriggerInfo) @@ -636,10 +636,10 @@ addEventTriggerToCache -> TriggerName -> TriggerOpsDef -> RetryConf - -> T.Text + -> WebhookConfInfo -> [EventHeaderInfo] -> m () -addEventTriggerToCache qt trid trn tdef rconf webhook headers = +addEventTriggerToCache qt trid trn tdef rconf webhookInfo headers = modTableInCache modEventTriggerInfo qt where modEventTriggerInfo ti = do @@ -650,7 +650,7 @@ addEventTriggerToCache qt trid trn tdef rconf webhook headers = (getOpInfo trn ti $ tdUpdate tdef) (getOpInfo trn ti $ tdDelete tdef) rconf - webhook + webhookInfo headers etim = tiEventTriggerInfoMap ti -- fail $ show (toJSON eti) diff --git a/server/src-lib/Hasura/RQL/Types/Subscribe.hs b/server/src-lib/Hasura/RQL/Types/Subscribe.hs index 703d684122b81..685d60f2b6f0d 100644 --- a/server/src-lib/Hasura/RQL/Types/Subscribe.hs +++ b/server/src-lib/Hasura/RQL/Types/Subscribe.hs @@ -12,7 +12,7 @@ module Hasura.RQL.Types.Subscribe , EventId , TriggerOpsDef(..) , EventTrigger(..) - , EventTriggerDef(..) + , EventTriggerConf(..) , RetryConf(..) , DeleteEventTriggerQuery(..) , DeliverEventQuery(..) @@ -20,6 +20,8 @@ module Hasura.RQL.Types.Subscribe , HeaderValue(..) , HeaderName , EventHeaderInfo(..) + , WebhookConf(..) + , WebhookConfInfo(..) ) where import Data.Aeson @@ -98,30 +100,47 @@ data EventHeaderInfo $(deriveToJSON (aesonDrop 3 snakeCase){omitNothingFields=True} ''EventHeaderInfo) +data WebhookConf = WCValue T.Text | WCEnv T.Text + deriving (Show, Eq, Lift) + +instance ToJSON WebhookConf where + toJSON (WCValue w) = String w + toJSON (WCEnv wEnv) = String wEnv + +data WebhookConfInfo + = WebhookConfInfo + { wciWebhookConf :: !WebhookConf + , wciCachedValue :: !T.Text + } deriving (Show, Eq, Lift) + +$(deriveToJSON (aesonDrop 3 snakeCase){omitNothingFields=True} ''WebhookConfInfo) + data CreateEventTriggerQuery = CreateEventTriggerQuery - { cetqName :: !T.Text - , cetqTable :: !QualifiedTable - , cetqInsert :: !(Maybe SubscribeOpSpec) - , cetqUpdate :: !(Maybe SubscribeOpSpec) - , cetqDelete :: !(Maybe SubscribeOpSpec) - , cetqRetryConf :: !(Maybe RetryConf) - , cetqWebhook :: !T.Text - , cetqHeaders :: !(Maybe [HeaderConf]) - , cetqReplace :: !Bool + { cetqName :: !T.Text + , cetqTable :: !QualifiedTable + , cetqInsert :: !(Maybe SubscribeOpSpec) + , cetqUpdate :: !(Maybe SubscribeOpSpec) + , cetqDelete :: !(Maybe SubscribeOpSpec) + , cetqRetryConf :: !(Maybe RetryConf) + , cetqWebhook :: !(Maybe T.Text) + , cetqWebhookFromEnv :: !(Maybe T.Text) + , cetqHeaders :: !(Maybe [HeaderConf]) + , cetqReplace :: !Bool } deriving (Show, Eq, Lift) instance FromJSON CreateEventTriggerQuery where parseJSON (Object o) = do - name <- o .: "name" - table <- o .: "table" - insert <- o .:? "insert" - update <- o .:? "update" - delete <- o .:? "delete" - retryConf <- o .:? "retry_conf" - webhook <- o .: "webhook" - headers <- o .:? "headers" - replace <- o .:? "replace" .!= False + name <- o .: "name" + table <- o .: "table" + insert <- o .:? "insert" + update <- o .:? "update" + delete <- o .:? "delete" + retryConf <- o .:? "retry_conf" + webhook <- o .:? "webhook" + webhookFromEnv <- o .:? "webhook_from_env" + headers <- o .:? "headers" + replace <- o .:? "replace" .!= False let regex = mkRegex "^\\w+$" mName = matchRegex regex (T.unpack name) case mName of @@ -130,8 +149,13 @@ instance FromJSON CreateEventTriggerQuery where case insert <|> update <|> delete of Just _ -> return () Nothing -> fail "must provide operation spec(s)" + case (webhook, webhookFromEnv) of + (Just _, Nothing) -> return () + (Nothing, Just _) -> return () + (Just _, Just _) -> fail "only one of webhook or webhook_from_env should be given" + _ -> fail "must provide webhook or webhook_from_env" mapM_ checkEmptyCols [insert, update, delete] - return $ CreateEventTriggerQuery name table insert update delete retryConf webhook headers replace + return $ CreateEventTriggerQuery name table insert update delete retryConf webhook webhookFromEnv headers replace where checkEmptyCols spec = case spec of @@ -169,16 +193,44 @@ data EventTrigger $(deriveJSON (aesonDrop 2 snakeCase){omitNothingFields=True} ''EventTrigger) -data EventTriggerDef - = EventTriggerDef - { etdName :: !TriggerName - , etdDefinition :: !TriggerOpsDef - , etdWebhook :: !T.Text - , etdRetryConf :: !RetryConf - , etdHeaders :: !(Maybe [HeaderConf]) +data EventTriggerConf + = EventTriggerConf + { etcName :: !TriggerName + , etcOpsDefinition :: !TriggerOpsDef + , etcWebhookConf :: !WebhookConf + , etcRetryConf :: !RetryConf + , etcHeaders :: !(Maybe [HeaderConf]) } deriving (Show, Eq, Lift) -$(deriveJSON (aesonDrop 3 snakeCase){omitNothingFields=True} ''EventTriggerDef) +instance FromJSON EventTriggerConf where + parseJSON (Object o) = do + name <- o .: "name" + opsDef <- o .: "definition" + webhook <- o .:? "webhook" + webhookFromEnv <- o .:? "webhook_from_env" + retryConf <- o .: "retry_conf" + headers <- o .: "headers" + webhookConf <- case (webhook, webhookFromEnv) of + (Just w, Nothing) -> return $ WCValue w + (Nothing, Just wEnv) -> return $ WCEnv wEnv + (Just _, Just _) -> fail "only one of webhook or webhook_from_env should be given" + _ -> fail "must provide webhook or webhook_from_env" + return $ EventTriggerConf name opsDef webhookConf retryConf headers + parseJSON _ = fail "expecting object for event_trigger_def" + +instance ToJSON EventTriggerConf where + toJSON (EventTriggerConf name def (WCValue w) rc headers) = object ["name" .= name + , "definition" .= def + , "webhook" .= w + , "retry_conf" .= rc + , "headers" .= headers + ] + toJSON (EventTriggerConf name def (WCEnv wEnv) rc headers) = object ["name" .= name + , "definition" .= def + , "webhook_from_env" .= wEnv + , "retry_conf" .= rc + , "headers" .= headers + ] data DeliverEventQuery = DeliverEventQuery diff --git a/server/src-rsr/initialise.sql b/server/src-rsr/initialise.sql index 69b8a129d6c6e..7265eb93a770e 100644 --- a/server/src-rsr/initialise.sql +++ b/server/src-rsr/initialise.sql @@ -188,13 +188,8 @@ CREATE TABLE hdb_catalog.event_triggers type TEXT NOT NULL, schema_name TEXT NOT NULL, table_name TEXT NOT NULL, - definition JSON, - query TEXT, - webhook TEXT NOT NULL, - num_retries INTEGER DEFAULT 0, - retry_interval INTEGER DEFAULT 10, - comment TEXT, - headers JSON + configuration JSON, + comment TEXT ); CREATE TABLE hdb_catalog.event_log From 3837d34afce2d9faaacda9e3a6d07c5ade141214 Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan A Date: Fri, 2 Nov 2018 17:31:47 +0530 Subject: [PATCH 2/9] catalog updates for event trigger configuration change --- server/src-exec/Ops.hs | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/server/src-exec/Ops.hs b/server/src-exec/Ops.hs index b19049aeaf2ef..648bc6e25ef02 100644 --- a/server/src-exec/Ops.hs +++ b/server/src-exec/Ops.hs @@ -29,7 +29,7 @@ import qualified Database.PG.Query as Q import qualified Database.PG.Query.Connection as Q curCatalogVer :: T.Text -curCatalogVer = "3" +curCatalogVer = "4" initCatalogSafe :: UTCTime -> Q.TxE QErr String initCatalogSafe initTime = do @@ -188,6 +188,31 @@ from2To3 = Q.catchE defaultTxErrorHandler $ do Q.unitQ "CREATE INDEX ON hdb_catalog.event_log (trigger_id)" () False Q.unitQ "CREATE INDEX ON hdb_catalog.event_invocation_logs (event_id)" () False +from3To4 :: Q.TxE QErr () +from3To4 = Q.catchE defaultTxErrorHandler $ do + Q.unitQ "ALTER TABLE hdb_catalog.event_triggers ADD COLUMN configuration JSON" () False + eventTriggers <- map uncurryEventTrigger <$> Q.listQ [Q.sql| + SELECT e.name, e.definition::json, e.webhook, e.num_retries, e.retry_interval, e.headers::json + FROM hdb_catalog.event_triggers e + |] () False + forM_ eventTriggers updateEventTrigger3To4 + Q.unitQ "ALTER TABLE hdb_catalog.event_triggers\ + \ DROP COLUMN definition\ + \, DROP COLUMN query\ + \, DROP COLUMN webhook\ + \, DROP COLUMN num_retries\ + \, DROP COLUMN retry_interval\ + \, DROP COLUMN headers" () False + where + uncurryEventTrigger (trn, Q.AltJ tDef, w, nr, rint, Q.AltJ headers) = EventTriggerConf trn tDef (WCValue w) (RetryConf nr rint) headers + updateEventTrigger3To4 etc@(EventTriggerConf name _ _ _ _) = Q.unitQ [Q.sql| + UPDATE hdb_catalog.event_triggers + SET + configuration = $1 + WHERE name = $2 + |] (Q.AltJ $ A.toJSON etc, name) True + + migrateCatalog :: UTCTime -> Q.TxE QErr String migrateCatalog migrationTime = do preVer <- getCatalogVersion @@ -196,12 +221,17 @@ migrateCatalog migrationTime = do | preVer == "0.8" -> from08ToCurrent | preVer == "1" -> from1ToCurrent | preVer == "2" -> from2ToCurrent + | preVer == "3" -> from3ToCurrent | otherwise -> throw400 NotSupported $ "migrate: unsupported version : " <> preVer where + from3ToCurrent = do + from3To4 + postMigrate + from2ToCurrent = do from2To3 - postMigrate + from3ToCurrent from1ToCurrent = do from1To2 From df4cf338755eaacacfe9c6df81a9fd4bd81a81c2 Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan A Date: Fri, 2 Nov 2018 18:04:16 +0530 Subject: [PATCH 3/9] tests for webhook from env --- .circleci/test-server.sh | 1 + .../event_triggers/webhook_env/setup.yaml | 26 ++++++++++++ .../event_triggers/webhook_env/teardown.yaml | 9 ++++ server/tests-py/test_events.py | 42 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 server/tests-py/queries/event_triggers/webhook_env/setup.yaml create mode 100644 server/tests-py/queries/event_triggers/webhook_env/teardown.yaml diff --git a/.circleci/test-server.sh b/.circleci/test-server.sh index a9d72444e937d..92857784a96ed 100755 --- a/.circleci/test-server.sh +++ b/.circleci/test-server.sh @@ -107,6 +107,7 @@ mkdir -p "$OUTPUT_FOLDER" export EVENT_WEBHOOK_HEADER="MyEnvValue" export HGE_URL="http://localhost:8080" +export WEBHOOK_FROM_ENV="http://127.0.0.1:5592" PID="" WH_PID="" diff --git a/server/tests-py/queries/event_triggers/webhook_env/setup.yaml b/server/tests-py/queries/event_triggers/webhook_env/setup.yaml new file mode 100644 index 0000000000000..f4e02ba8853de --- /dev/null +++ b/server/tests-py/queries/event_triggers/webhook_env/setup.yaml @@ -0,0 +1,26 @@ +type: bulk +args: +- type: run_sql + args: + sql: | + create table hge_tests.test_t1( + c1 int, + c2 text + ); +- type: track_table + args: + schema: hge_tests + name: test_t1 +- type: create_event_trigger + args: + name: t1_all + table: + schema: hge_tests + name: test_t1 + insert: + columns: "*" + update: + columns: "*" + delete: + columns: "*" + webhook_from_env: WEBHOOK_FROM_ENV diff --git a/server/tests-py/queries/event_triggers/webhook_env/teardown.yaml b/server/tests-py/queries/event_triggers/webhook_env/teardown.yaml new file mode 100644 index 0000000000000..a0766f4d43ca0 --- /dev/null +++ b/server/tests-py/queries/event_triggers/webhook_env/teardown.yaml @@ -0,0 +1,9 @@ +type: bulk +args: +- type: delete_event_trigger + args: + name: t1_all +- type: run_sql + args: + sql: | + drop table hge_tests.test_t1 diff --git a/server/tests-py/test_events.py b/server/tests-py/test_events.py index 864e4284a07f0..367c535a5be31 100755 --- a/server/tests-py/test_events.py +++ b/server/tests-py/test_events.py @@ -442,3 +442,45 @@ def test_selected_payload_dep(self, hge_ctx): }) assert st_code == 400, resp assert resp['code'] == "dependency-error", resp + +class TestWebhookEnv(object): + + @pytest.fixture(autouse=True) + def transact(self, request, hge_ctx): + print("In setup method") + st_code, resp = hge_ctx.v1q_f('queries/event_triggers/webhook_env/setup.yaml') + assert st_code == 200, resp + yield + st_code, resp = hge_ctx.v1q_f('queries/event_triggers/webhook_env/teardown.yaml') + assert st_code == 200, resp + + def test_basic(self, hge_ctx): + table = {"schema": "hge_tests", "name": "test_t1"} + + init_row = {"c1": 1, "c2": "hello"} + exp_ev_data = { + "old": None, + "new": init_row + } + headers = {} + st_code, resp = insert(hge_ctx, table, init_row) + assert st_code == 200, resp + check_event(hge_ctx, "t1_all", table, "INSERT", exp_ev_data, headers, "/") + + where_exp = {"c1": 1} + set_exp = {"c2": "world"} + exp_ev_data = { + "old": init_row, + "new": {"c1": 1, "c2": "world"} + } + st_code, resp = update(hge_ctx, table, where_exp, set_exp) + assert st_code == 200, resp + check_event(hge_ctx, "t1_all", table, "UPDATE", exp_ev_data, headers, "/") + + exp_ev_data = { + "old": {"c1": 1, "c2": "world"}, + "new": None + } + st_code, resp = delete(hge_ctx, table, where_exp) + assert st_code == 200, resp + check_event(hge_ctx, "t1_all", table, "DELETE", exp_ev_data, headers, "/") From b6e84adfb3962033dee7e9b1d5db081f250b8d99 Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan A Date: Fri, 2 Nov 2018 18:22:40 +0530 Subject: [PATCH 4/9] fix: headers is optional --- server/src-lib/Hasura/RQL/Types/Subscribe.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src-lib/Hasura/RQL/Types/Subscribe.hs b/server/src-lib/Hasura/RQL/Types/Subscribe.hs index 685d60f2b6f0d..d5ba68b706e9b 100644 --- a/server/src-lib/Hasura/RQL/Types/Subscribe.hs +++ b/server/src-lib/Hasura/RQL/Types/Subscribe.hs @@ -209,7 +209,7 @@ instance FromJSON EventTriggerConf where webhook <- o .:? "webhook" webhookFromEnv <- o .:? "webhook_from_env" retryConf <- o .: "retry_conf" - headers <- o .: "headers" + headers <- o .:? "headers" webhookConf <- case (webhook, webhookFromEnv) of (Just w, Nothing) -> return $ WCValue w (Nothing, Just wEnv) -> return $ WCEnv wEnv From dac3032085c0c47453208bf1701e99c7e08f629a Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan A Date: Thu, 8 Nov 2018 12:57:53 +0530 Subject: [PATCH 5/9] no-op changes --- server/src-lib/Hasura/RQL/DDL/Schema/Table.hs | 2 +- server/src-lib/Hasura/RQL/DDL/Subscribe.hs | 23 +++++++++------- server/src-lib/Hasura/RQL/Types/Subscribe.hs | 26 ++++++++++--------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs index 953104c047c54..691c7d432d734 100644 --- a/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs +++ b/server/src-lib/Hasura/RQL/DDL/Schema/Table.hs @@ -384,7 +384,7 @@ buildSchemaCache = flip execStateT emptySchemaCache $ do allCols <- getCols . tiFieldInfoMap <$> askTabInfo qt webhookInfo <- getWebhookInfoFromConf webhookConf headers <- getHeaderInfosFromConf headerConfs - addEventTriggerToCache (QualifiedTable sn tn) trid trn opsdef retryConf webhookInfo headers + addEventTriggerToCache qt trid trn opsdef retryConf webhookInfo headers liftTx $ mkTriggerQ trid trn qt allCols opsdef where permHelper sn tn rn pDef pa = do diff --git a/server/src-lib/Hasura/RQL/DDL/Subscribe.hs b/server/src-lib/Hasura/RQL/DDL/Subscribe.hs index d97682bf9d9b2..622d39922731a 100644 --- a/server/src-lib/Hasura/RQL/DDL/Subscribe.hs +++ b/server/src-lib/Hasura/RQL/DDL/Subscribe.hs @@ -308,19 +308,22 @@ getHeaderInfosFromConf = mapM getHeader getHeader hconf = case hconf of (HeaderConf _ (HVValue val)) -> return $ EventHeaderInfo hconf val (HeaderConf _ (HVEnv val)) -> do - mEnv <- liftIO $ lookupEnv (T.unpack val) - case mEnv of - Nothing -> throw400 NotFound $ "environment variable '" <> val <> "' not set" - Just envval -> return $ EventHeaderInfo hconf (T.pack envval) + envVal <- getEnv val + return $ EventHeaderInfo hconf envVal getWebhookInfoFromConf :: (P2C m) => WebhookConf -> m WebhookConfInfo getWebhookInfoFromConf wc = case wc of - WCValue w -> return $ WebhookConfInfo wc w - WCEnv we -> do - mEnv <- liftIO $ lookupEnv (T.unpack we) - case mEnv of - Nothing -> throw400 NotFound $ "environment variable '" <> we <> "' not set" - Just envval -> return $ WebhookConfInfo wc (T.pack envval) + WCValue w -> return $ WebhookConfInfo wc w + WCEnv we -> do + envVal <- getEnv we + return $ WebhookConfInfo wc envVal + +getEnv :: (QErrM m, MonadIO m) => T.Text -> m T.Text +getEnv env = do + mEnv <- liftIO $ lookupEnv (T.unpack env) + case mEnv of + Nothing -> throw400 NotFound $ "environment variable '" <> env <> "' not set" + Just envVal -> return (T.pack envVal) toInt64 :: (Integral a) => a -> Int64 toInt64 = fromIntegral diff --git a/server/src-lib/Hasura/RQL/Types/Subscribe.hs b/server/src-lib/Hasura/RQL/Types/Subscribe.hs index d5ba68b706e9b..a17e3f77dac7c 100644 --- a/server/src-lib/Hasura/RQL/Types/Subscribe.hs +++ b/server/src-lib/Hasura/RQL/Types/Subscribe.hs @@ -219,18 +219,20 @@ instance FromJSON EventTriggerConf where parseJSON _ = fail "expecting object for event_trigger_def" instance ToJSON EventTriggerConf where - toJSON (EventTriggerConf name def (WCValue w) rc headers) = object ["name" .= name - , "definition" .= def - , "webhook" .= w - , "retry_conf" .= rc - , "headers" .= headers - ] - toJSON (EventTriggerConf name def (WCEnv wEnv) rc headers) = object ["name" .= name - , "definition" .= def - , "webhook_from_env" .= wEnv - , "retry_conf" .= rc - , "headers" .= headers - ] + toJSON (EventTriggerConf name def (WCValue w) rc headers) = object + [ "name" .= name + , "definition" .= def + , "webhook" .= w + , "retry_conf" .= rc + , "headers" .= headers + ] + toJSON (EventTriggerConf name def (WCEnv wEnv) rc headers) = object + [ "name" .= name + , "definition" .= def + , "webhook_from_env" .= wEnv + , "retry_conf" .= rc + , "headers" .= headers + ] data DeliverEventQuery = DeliverEventQuery From 1209cf44a0e8fe39655ab12a4e10e1cf0249b52e Mon Sep 17 00:00:00 2001 From: Karthik Venkateswaran Date: Mon, 12 Nov 2018 18:24:15 +0530 Subject: [PATCH 6/9] Add console changes --- .../Common/DropdownButton/DropdownButton.js | 83 +++++++++++++++++++ .../Services/EventTrigger/Add/AddActions.js | 15 +++- .../Services/EventTrigger/Add/AddState.js | 1 + .../Services/EventTrigger/Add/AddTrigger.js | 83 ++++++++++++++++--- .../EventTrigger/Settings/Settings.js | 71 ++++++++++------ .../EventTrigger/TableCommon/Table.scss | 4 +- .../EventTrigger/TableCommon/TableStyles.scss | 1 + console/src/helpers/semver.js | 1 + 8 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 console/src/components/Common/DropdownButton/DropdownButton.js diff --git a/console/src/components/Common/DropdownButton/DropdownButton.js b/console/src/components/Common/DropdownButton/DropdownButton.js new file mode 100644 index 0000000000000..70309071f8e5c --- /dev/null +++ b/console/src/components/Common/DropdownButton/DropdownButton.js @@ -0,0 +1,83 @@ +import React from 'react'; +import PropTypes from 'prop-types'; + +import InputGroup from 'react-bootstrap/lib/InputGroup'; +import DropdownButton from 'react-bootstrap/lib/DropdownButton'; +import MenuItem from 'react-bootstrap/lib/MenuItem'; + +class DropButton extends React.Component { + render() { + const { + title, + dropdownOptions, + value, + required, + onInputChange, + onButtonChange, + dataKey, + dataIndex, + bsClass, + disabled, + inputVal, + inputPlaceHolder, + id, + testId, + } = this.props; + return ( + + + {dropdownOptions.map((d, i) => ( + + {d.display_text} + + ))} + + + + ); + } +} + +DropButton.propTypes = { + dispatch: PropTypes.func.isRequired, + dropdownOptions: PropTypes.array.isRequired, + title: PropTypes.string.isRequired, + value: PropTypes.string.isRequired, + dataKey: PropTypes.string.isRequired, + dataIndex: PropTypes.string.isRequired, + inputVal: PropTypes.string.isRequired, + inputPlaceHolder: PropTypes.string, + required: PropTypes.bool.isRequired, + onButtonChange: PropTypes.func.isRequired, + onInputChange: PropTypes.func.isRequired, + bsClass: PropTypes.string, + id: PropTypes.string, + testId: PropTypes.string, + disabled: PropTypes.bool.isRequired, +}; + +export default DropButton; diff --git a/console/src/components/Services/EventTrigger/Add/AddActions.js b/console/src/components/Services/EventTrigger/Add/AddActions.js index e696531deb8b4..c40525c568b48 100644 --- a/console/src/components/Services/EventTrigger/Add/AddActions.js +++ b/console/src/components/Services/EventTrigger/Add/AddActions.js @@ -32,6 +32,7 @@ const SET_HEADERKEY = 'AddTrigger/SET_HEADERKEY'; const SET_HEADERTYPE = 'AddTrigger/SET_HEADERTYPE'; const SET_HEADERVALUE = 'AddTrigger/SET_HEADERVALUE'; const ADD_HEADER = 'AddTrigger/ADD_HEADER'; +const UPDATE_WEBHOOK_URL_TYPE = 'AddTrigger/UPDATE_WEBHOOK_URL_TYPE'; const setTriggerName = value => ({ type: SET_TRIGGERNAME, value }); const setTableName = value => ({ type: SET_TABLENAME, value }); @@ -65,6 +66,10 @@ const validationError = error => { return { type: VALIDATION_ERROR, error }; }; +const getWebhookKey = (type, val) => { + return { [type === 'url' ? 'webhook' : 'webhook_from_env']: val }; +}; + const createTrigger = () => { return (dispatch, getState) => { dispatch({ type: MAKING_REQUEST }); @@ -74,6 +79,7 @@ const createTrigger = () => { const triggerName = currentState.triggerName; const tableName = currentState.tableName; const webhook = currentState.webhookURL; + const webhookType = currentState.webhookUrlType; // apply migrations const migrationName = 'create_trigger_' + triggerName.trim(); @@ -82,7 +88,8 @@ const createTrigger = () => { args: { name: triggerName, table: { name: tableName, schema: currentSchema }, - webhook: webhook, + // webhook: webhook, + ...getWebhookKey(webhookType, webhook), }, }; const downPayload = { @@ -352,6 +359,11 @@ const addTriggerReducer = (state = defaultState, action) => { const deselectedOperations = state.selectedOperations; deselectedOperations[action.data] = false; return { ...state, selectedOperations: { ...deselectedOperations } }; + case UPDATE_WEBHOOK_URL_TYPE: + return { + ...state, + webhookUrlType: action.data, + }; default: return state; } @@ -376,5 +388,6 @@ export { operationToggleAllColumns, setOperationSelection, setDefaults, + UPDATE_WEBHOOK_URL_TYPE, }; export { validationError }; diff --git a/console/src/components/Services/EventTrigger/Add/AddState.js b/console/src/components/Services/EventTrigger/Add/AddState.js index e3f692ba5ed31..c67c43567c408 100644 --- a/console/src/components/Services/EventTrigger/Add/AddState.js +++ b/console/src/components/Services/EventTrigger/Add/AddState.js @@ -6,6 +6,7 @@ const defaultState = { operations: { insert: [], update: [], delete: [] }, selectedOperations: { insert: false, update: false, delete: false }, webhookURL: '', + webhookUrlType: 'url', retryConf: null, ongoingRequest: false, lastError: null, diff --git a/console/src/components/Services/EventTrigger/Add/AddTrigger.js b/console/src/components/Services/EventTrigger/Add/AddTrigger.js index a48f497fc2c76..9798adf9a781d 100644 --- a/console/src/components/Services/EventTrigger/Add/AddTrigger.js +++ b/console/src/components/Services/EventTrigger/Add/AddTrigger.js @@ -20,12 +20,15 @@ import { operationToggleAllColumns, setOperationSelection, setDefaults, + UPDATE_WEBHOOK_URL_TYPE, } from './AddActions'; import { listDuplicate } from '../../../../utils/data'; import { showErrorNotification } from '../Notification'; import { createTrigger } from './AddActions'; import { fetchTableListBySchema } from './AddActions'; +import DropdownButton from '../../../Common/DropdownButton/DropdownButton'; + import semverCheck from '../../../../helpers/semver'; class AddTrigger extends Component { @@ -35,18 +38,23 @@ class AddTrigger extends Component { this.state = { advancedExpanded: false, supportColumnChangeFeature: false, + supportWebhookEnv: false, }; } componentDidMount() { // set defaults this.props.dispatch(setDefaults()); if (this.props.serverVersion) { - this.checkSemVer(this.props.serverVersion); + this.checkSemVer(this.props.serverVersion).then(() => { + this.checkWebhookEnvSupport(this.props.serverVersion); + }); } } componentWillReceiveProps(nextProps) { if (nextProps.serverVersion !== this.props.serverVersion) { - this.checkSemVer(nextProps.serverVersion); + this.checkSemVer(nextProps.serverVersion).then(() => { + this.checkWebhookEnvSupport(nextProps.serverVersion); + }); } } componentWillUnmount() { @@ -69,6 +77,13 @@ class AddTrigger extends Component { this.updateSupportColumnChangeFeature(false); console.error(e); } + return Promise.resolve(); + } + + checkWebhookEnvSupport(version) { + const supportWebhookEnv = semverCheck('webhookEnvSupport', version); + this.setState({ ...this.state, supportWebhookEnv }); + return Promise.resolve(); } updateSupportColumnChangeFeature(val) { @@ -78,6 +93,14 @@ class AddTrigger extends Component { }); } + updateWebhookUrlType(e) { + const field = e.target.getAttribute('value'); + if (field === 'env' || field === 'url') { + this.props.dispatch({ type: UPDATE_WEBHOOK_URL_TYPE, data: field }); + this.props.dispatch(setWebhookURL('')); + } + } + submitValidation(e) { // validations e.preventDefault(); @@ -172,6 +195,8 @@ class AddTrigger extends Component { lastSuccess, internalError, headers, + webhookURL, + webhookUrlType, } = this.props; const { supportColumnChangeFeature } = this.state; @@ -578,16 +603,50 @@ class AddTrigger extends Component {