From 94cf198b35aab9fdb20fa68005e3d7cef01b944d Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Thu, 2 Jul 2020 01:30:02 +0530 Subject: [PATCH 1/6] include scheduled triggers metadata in the webhook body --- .../src-lib/Hasura/Eventing/ScheduledTrigger.hs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs index 96859e3d60092..127e7c27f693a 100644 --- a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs +++ b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs @@ -211,6 +211,18 @@ data ScheduledEventType = -- so all the configuration is fetched along the scheduled events. deriving (Eq, Show) +data ScheduledEventWebhookBody + = ScheduledEventWebhookBody + { sewbId :: !Text + , sewbName :: !(Maybe TriggerName) + , sewbScheduledTime :: !UTCTime + , sewbPayload :: !J.Value + , sewbComment :: !(Maybe Text) + , sewbCreatedAt :: !UTCTime + } deriving (Show, Eq) + +$(J.deriveToJSON (J.aesonDrop 4 J.snakeCase) {J.omitNothingFields = True} ''ScheduledEventWebhookBody) + -- | runCronEventsGenerator makes sure that all the cron triggers -- have an adequate buffer of cron events. runCronEventsGenerator :: @@ -443,7 +455,9 @@ processScheduledEvent httpTimeout = HTTP.responseTimeoutMicro (timeoutSeconds * 1000000) headers = addDefaultHeaders $ map encodeHeader sefHeaders extraLogCtx = ExtraLogContext (Just currentTime) sefId - res <- runExceptT $ tryWebhook headers httpTimeout sefPayload (T.unpack sefWebhook) + webhookBody = + ScheduledEventWebhookBody sefId sefName sefScheduledTime sefPayload sefComment currentTime + res <- runExceptT $ tryWebhook headers httpTimeout (J.toJSON webhookBody) (T.unpack sefWebhook) logHTTPForST res extraLogCtx let decodedHeaders = map (decodeHeader logEnv sefHeaders) headers either From 4307edd4db862258ca4e532182fe1ca26b299a36 Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Thu, 2 Jul 2020 02:13:58 +0530 Subject: [PATCH 2/6] save the request sent to the webhook in the invocation logs --- .../Hasura/Eventing/ScheduledTrigger.hs | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs index 127e7c27f693a..9a55559d79d7f 100644 --- a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs +++ b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs @@ -455,35 +455,42 @@ processScheduledEvent httpTimeout = HTTP.responseTimeoutMicro (timeoutSeconds * 1000000) headers = addDefaultHeaders $ map encodeHeader sefHeaders extraLogCtx = ExtraLogContext (Just currentTime) sefId - webhookBody = + webhookReqBody = ScheduledEventWebhookBody sefId sefName sefScheduledTime sefPayload sefComment currentTime - res <- runExceptT $ tryWebhook headers httpTimeout (J.toJSON webhookBody) (T.unpack sefWebhook) + webhookReqBodyJson = J.toJSON webhookReqBody + res <- runExceptT $ tryWebhook headers httpTimeout webhookReqBodyJson (T.unpack sefWebhook) logHTTPForST res extraLogCtx let decodedHeaders = map (decodeHeader logEnv sefHeaders) headers either - (processError pgpool se decodedHeaders type') - (processSuccess pgpool se decodedHeaders type') + (processError pgpool se decodedHeaders type' webhookReqBodyJson) + (processSuccess pgpool se decodedHeaders type' webhookReqBodyJson) res processError :: (MonadIO m, MonadError QErr m) - => Q.PGPool -> ScheduledEventFull -> [HeaderConf] -> ScheduledEventType -> HTTPErr a -> m () -processError pgpool se decodedHeaders type' err = do + => Q.PGPool + -> ScheduledEventFull + -> [HeaderConf] + -> ScheduledEventType + -> J.Value + -> HTTPErr a + -> m () +processError pgpool se decodedHeaders type' reqJson err = do let invocation = case err of HClient excp -> do let errMsg = TBS.fromLBS $ J.encode $ show excp - mkInvocation se 1000 decodedHeaders errMsg [] + mkInvocation se 1000 decodedHeaders errMsg [] reqJson HParse _ detail -> do let errMsg = TBS.fromLBS $ J.encode detail - mkInvocation se 1001 decodedHeaders errMsg [] + mkInvocation se 1001 decodedHeaders errMsg [] reqJson HStatus errResp -> do let respPayload = hrsBody errResp respHeaders = hrsHeaders errResp respStatus = hrsStatus errResp - mkInvocation se respStatus decodedHeaders respPayload respHeaders + mkInvocation se respStatus decodedHeaders respPayload respHeaders reqJson HOther detail -> do let errMsg = (TBS.fromLBS $ J.encode detail) - mkInvocation se 500 decodedHeaders errMsg [] + mkInvocation se 500 decodedHeaders errMsg [] reqJson liftExceptTIO $ Q.runTx pgpool (Q.RepeatableRead, Just Q.ReadWrite) $ do insertInvocation invocation type' @@ -540,12 +547,18 @@ and it can transition to other states in the following ways: processSuccess :: (MonadIO m, MonadError QErr m) - => Q.PGPool -> ScheduledEventFull -> [HeaderConf] -> ScheduledEventType -> HTTPResp a -> m () -processSuccess pgpool se decodedHeaders type' resp = do + => Q.PGPool + -> ScheduledEventFull + -> [HeaderConf] + -> ScheduledEventType + -> J.Value + -> HTTPResp a + -> m () +processSuccess pgpool se decodedHeaders type' reqBodyJson resp = do let respBody = hrsBody resp respHeaders = hrsHeaders resp respStatus = hrsStatus resp - invocation = mkInvocation se respStatus decodedHeaders respBody respHeaders + invocation = mkInvocation se respStatus decodedHeaders respBody respHeaders reqBodyJson liftExceptTIO $ Q.runTx pgpool (Q.RepeatableRead, Just Q.ReadWrite) $ do insertInvocation invocation type' @@ -576,17 +589,22 @@ setRetry se time type' = |] (time, sefId se) True mkInvocation - :: ScheduledEventFull -> Int -> [HeaderConf] -> TBS.TByteString -> [HeaderConf] + :: ScheduledEventFull + -> Int + -> [HeaderConf] + -> TBS.TByteString + -> [HeaderConf] + -> J.Value -> (Invocation 'ScheduledType) -mkInvocation se status reqHeaders respBody respHeaders +mkInvocation ScheduledEventFull {sefId} status reqHeaders respBody respHeaders reqBodyJson = let resp = if isClientError status then mkClientErr respBody else mkResp status respBody respHeaders in Invocation - (sefId se) + sefId status - (mkWebhookReq (J.toJSON se) reqHeaders invocationVersionST) + (mkWebhookReq reqBodyJson reqHeaders invocationVersionST) resp insertInvocation :: (Invocation 'ScheduledType) -> ScheduledEventType -> Q.TxE QErr () From dd1da4b9b9ed013f1da3c0e09b48492ebf4a104b Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Thu, 2 Jul 2020 02:31:30 +0530 Subject: [PATCH 3/6] modify the scheduled trigger test --- server/tests-py/test_scheduled_triggers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/tests-py/test_scheduled_triggers.py b/server/tests-py/test_scheduled_triggers.py index c2587eca1e870..fd697c9cd6b99 100644 --- a/server/tests-py/test_scheduled_triggers.py +++ b/server/tests-py/test_scheduled_triggers.py @@ -81,7 +81,10 @@ def test_check_fired_webhook_event(self,hge_ctx,scheduled_triggers_evts_webhook) event = scheduled_triggers_evts_webhook.get_event(65) validate_event_webhook(event['path'],'/test') validate_event_headers(event['headers'],{"header-key":"header-value"}) - assert event['body'] == self.webhook_payload + assert event['body']['payload'] == self.webhook_payload + payload_keys = dict.keys(event['body']) + for k in ["scheduled_time","created_at","id"]: # additional keys + assert k in payload_keys assert scheduled_triggers_evts_webhook.is_queue_empty() def test_check_events_statuses(self,hge_ctx): From 4cce049f842434a7fd4bb9af87baa66a2bb5ea34 Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Thu, 2 Jul 2020 12:13:29 +0530 Subject: [PATCH 4/6] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94c1ea84549b2..b4bd158ea749c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - server: add new `--conn-lifetime` and `HASURA_GRAPHQL_PG_CONN_LIFETIME` options for expiring connections after some amount of active time (#5087) - server: shrink libpq connection request/response buffers back to 1MB if they grow beyond 2MB, fixing leak-like behavior on active servers (#5087) +- server: include scheduled event metadata (`created_at`,`scheduled_time`,`id`, etc) along with the configured payload in the request body to the webhook.NOTE: This is breaking for beta versions as the payload is now inside a key called `payload` - docs: add note for managed databases in postgres requirements (close #1677, #3783) (#5228) - docs: add 1-click deployment to Nhost page to the deployment guides (#5180) - docs: add hasura cloud to getting started section (close #5206) (#5208) From 16b634310aa2340ede5e02d4bd74508723d4d85c Mon Sep 17 00:00:00 2001 From: Tirumarai Selvan Date: Thu, 2 Jul 2020 12:51:40 +0530 Subject: [PATCH 5/6] emphasize note on changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4bd158ea749c..d47efa7bce531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ - server: add new `--conn-lifetime` and `HASURA_GRAPHQL_PG_CONN_LIFETIME` options for expiring connections after some amount of active time (#5087) - server: shrink libpq connection request/response buffers back to 1MB if they grow beyond 2MB, fixing leak-like behavior on active servers (#5087) -- server: include scheduled event metadata (`created_at`,`scheduled_time`,`id`, etc) along with the configured payload in the request body to the webhook.NOTE: This is breaking for beta versions as the payload is now inside a key called `payload` +- server: include scheduled event metadata (`created_at`,`scheduled_time`,`id`, etc) along with the configured payload in the request body to the webhook. +**WARNING:** This is breaking for beta versions as the payload is now inside a key called `payload`. - docs: add note for managed databases in postgres requirements (close #1677, #3783) (#5228) - docs: add 1-click deployment to Nhost page to the deployment guides (#5180) - docs: add hasura cloud to getting started section (close #5206) (#5208) From b86948eb78757424b90b0044deda26c85272cb25 Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Fri, 3 Jul 2020 04:51:08 +0530 Subject: [PATCH 6/6] refactor ScheduledEventWebhookBody to ScheduledEventWebhookPayload --- .../Hasura/Eventing/ScheduledTrigger.hs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs index a82f2e5fe1a88..98c5d685e9769 100644 --- a/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs +++ b/server/src-lib/Hasura/Eventing/ScheduledTrigger.hs @@ -224,17 +224,17 @@ data ScheduledEventType = -- so all the configuration is fetched along the scheduled events. deriving (Eq, Show) -data ScheduledEventWebhookBody - = ScheduledEventWebhookBody - { sewbId :: !Text - , sewbName :: !(Maybe TriggerName) - , sewbScheduledTime :: !UTCTime - , sewbPayload :: !J.Value - , sewbComment :: !(Maybe Text) - , sewbCreatedAt :: !UTCTime +data ScheduledEventWebhookPayload + = ScheduledEventWebhookPayload + { sewpId :: !Text + , sewpName :: !(Maybe TriggerName) + , sewpScheduledTime :: !UTCTime + , sewpPayload :: !J.Value + , sewpComment :: !(Maybe Text) + , sewpCreatedAt :: !UTCTime } deriving (Show, Eq) -$(J.deriveToJSON (J.aesonDrop 4 J.snakeCase) {J.omitNothingFields = True} ''ScheduledEventWebhookBody) +$(J.deriveToJSON (J.aesonDrop 4 J.snakeCase) {J.omitNothingFields = True} ''ScheduledEventWebhookPayload) -- | runCronEventsGenerator makes sure that all the cron triggers -- have an adequate buffer of cron events. @@ -481,9 +481,9 @@ processScheduledEvent httpTimeout = HTTP.responseTimeoutMicro (timeoutSeconds * 1000000) headers = addDefaultHeaders $ map encodeHeader sefHeaders extraLogCtx = ExtraLogContext (Just currentTime) sefId - webhookReqBody = - ScheduledEventWebhookBody sefId sefName sefScheduledTime sefPayload sefComment currentTime - webhookReqBodyJson = J.toJSON webhookReqBody + webhookReqPayload = + ScheduledEventWebhookPayload sefId sefName sefScheduledTime sefPayload sefComment currentTime + webhookReqBodyJson = J.toJSON webhookReqPayload res <- runExceptT $ tryWebhook headers httpTimeout webhookReqBodyJson (T.unpack sefWebhook) logHTTPForST res extraLogCtx let decodedHeaders = map (decodeHeader logEnv sefHeaders) headers