Skip to content

Commit 529cb48

Browse files
authored
feat: Implement webhooks library and perform webhook call on project event (#96)
* Init basic framework for webhooks * Implement WebhookManager * Modify mlp project service to integrate webhook * Refactor interface name * Move setup code into package * Add documentation * Add basic test cases * Set webhook default method to POST * Add retry, configuration to specify payload * Refactor, add comments * Add readme to explain webhook use cases * Update mock clients * Add test case for webhook validation * Fix test cases * Modify exposed methods, add unit tests in project_service * Move mocks into test table runner * Add missing param arsg in project_api_test.go * Fix linting issues * Fix lines issue * Add noOp on success callback, fix formatting * Clean up comments * Address first round of comments * Initialize async and sync webhooks at the start * Fix failing test cases * Add test, fix typo * Remove unneccessary comment * Remove webhooks from default config * Address PR comments * Use validator package, remove onError * Fix broken test * Update readme * Fire async webhook in manager instead * Use context in retry * Add webhook for UpdateProject * Add update project with webhook test * Handle case where webhook event may not be set
1 parent 0256c00 commit 529cb48

16 files changed

Lines changed: 1421 additions & 16 deletions

api/api/projects_api_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ func TestCreateProject(t *testing.T) {
206206
_, err := prjRepository.Save(tC.existingProject)
207207
assert.NoError(t, err)
208208
}
209-
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false)
209+
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false, nil)
210210
assert.NoError(t, err)
211211

212212
appCtx := &AppContext{
@@ -316,7 +316,7 @@ func TestListProjects(t *testing.T) {
316316
assert.NoError(t, err)
317317
}
318318
}
319-
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false)
319+
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false, nil)
320320
assert.NoError(t, err)
321321

322322
appCtx := &AppContext{
@@ -476,7 +476,7 @@ func TestUpdateProject(t *testing.T) {
476476
_, err := prjRepository.Save(tC.existingProject)
477477
assert.NoError(t, err)
478478
}
479-
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false)
479+
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false, nil)
480480
assert.NoError(t, err)
481481

482482
appCtx := &AppContext{
@@ -595,7 +595,7 @@ func TestGetProject(t *testing.T) {
595595
_, err := prjRepository.Save(tC.existingProject)
596596
assert.NoError(t, err)
597597
}
598-
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false)
598+
projectService, err := service.NewProjectsService(mlflowTrackingURL, prjRepository, nil, false, nil)
599599
assert.NoError(t, err)
600600

601601
appCtx := &AppContext{

api/api/router.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
1818
"github.com/caraml-dev/mlp/api/pkg/instrumentation/newrelic"
1919
"github.com/caraml-dev/mlp/api/pkg/secretstorage"
20+
"github.com/caraml-dev/mlp/api/pkg/webhooks"
2021
"github.com/caraml-dev/mlp/api/repository"
2122
"github.com/caraml-dev/mlp/api/service"
2223
"github.com/caraml-dev/mlp/api/validation"
@@ -61,11 +62,19 @@ func NewAppContext(db *gorm.DB, cfg *config.Config) (ctx *AppContext, err error)
6162
return nil, fmt.Errorf("failed to initialize applications service: %v", err)
6263
}
6364

65+
var projectsWebhookManager webhooks.WebhookManager
66+
if cfg.Webhooks != nil && cfg.Webhooks.Enabled {
67+
projectsWebhookManager, err = webhooks.InitializeWebhooks(cfg.Webhooks, service.EventList)
68+
if err != nil {
69+
return nil, fmt.Errorf("failed to initialize projects webhook manager: %v", err)
70+
}
71+
}
72+
6473
projectsService, err := service.NewProjectsService(
6574
cfg.Mlflow.TrackingURL,
6675
repository.NewProjectRepository(db),
6776
authEnforcer,
68-
cfg.Authorization.Enabled)
77+
cfg.Authorization.Enabled, projectsWebhookManager)
6978

7079
if err != nil {
7180
return nil, fmt.Errorf("failed to initialize projects service: %v", err)

api/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515

1616
"github.com/caraml-dev/mlp/api/models"
1717
modelsv2 "github.com/caraml-dev/mlp/api/models/v2"
18+
"github.com/caraml-dev/mlp/api/pkg/webhooks"
1819
)
1920

2021
type Config struct {
@@ -33,6 +34,7 @@ type Config struct {
3334
Mlflow *MlflowConfig `validate:"required"`
3435
DefaultSecretStorage *SecretStorage `validate:"required"`
3536
UI *UIConfig
37+
Webhooks *webhooks.Config
3638
}
3739

3840
// SecretStorage represents the configuration for a secret storage.

api/pkg/webhooks/README.md

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Webhooks
2+
3+
- The package is meant to be used across caraml components (e.g. merlin, turing, mlp) to call webhooks when specific events occur.
4+
- The package contains the webhook client implementation and abstracts the logic from the user. It provides some helper functions for the user to call in their code when specific events occur.
5+
- The payload to the webhook server and the response can be arbitrary, and it is up to the user to choose what payload to send to the webhook server(s), but only 1 response will be used in the callback
6+
7+
### How to use?
8+
9+
1. In the caller package (eg, mlp, merlin), define the list of events that requires webhooks. For example:
10+
11+
```go
12+
const (
13+
ProjectCreatedEvent wh.EventType = "OnProjectCreated"
14+
ProjectUpdatedEvent wh.EventType = "OnProjectUpdated"
15+
)
16+
17+
var EventList = []wh.EventType{
18+
ProjectCreatedEvent,
19+
ProjectUpdatedEvent,
20+
}
21+
```
22+
23+
2. Define the event to webhook configuration. Optionally, the configuration can be provided in a yaml file and parsed via the `Config` struct. In the config file, define the event to webhook mapping for those events as required. For example, if projects need extra labels from an external source, we define the webhook config for the `OnProjectCreated` event
24+
25+
```yaml
26+
webhooks:
27+
enabled: true
28+
config:
29+
OnProjectCreated:
30+
- url: http://localhost:8081/project_created
31+
method: POST
32+
finalResponse: true
33+
name: webhook1
34+
```
35+
36+
3. Call InitializeWebhooks() to get a WebhookManager instance.
37+
This method will initialize the webhook clients for each event type based on the mapping provided
38+
39+
```go
40+
projectsWebhookManager, err := webhooks.InitializeWebhooks(cfg.Webhooks, service.EventList)
41+
```
42+
43+
4. Call
44+
45+
```go
46+
InvokeWebhooks(context.Context, EventType, payload interface{}, onSuccess func([]byte) error, onError func(error) error) error
47+
```
48+
49+
method in the caller code based on the event.
50+
51+
#### Optional webhooks events
52+
53+
In the event that there are multiple events to be configured, for example `OnProjectCreated` and `OnProjectUpdated`, and only `OnProjectCreated` webhooks should be fired, use the `IsEventConfigured()` method provided by the `WebhookManager` to check if the event is set before calling `InvokeWebhooks()`
54+
55+
For example:
56+
57+
```go
58+
if webhookManager == nil || !webhookManager.IsEventConfigured(ProjectUpdatedEvent) {
59+
// do step if webhooks disabled, or event not set
60+
...
61+
} else {
62+
err := webhookManager.InvokeWebhooks(ctx, ProjectUpdatedEvent, project, func(p []byte) error {
63+
// onSuccess steps
64+
...
65+
}, webhooks.NoOpErrorHandler)
66+
}
67+
```
68+
69+
### Single Webhook Configuration
70+
71+
```yaml
72+
webhooks:
73+
enabled: true
74+
config:
75+
OnProjectCreated:
76+
- name: webhook1
77+
url: http://webhook1
78+
method: POST
79+
finalResponse: true
80+
```
81+
82+
- This configuration is the most straight forward. It configures 1 webhook client to be called when the `OnProjectCreated` event happens.
83+
- The payload to the webhook is the json payload of the `payload` argument passed to `InvokeWebhooks`.
84+
- The response from this webhook is used as the final response to the callback passed to the `onSuccess` argument.
85+
86+
### Multiple Webhooks use case
87+
88+
- The library supports multiple webhooks per event to a certain extent.
89+
90+
#### Use case 1
91+
92+
- sync and async webhook
93+
- This can be specified by:
94+
95+
```yaml
96+
webhooks:
97+
enabled: true
98+
config:
99+
OnProjectCreated:
100+
- name: webhook1
101+
url: http://webhook1
102+
method: POST
103+
finalResponse: true
104+
- name: webhook2
105+
url: http://webhook2
106+
method: POST
107+
async: true
108+
```
109+
110+
- The async webhook2 will be called only after webhook1 completes.
111+
- If there are multiple sync and async webhooks, the async webhooks will be called only after all sync webhooks have completed.
112+
113+
#### Use case 2
114+
115+
- 3 sync clients, where the response of the first webhook is used as the payload for the second webhook.
116+
- This can be specified by:
117+
118+
```yaml
119+
webhooks:
120+
enabled: true
121+
config:
122+
OnProjectCreated:
123+
- url: http://webhook1
124+
method: POST
125+
finalResponse: true
126+
name: webhook1
127+
- url: http://webhook2
128+
method: POST
129+
useDataFrom: webhook1 # <-- specify to use data from webhook1
130+
name: webhook2
131+
- url: http://webhook3
132+
method: POST
133+
name: webhook3
134+
```
135+
136+
- The order of webhook matters, webhook1 will be called before webhook2. If webhook2 is defined before webhook1 but uses the response from webhook1, there will be a validation error on initialization.
137+
- Since `useDataFrom` for webhook1 is not set, webhook1 uses the original payload passed to `InvokeWebhooks` function.
138+
- webhook2 will use the response from webhook1 as its payload. The response from webhook2 is not used.
139+
- webhook3 will use the same payload as webhook1, but will only be called after webhook2
140+
- Here, the finalResponse is set to true for webhook1. This means that the response from webhook1 will be passed as an argument to the `onSuccess` function
141+
142+
### Error Handling
143+
144+
- For synchronous webhooks, all webhooks must be successful before the `onSuccess` handler is called. This means that the caller of this package
145+
only needs to consider how to handle the successful response.
146+
- In the event any sync webhooks fail, the `onError` handler is called
147+
- For webhooks that do not need to succeed (for whatever reason), pass them as async webhooks.

api/pkg/webhooks/client.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// package webhooks provides a webhook manager that can be used to invoke webhooks for different events.
2+
3+
/*
4+
Usage:
5+
1. In the caller package (eg, mlp, merlin), define the list of events that requires webhooks. For example:
6+
7+
```go
8+
const (
9+
ProjectCreatedEvent wh.EventType = "OnProjectCreated"
10+
ProjectUpdatedEvent wh.EventType = "OnProjectUpdated"
11+
)
12+
13+
var EventList = []wh.EventType{
14+
ProjectCreatedEvent,
15+
ProjectUpdatedEvent,
16+
}
17+
```
18+
19+
2. Define the event to webhook configuration. Optionally, the configuration can be provided in a yaml file
20+
and parsed via the `Config` struct.
21+
In the config file, define the event to webhook mapping for those events as required.
22+
For example, if projects need extra labels from an external source,
23+
we define the webhook config for the `OnProjectCreated` event
24+
25+
```go
26+
webhooks:
27+
enabled: true
28+
config:
29+
OnProjectCreated:
30+
- url: http://localhost:8081/project_created
31+
method: POST
32+
onError: abort
33+
```
34+
35+
3. Call InitializeWebhooks() to get a WebhookManager instance.
36+
This method will initialize the webhook clients for each event type based on the mapping provided
37+
38+
```go
39+
projectsWebhookManager, err := webhooks.InitializeWebhooks(cfg.Webhooks, service.EventList)
40+
```
41+
42+
4. Call `InvokeWebhooks()` method in the caller code based on the event
43+
*/
44+
45+
package webhooks
46+
47+
import (
48+
"bytes"
49+
"context"
50+
"fmt"
51+
"io"
52+
"net/http"
53+
"time"
54+
55+
"github.com/avast/retry-go/v4"
56+
"github.com/caraml-dev/mlp/api/log"
57+
"github.com/go-playground/validator/v10"
58+
)
59+
60+
type EventType string
61+
type ServiceType string
62+
63+
type WebhookClient interface {
64+
Invoke(context.Context, []byte) ([]byte, error)
65+
IsAsync() bool
66+
IsFinalResponse() bool
67+
GetUseDataFrom() string
68+
GetName() string
69+
}
70+
71+
type simpleWebhookClient struct {
72+
WebhookConfig
73+
}
74+
75+
func NoOpErrorHandler(err error) error { return err }
76+
func NoOpCallback([]byte) error { return nil }
77+
78+
func (g *simpleWebhookClient) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
79+
// create http request to webhook
80+
var content []byte
81+
err := retry.Do(
82+
func() error {
83+
client := http.Client{
84+
Timeout: time.Duration(*g.Timeout) * time.Second,
85+
}
86+
req, err := http.NewRequestWithContext(ctx, g.Method, g.URL, bytes.NewBuffer(payload))
87+
// TODO: Add option for authentication headers
88+
if err != nil {
89+
return err
90+
}
91+
resp, err := client.Do(req)
92+
if err != nil {
93+
log.Errorf("Error making client request %s", err)
94+
return err
95+
}
96+
defer resp.Body.Close()
97+
content, err = io.ReadAll(resp.Body)
98+
if err != nil {
99+
return err
100+
}
101+
if err := validateWebhookResponse(content); err != nil {
102+
return err
103+
}
104+
// check http status code
105+
if resp.StatusCode != http.StatusOK {
106+
return fmt.Errorf("response status code %d not 200", resp.StatusCode)
107+
}
108+
return nil
109+
110+
}, retry.Attempts(uint(g.NumRetries)), retry.Context(ctx),
111+
)
112+
if err != nil {
113+
return nil, err
114+
}
115+
return content, nil
116+
}
117+
118+
func (g *simpleWebhookClient) IsAsync() bool {
119+
return g.Async
120+
}
121+
122+
func (g *simpleWebhookClient) IsFinalResponse() bool {
123+
return g.FinalResponse
124+
}
125+
126+
func (g *simpleWebhookClient) GetUseDataFrom() string {
127+
return g.UseDataFrom
128+
}
129+
130+
func (g *simpleWebhookClient) GetName() string {
131+
return g.Name
132+
}
133+
134+
func validateWebhookConfig(webhookConfig *WebhookConfig) error {
135+
validate := validator.New()
136+
137+
err := validate.Struct(webhookConfig)
138+
if err != nil {
139+
return fmt.Errorf("failed to validate configuration: %s", err)
140+
}
141+
if webhookConfig.NumRetries < 0 {
142+
return fmt.Errorf("numRetries must be a non-negative integer")
143+
}
144+
return nil
145+
}
146+
147+
func setDefaults(webhookConfig *WebhookConfig) {
148+
if webhookConfig.Method == "" {
149+
webhookConfig.Method = http.MethodPost // Default to POST, TODO: decide if GET is allowed
150+
}
151+
if webhookConfig.Timeout == nil {
152+
def := 10
153+
webhookConfig.Timeout = &def
154+
}
155+
}

0 commit comments

Comments
 (0)