Skip to content

Commit 43e888f

Browse files
spy16rohilsurana
spy16
and
rohilsurana
authored
fix: retain previous output during intermediate states (#106)
* fix: retain previous output during intermediate states * feat: add new resource labels to resource revision * fix: firehose replicas should always be greater than 0 * fix: create resource revision only for user triggered updates * feat: add revision reason to resource revisions Co-authored-by: Rohil Surana <[email protected]>
1 parent 9a3050e commit 43e888f

File tree

18 files changed

+68
-45
lines changed

18 files changed

+68
-45
lines changed

‎core/mocks/resource_store.go

+13-11
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎core/module/driver.go

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ type Driver interface {
3636
type Plan struct {
3737
Resource resource.Resource
3838
ScheduleRunAt time.Time
39+
Reason string
3940
}
4041

4142
// Loggable extension of driver allows streaming log data for a resource.

‎core/resource/resource.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ type Store interface {
2121
List(ctx context.Context, filter Filter) ([]Resource, error)
2222

2323
Create(ctx context.Context, r Resource, hooks ...MutationHook) error
24-
Update(ctx context.Context, r Resource, hooks ...MutationHook) error
24+
Update(ctx context.Context, r Resource, saveRevision bool, reason string, hooks ...MutationHook) error
2525
Delete(ctx context.Context, urn string, hooks ...MutationHook) error
2626

2727
Revisions(ctx context.Context, selector RevisionsSelector) ([]Revision, error)
@@ -68,6 +68,7 @@ type RevisionsSelector struct {
6868
type Revision struct {
6969
ID int64 `json:"id"`
7070
URN string `json:"urn"`
71+
Reason string `json:"reason"`
7172
Labels map[string]string `json:"labels"`
7273
CreatedAt time.Time `json:"created_at"`
7374

‎core/sync.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (s *Service) syncChange(ctx context.Context, urn string) (*resource.Resourc
103103
return nil, err
104104
}
105105
} else {
106-
if err := s.upsert(ctx, module.Plan{Resource: *res}, false); err != nil {
106+
if err := s.upsert(ctx, module.Plan{Resource: *res}, false, false, ""); err != nil {
107107
return nil, err
108108
}
109109
}

‎core/write.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func (s *Service) execAction(ctx context.Context, res resource.Resource, act mod
6969
planned.Resource.UpdatedAt = s.clock()
7070
}
7171

72-
if err := s.upsert(ctx, *planned, isCreate(act.Name)); err != nil {
72+
if err := s.upsert(ctx, *planned, isCreate(act.Name), true, planned.Reason); err != nil {
7373
return nil, err
7474
}
7575
return &planned.Resource, nil
@@ -101,7 +101,7 @@ func (s *Service) planChange(ctx context.Context, res resource.Resource, act mod
101101
return planned, nil
102102
}
103103

104-
func (s *Service) upsert(ctx context.Context, plan module.Plan, isCreate bool) error {
104+
func (s *Service) upsert(ctx context.Context, plan module.Plan, isCreate bool, saveRevision bool, reason string) error {
105105
var hooks []resource.MutationHook
106106
hooks = append(hooks, func(ctx context.Context) error {
107107
if plan.Resource.State.IsTerminal() {
@@ -122,7 +122,7 @@ func (s *Service) upsert(ctx context.Context, plan module.Plan, isCreate bool) e
122122
if isCreate {
123123
err = s.store.Create(ctx, plan.Resource, hooks...)
124124
} else {
125-
err = s.store.Update(ctx, plan.Resource, hooks...)
125+
err = s.store.Update(ctx, plan.Resource, saveRevision, reason, hooks...)
126126
}
127127

128128
if err != nil {

‎core/write_test.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,8 @@ func TestService_UpdateResource(t *testing.T) {
394394
Once()
395395

396396
resourceRepo.EXPECT().
397-
Update(mock.Anything, mock.Anything, mock.Anything).
398-
Run(func(ctx context.Context, r resource.Resource, hooks ...resource.MutationHook) {
397+
Update(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
398+
Run(func(ctx context.Context, r resource.Resource, saveRevision bool, reason string, hooks ...resource.MutationHook) {
399399
assert.Len(t, hooks, 1)
400400
assert.NoError(t, hooks[0](ctx))
401401
}).
@@ -453,9 +453,9 @@ func TestService_UpdateResource(t *testing.T) {
453453
Return(&testResource, nil).Once()
454454

455455
resourceRepo.EXPECT().
456-
Update(mock.Anything, mock.Anything, mock.Anything).
456+
Update(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
457457
Return(nil).
458-
Run(func(ctx context.Context, r resource.Resource, hooks ...resource.MutationHook) {
458+
Run(func(ctx context.Context, r resource.Resource, saveRevision bool, reason string, hooks ...resource.MutationHook) {
459459
assert.Len(t, hooks, 1)
460460
assert.NoError(t, hooks[0](ctx))
461461
}).
@@ -578,7 +578,7 @@ func TestService_DeleteResource(t *testing.T) {
578578
Once()
579579

580580
resourceRepo.EXPECT().
581-
Update(mock.Anything, mock.Anything, mock.Anything).
581+
Update(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
582582
Return(testErr).
583583
Once()
584584

@@ -625,7 +625,7 @@ func TestService_DeleteResource(t *testing.T) {
625625
Once()
626626

627627
resourceRepo.EXPECT().
628-
Update(mock.Anything, mock.Anything, mock.Anything).
628+
Update(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
629629
Return(nil).
630630
Once()
631631

@@ -780,7 +780,7 @@ func TestService_ApplyAction(t *testing.T) {
780780
}, nil).
781781
Once()
782782
resourceRepo.EXPECT().
783-
Update(mock.Anything, mock.Anything, mock.Anything).
783+
Update(mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
784784
Return(nil).
785785
Once()
786786

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/stretchr/testify v1.7.1
2121
github.com/xeipuuv/gojsonschema v1.2.0
2222
go.buf.build/odpf/gw/odpf/proton v1.1.122
23-
go.buf.build/odpf/gwv/odpf/proton v1.1.133
23+
go.buf.build/odpf/gwv/odpf/proton v1.1.172
2424
go.opencensus.io v0.23.0
2525
go.uber.org/zap v1.21.0
2626
google.golang.org/grpc v1.46.2

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -1409,8 +1409,8 @@ go.buf.build/odpf/gw/odpf/proton v1.1.122 h1:6NM4D8VwKIdq6F0A5nXnmxPp7LnzuwsGCeV
14091409
go.buf.build/odpf/gw/odpf/proton v1.1.122/go.mod h1:FySqyI0YPPldpzXULKDcIC/bMJIdGaO6j36i1ZKJSvE=
14101410
go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.7/go.mod h1:2Tg6rYIoDhpl39Zd2+WBOF9uG4XxAOs0bK2Z2/bwTOc=
14111411
go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.46/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo=
1412-
go.buf.build/odpf/gwv/odpf/proton v1.1.133 h1:lYGtd7HoAA/KtEOi9ncfx44Pdi9RTr7HyyqdqCTytRI=
1413-
go.buf.build/odpf/gwv/odpf/proton v1.1.133/go.mod h1:V6NNZKrRPHjMkIPiSXvwUHks0D8bUGPXAjXUaujG/90=
1412+
go.buf.build/odpf/gwv/odpf/proton v1.1.172 h1:cGk4ctsVhBK4d6mV+QVrJD0rWkXtDO+ogCA8l3BCkhk=
1413+
go.buf.build/odpf/gwv/odpf/proton v1.1.172/go.mod h1:V6NNZKrRPHjMkIPiSXvwUHks0D8bUGPXAjXUaujG/90=
14141414
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
14151415
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
14161416
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=

‎internal/server/v1/resources/mappers.go

+1
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ func revisionToProto(revision resource.Revision) (*entropyv1beta1.ResourceRevisi
138138
return &entropyv1beta1.ResourceRevision{
139139
Id: strconv.FormatInt(revision.ID, decimalBase),
140140
Urn: revision.URN,
141+
Reason: revision.Reason,
141142
Labels: revision.Labels,
142143
CreatedAt: timestamppb.New(revision.CreatedAt),
143144
Spec: spec,

‎internal/store/postgres/resource_store.go

+13-11
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,11 @@ func (st *Store) Create(ctx context.Context, r resource.Resource, hooks ...resou
114114
return translateErr(err)
115115
}
116116

117-
// TODO: Add labels for revisions
118117
rev := resource.Revision{
119118
URN: r.URN,
120119
Spec: r.Spec,
121-
Labels: map[string]string{},
120+
Labels: r.Labels,
121+
Reason: "resource created",
122122
}
123123

124124
if err := insertRevision(ctx, tx, rev); err != nil {
@@ -135,7 +135,7 @@ func (st *Store) Create(ctx context.Context, r resource.Resource, hooks ...resou
135135
return nil
136136
}
137137

138-
func (st *Store) Update(ctx context.Context, r resource.Resource, hooks ...resource.MutationHook) error {
138+
func (st *Store) Update(ctx context.Context, r resource.Resource, saveRevision bool, reason string, hooks ...resource.MutationHook) error {
139139
updateResource := func(ctx context.Context, tx *sqlx.Tx) error {
140140
id, err := translateURNToID(ctx, tx, r.URN)
141141
if err != nil {
@@ -165,15 +165,17 @@ func (st *Store) Update(ctx context.Context, r resource.Resource, hooks ...resou
165165
return err
166166
}
167167

168-
// TODO: Add labels for revisions
169-
rev := resource.Revision{
170-
URN: r.URN,
171-
Spec: r.Spec,
172-
Labels: map[string]string{},
173-
}
168+
if saveRevision {
169+
rev := resource.Revision{
170+
URN: r.URN,
171+
Spec: r.Spec,
172+
Labels: r.Labels,
173+
Reason: reason,
174+
}
174175

175-
if err := insertRevision(ctx, tx, rev); err != nil {
176-
return translateErr(err)
176+
if err := insertRevision(ctx, tx, rev); err != nil {
177+
return translateErr(err)
178+
}
177179
}
178180

179181
return runAllHooks(ctx, hooks)

‎internal/store/postgres/revision_model.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ import (
1414
type revisionModel struct {
1515
ID int64 `db:"id"`
1616
URN string `db:"urn"`
17+
Reason string `db:"reason"`
1718
CreatedAt time.Time `db:"created_at"`
1819
SpecConfigs []byte `db:"spec_configs"`
1920
}
2021

2122
func readRevisionRecord(ctx context.Context, r sqlx.QueryerContext, id int64, into *revisionModel) error {
22-
cols := []string{"id", "urn", "created_at", "spec_configs"}
23+
cols := []string{"id", "urn", "reason", "created_at", "spec_configs"}
2324
builder := sq.Select(cols...).From(tableRevisions).Where(sq.Eq{"id": id})
2425

2526
query, args, err := builder.PlaceholderFormat(sq.Dollar).ToSql()

‎internal/store/postgres/revision_store.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ func (st *Store) getRevisionByID(ctx context.Context, id int64) (*resource.Revis
7575
return &resource.Revision{
7676
ID: rec.ID,
7777
URN: rec.URN,
78+
Reason: rec.Reason,
7879
Labels: tagsToLabelMap(tags),
7980
CreatedAt: rec.CreatedAt,
8081
Spec: resource.Spec{
@@ -98,8 +99,8 @@ func insertRevision(ctx context.Context, tx *sqlx.Tx, rev resource.Revision) err
9899

99100
func insertRevisionRecord(ctx context.Context, runner sq.BaseRunner, r resource.Revision) (int64, error) {
100101
q := sq.Insert(tableRevisions).
101-
Columns("urn", "spec_configs").
102-
Values(r.URN, r.Spec.Configs).
102+
Columns("urn", "reason", "spec_configs").
103+
Values(r.URN, r.Reason, r.Spec.Configs).
103104
Suffix(`RETURNING "id"`).
104105
PlaceholderFormat(sq.Dollar)
105106

‎internal/store/postgres/schema.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,5 @@ CREATE TABLE IF NOT EXISTS modules (
6565
updated_at timestamp with time zone NOT NULL DEFAULT current_timestamp
6666
);
6767

68-
CREATE INDEX IF NOT EXISTS idx_modules_project ON modules (project);
68+
CREATE INDEX IF NOT EXISTS idx_modules_project ON modules (project);
69+
ALTER TABLE revisions ADD COLUMN IF NOT EXISTS reason TEXT DEFAULT '<none>' NOT NULL;

‎modules/firehose/plan.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ func (*firehoseModule) planCreate(res module.ExpandedResource, act module.Action
4444
if reqConf.StopTime != nil {
4545
plan.ScheduleRunAt = *reqConf.StopTime
4646
}
47+
plan.Reason = "firehose created"
4748
return &plan, nil
4849
}
4950

@@ -70,6 +71,7 @@ func (*firehoseModule) planChange(res module.ExpandedResource, act module.Action
7071
if conf.StopTime != nil {
7172
plan.ScheduleRunAt = *conf.StopTime
7273
}
74+
plan.Reason = "firehose config updated"
7375

7476
case ScaleAction:
7577
var scaleParams struct {
@@ -79,17 +81,21 @@ func (*firehoseModule) planChange(res module.ExpandedResource, act module.Action
7981
return nil, errors.ErrInvalid.WithMsgf("invalid config json: %v", err)
8082
}
8183
conf.Firehose.Replicas = scaleParams.Replicas
84+
plan.Reason = "firehose scaled"
8285

8386
case StartAction:
8487
conf.State = stateRunning
88+
plan.Reason = "firehose started"
8589

8690
case StopAction:
8791
conf.State = stateStopped
92+
plan.Reason = "firehose stopped"
8893
}
8994

9095
r.Spec.Configs = conf.JSON()
9196
r.State = resource.State{
9297
Status: resource.StatusPending,
98+
Output: res.State.Output,
9399
ModuleData: moduleData{
94100
PendingSteps: []string{releaseUpdate},
95101
}.JSON(),
@@ -125,12 +131,13 @@ func (*firehoseModule) planReset(res module.ExpandedResource, act module.ActionR
125131
r.Spec.Configs = conf.JSON()
126132
r.State = resource.State{
127133
Status: resource.StatusPending,
134+
Output: res.State.Output,
128135
ModuleData: moduleData{
129136
PendingSteps: []string{releaseUpdate, consumerReset, releaseUpdate},
130137
ResetTo: resetTo,
131138
StateOverride: stateStopped,
132139
}.JSON(),
133140
}
134141

135-
return &module.Plan{Resource: r}, nil
142+
return &module.Plan{Resource: r, Reason: "firehose consumer reset"}, nil
136143
}

‎modules/firehose/plan_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ func TestFirehoseModule_Plan(t *testing.T) {
6464
ModuleData: []byte(`{"pending_steps":["release_create"]}`),
6565
},
6666
},
67+
Reason: "firehose created",
6768
},
6869
},
6970
{
@@ -96,6 +97,7 @@ func TestFirehoseModule_Plan(t *testing.T) {
9697
ModuleData: []byte(`{"pending_steps":["release_update"]}`),
9798
},
9899
},
100+
Reason: "firehose scaled",
99101
},
100102
},
101103
{
@@ -119,6 +121,7 @@ func TestFirehoseModule_Plan(t *testing.T) {
119121
ModuleData: []byte(`{"pending_steps":["release_update","consumer_reset","release_update"],"reset_to":"2022-06-22T00:00:00+00:00","state_override":"STOPPED"}`),
120122
},
121123
},
124+
Reason: "firehose consumer reset",
122125
},
123126
},
124127
{
@@ -143,6 +146,7 @@ func TestFirehoseModule_Plan(t *testing.T) {
143146
},
144147
},
145148
ScheduleRunAt: parseTime("3022-07-13T00:40:14.028016Z"),
149+
Reason: "firehose created",
146150
},
147151
},
148152
}

‎modules/firehose/schema/config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"properties": {
2424
"replicas": {
2525
"type": "number",
26-
"default": 1
26+
"default": 1,
27+
"minimum": 1
2728
},
2829
"kafka_broker_address": {
2930
"type": "string"

‎modules/firehose/schema/scale.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
"type": "object",
55
"properties": {
66
"replicas": {
7-
"type": "number"
7+
"type": "number",
8+
"minimum": 1
89
}
910
},
1011
"required": ["replicas"]

‎modules/kubernetes/kubernetes.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func (m *kubeModule) Plan(ctx context.Context, res module.ExpandedResource, act
5656
Status: resource.StatusCompleted,
5757
Output: output,
5858
}
59-
return &module.Plan{Resource: res.Resource}, nil
59+
return &module.Plan{Resource: res.Resource, Reason: "kubernetes cluster details updated"}, nil
6060
}
6161

6262
func (*kubeModule) Sync(_ context.Context, res module.ExpandedResource) (*resource.State, error) {

0 commit comments

Comments
 (0)