Skip to content

Commit e3d46f9

Browse files
utkuozdemirutkuozdemir
committed
feat: implement compression of config fields on resources
Add compression support. Signed-off-by: Utku Ozdemir <[email protected]>
1 parent 4ed9049 commit e3d46f9

File tree

86 files changed

+30627
-960
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+30627
-960
lines changed

‎.dockerignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T12:11:06Z by kres 6d3182c.
3+
# Generated on 2024-09-07T21:30:27Z by kres 8be5fa7.
44

55
*
66
!frontend/src

‎.github/workflows/ci.yaml

+8-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-08-16T15:47:13Z by kres 7be2a05.
3+
# Generated on 2024-09-07T21:30:27Z by kres 8be5fa7.
44

55
name: default
66
concurrency:
@@ -71,7 +71,7 @@ jobs:
7171
timeout-minutes: 10
7272
- name: Mask secrets
7373
run: |
74-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
74+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
7575
- name: Set secrets for job
7676
run: |
7777
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -253,7 +253,7 @@ jobs:
253253
timeout-minutes: 10
254254
- name: Mask secrets
255255
run: |
256-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
256+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
257257
- name: Set secrets for job
258258
run: |
259259
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -331,7 +331,7 @@ jobs:
331331
timeout-minutes: 10
332332
- name: Mask secrets
333333
run: |
334-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
334+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
335335
- name: Set secrets for job
336336
run: |
337337
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -409,7 +409,7 @@ jobs:
409409
timeout-minutes: 10
410410
- name: Mask secrets
411411
run: |
412-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
412+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
413413
- name: Set secrets for job
414414
run: |
415415
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -487,7 +487,7 @@ jobs:
487487
timeout-minutes: 10
488488
- name: Mask secrets
489489
run: |
490-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
490+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
491491
- name: Set secrets for job
492492
run: |
493493
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -565,7 +565,7 @@ jobs:
565565
timeout-minutes: 10
566566
- name: Mask secrets
567567
run: |
568-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
568+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
569569
- name: Set secrets for job
570570
run: |
571571
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"
@@ -643,7 +643,7 @@ jobs:
643643
timeout-minutes: 10
644644
- name: Mask secrets
645645
run: |
646-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
646+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
647647
- name: Set secrets for job
648648
run: |
649649
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-backups-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-backups-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-scaling-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-scaling-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-short-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-short-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-templates-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-templates-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-upgrades-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-07-23T13:55:08Z by kres faf91e3.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-upgrades-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎.github/workflows/e2e-workload-proxy-cron.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
22
#
3-
# Generated on 2024-08-15T23:44:03Z by kres 7be2a05.
3+
# Generated on 2024-09-06T00:35:46Z by kres 8be5fa7.
44

55
name: e2e-workload-proxy-cron
66
concurrency:
@@ -54,7 +54,7 @@ jobs:
5454
timeout-minutes: 10
5555
- name: Mask secrets
5656
run: |
57-
echo -e "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
57+
echo "$(sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | "::add-mask::" + .value')"
5858
- name: Set secrets for job
5959
run: |
6060
sops -d .secrets.yaml | yq -e '.secrets | to_entries[] | .key + "=" + .value' >> "$GITHUB_ENV"

‎client/api/omni/specs/cluster_machine_config.go

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// This Source Code Form is subject to the terms of the Mozilla Public
2+
// License, v. 2.0. If a copy of the MPL was not distributed with this
3+
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4+
5+
package specs
6+
7+
import (
8+
"errors"
9+
10+
"gopkg.in/yaml.v3"
11+
)
12+
13+
// MarshalJSON implements json.Marshaler interface.
14+
//
15+
// It represents compressed fields as uncompressed in the output.
16+
func (x *ClusterMachineConfigSpec) MarshalJSON() ([]byte, error) {
17+
obj := x.CloneVT()
18+
19+
buffer, err := obj.GetUncompressedData()
20+
if err != nil {
21+
return nil, err
22+
}
23+
24+
defer buffer.Free()
25+
26+
obj.Data = buffer.Data()
27+
obj.CompressedData = nil
28+
29+
return jsonMarshaler.Marshal(obj)
30+
}
31+
32+
// UnmarshalJSON implements json.Unmarshaler interface.
33+
func (x *ClusterMachineConfigSpec) UnmarshalJSON(data []byte) error {
34+
return unmarshalJSON(x, data)
35+
}
36+
37+
// MarshalYAML implements yaml.Marshaler interface.
38+
//
39+
// It represents compressed fields as uncompressed in the output.
40+
func (x *ClusterMachineConfigSpec) MarshalYAML() (any, error) {
41+
obj := x.CloneVT()
42+
43+
buffer, err := obj.GetUncompressedData()
44+
if err != nil {
45+
return nil, err
46+
}
47+
48+
defer buffer.Free()
49+
50+
obj.Data = buffer.Data()
51+
obj.CompressedData = nil
52+
53+
type alias *ClusterMachineConfigSpec // prevent recursion
54+
55+
return alias(obj), nil
56+
}
57+
58+
// UnmarshalYAML implements yaml.Unmarshaler interface.
59+
func (x *ClusterMachineConfigSpec) UnmarshalYAML(node *yaml.Node) error {
60+
type alias ClusterMachineConfigSpec // prevent recursion
61+
62+
aux := (*alias)(x)
63+
64+
return unmarshalYAML(x, aux, node)
65+
}
66+
67+
// GetUncompressedData returns the config data from the ClusterMachineConfigSpec, decompressing it if necessary.
68+
func (x *ClusterMachineConfigSpec) GetUncompressedData(opts ...CompressionOption) (Buffer, error) {
69+
if x == nil {
70+
return newNoOpBuffer(nil), nil
71+
}
72+
73+
if x.CompressedData == nil {
74+
return newNoOpBuffer(x.Data), nil
75+
}
76+
77+
return doDecompress(x.CompressedData, getCompressionConfig(opts))
78+
}
79+
80+
// SetUncompressedData sets the config data in the ClusterMachineConfigSpec, compressing it if requested.
81+
func (x *ClusterMachineConfigSpec) SetUncompressedData(data []byte, opts ...CompressionOption) error {
82+
if x == nil {
83+
return errors.New("ClusterMachineConfigSpec is nil")
84+
}
85+
86+
config := getCompressionConfig(opts)
87+
compress := config.Enabled
88+
89+
if !compress {
90+
x.Data = data
91+
x.CompressedData = nil
92+
93+
return nil
94+
}
95+
96+
compressed := doCompress(data, config)
97+
98+
x.CompressedData = compressed
99+
x.Data = nil
100+
101+
return nil
102+
}

0 commit comments

Comments
 (0)