Skip to content

Commit 8dcbc26

Browse files
nielsbaumannielsbauman
committed
Add frozen_after WIP
1 parent 75be786 commit 8dcbc26

File tree

10 files changed

+238
-34
lines changed

10 files changed

+238
-34
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,34 @@
174174
- match: {component_templates.0.component_template.template.lifecycle.downsampling.0.after: "1d"}
175175
- match: {component_templates.0.component_template.template.lifecycle.downsampling.0.fixed_interval: "5m"}
176176

177+
---
178+
"Add data stream lifecycle with frozen after":
179+
- requires:
180+
capabilities:
181+
- method: PUT
182+
path: /_component_template/{name}
183+
capabilities: [ "dlm.frozen_after" ]
184+
test_runner_features: [ "capabilities" ]
185+
reason: "Frozen after added"
186+
187+
- do:
188+
cluster.put_component_template:
189+
name: test-lifecycle
190+
body:
191+
template:
192+
lifecycle:
193+
data_retention: "10d"
194+
frozen_after: "30d"
195+
196+
- do:
197+
cluster.get_component_template:
198+
name: test-lifecycle
199+
200+
- match: {component_templates.0.name: test-lifecycle}
201+
- match: {component_templates.0.component_template.template.lifecycle.enabled: true}
202+
- match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"}
203+
- match: {component_templates.0.component_template.template.lifecycle.frozen_after: "30d"}
204+
177205
---
178206
"Get data stream lifecycle with default rollover":
179207
- requires:

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,3 +231,33 @@
231231
- match: {index_templates.0.index_template.template.lifecycle.downsampling_method: "last_value"}
232232
- match: {index_templates.0.index_template.template.lifecycle.downsampling.0.after: "1d"}
233233
- match: {index_templates.0.index_template.template.lifecycle.downsampling.0.fixed_interval: "5m"}
234+
235+
---
236+
"Add data stream lifecycle with frozen_after":
237+
- requires:
238+
capabilities:
239+
- method: PUT
240+
path: /_index_template/{name}
241+
capabilities: [ "dlm.frozen_after" ]
242+
test_runner_features: [ "capabilities" ]
243+
reason: "Frozen after added"
244+
245+
- do:
246+
indices.put_index_template:
247+
name: test-lifecycle
248+
body:
249+
index_patterns: frozen-*
250+
data_stream: {}
251+
template:
252+
lifecycle:
253+
data_retention: "10d"
254+
frozen_after: "30d"
255+
256+
- do:
257+
indices.get_index_template:
258+
name: test-lifecycle
259+
260+
- match: {index_templates.0.name: test-lifecycle}
261+
- match: {index_templates.0.index_template.template.lifecycle.enabled: true}
262+
- match: {index_templates.0.index_template.template.lifecycle.data_retention: "10d"}
263+
- match: {index_templates.0.index_template.template.lifecycle.frozen_after: "30d"}

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java‎

Lines changed: 95 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.common.logging.HeaderWarning;
2323
import org.elasticsearch.common.settings.Setting;
2424
import org.elasticsearch.common.settings.Settings;
25+
import org.elasticsearch.common.util.FeatureFlag;
2526
import org.elasticsearch.core.Nullable;
2627
import org.elasticsearch.core.TimeValue;
2728
import org.elasticsearch.core.Tuple;
@@ -53,9 +54,12 @@
5354
*/
5455
public class DataStreamLifecycle implements SimpleDiffable<DataStreamLifecycle>, ToXContentObject {
5556

57+
public static final FeatureFlag DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG = new FeatureFlag("dlm_searchable_snapshots");
58+
5659
// Versions over the wire
5760
public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X;
5861
public static final TransportVersion ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM = TransportVersion.fromName("add_sample_method_downsample_dlm");
62+
public static final TransportVersion SEARCHABLE_SNAPSHOTS_DLM_TV = TransportVersion.fromName("searchable_snapshots_dlm");
5963
public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention";
6064

6165
public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode";
@@ -105,6 +109,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
105109
public static final ParseField RETENTION_SOURCE_FIELD = new ParseField("retention_determined_by");
106110
public static final ParseField DOWNSAMPLING_FIELD = new ParseField("downsampling");
107111
public static final ParseField DOWNSAMPLING_METHOD_FIELD = new ParseField("downsampling_method");
112+
public static final ParseField FROZEN_AFTER_FIELD = new ParseField("frozen_after");
108113
private static final ParseField ROLLOVER_FIELD = new ParseField("rollover");
109114

110115
@SuppressWarnings("unchecked")
@@ -116,7 +121,8 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
116121
(Boolean) args[0],
117122
(TimeValue) args[1],
118123
(List<DownsamplingRound>) args[2],
119-
(DownsampleConfig.SamplingMethod) args[3]
124+
(DownsampleConfig.SamplingMethod) args[3],
125+
(TimeValue) args[4]
120126
)
121127
);
122128

@@ -146,6 +152,16 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
146152
DOWNSAMPLING_METHOD_FIELD,
147153
ObjectParser.ValueType.STRING
148154
);
155+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled()) {
156+
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
157+
String value = p.textOrNull();
158+
if (value == null) {
159+
return null;
160+
} else {
161+
return TimeValue.parseTimeValue(value, FROZEN_AFTER_FIELD.getPreferredName());
162+
}
163+
}, FROZEN_AFTER_FIELD, ObjectParser.ValueType.STRING_OR_NULL);
164+
}
149165
}
150166

151167
private static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = TransportVersion.fromName("introduce_failures_lifecycle");
@@ -158,6 +174,8 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
158174
private final List<DownsamplingRound> downsamplingRounds;
159175
@Nullable
160176
private final DownsampleConfig.SamplingMethod downsamplingMethod;
177+
@Nullable
178+
private final TimeValue frozenAfter;
161179

162180
/**
163181
* This constructor is visible for testing, please use {@link DataStreamLifecycle#dataLifecycleBuilder()} or
@@ -168,7 +186,8 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
168186
@Nullable Boolean enabled,
169187
@Nullable TimeValue dataRetention,
170188
@Nullable List<DownsamplingRound> downsamplingRounds,
171-
@Nullable DownsampleConfig.SamplingMethod downsamplingMethod
189+
@Nullable DownsampleConfig.SamplingMethod downsamplingMethod,
190+
@Nullable TimeValue frozenAfter
172191
) {
173192
this.lifecycleType = lifecycleType;
174193
this.enabled = enabled == null || enabled;
@@ -182,6 +201,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) {
182201
throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR);
183202
}
184203
this.downsamplingMethod = downsamplingMethod;
204+
this.frozenAfter = frozenAfter;
185205
}
186206

187207
/**
@@ -315,6 +335,11 @@ public DownsampleConfig.SamplingMethod downsamplingMethod() {
315335
return downsamplingMethod;
316336
}
317337

338+
@Nullable
339+
public TimeValue frozenAfter() {
340+
return frozenAfter;
341+
}
342+
318343
@Override
319344
public boolean equals(Object o) {
320345
if (this == o) return true;
@@ -325,12 +350,13 @@ public boolean equals(Object o) {
325350
&& Objects.equals(dataRetention, that.dataRetention)
326351
&& Objects.equals(downsamplingRounds, that.downsamplingRounds)
327352
&& Objects.equals(downsamplingMethod, that.downsamplingMethod)
353+
&& Objects.equals(frozenAfter, that.frozenAfter)
328354
&& enabled == that.enabled;
329355
}
330356

331357
@Override
332358
public int hashCode() {
333-
return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod);
359+
return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod, frozenAfter);
334360
}
335361

336362
@Override
@@ -357,6 +383,9 @@ public void writeTo(StreamOutput out) throws IOException {
357383
if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) {
358384
out.writeOptionalWriteable(downsamplingMethod);
359385
}
386+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled() && out.getTransportVersion().supports(SEARCHABLE_SNAPSHOTS_DLM_TV)) {
387+
out.writeOptionalTimeValue(frozenAfter);
388+
}
360389
}
361390

362391
public DataStreamLifecycle(StreamInput in) throws IOException {
@@ -384,6 +413,9 @@ public DataStreamLifecycle(StreamInput in) throws IOException {
384413
downsamplingMethod = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)
385414
? in.readOptionalWriteable(DownsampleConfig.SamplingMethod::read)
386415
: null;
416+
frozenAfter = DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled() && in.getTransportVersion().supports(SEARCHABLE_SNAPSHOTS_DLM_TV)
417+
? in.readOptionalTimeValue()
418+
: null;
387419
}
388420

389421
/**
@@ -429,18 +461,7 @@ public static Diff<DataStreamLifecycle> readDiffFrom(StreamInput in) throws IOEx
429461

430462
@Override
431463
public String toString() {
432-
return "DataStreamLifecycle{"
433-
+ "lifecycleTarget="
434-
+ lifecycleType
435-
+ ", enabled="
436-
+ enabled
437-
+ ", dataRetention="
438-
+ dataRetention
439-
+ ", downsamplingRounds="
440-
+ downsamplingRounds
441-
+ ", downsamplingMethod="
442-
+ downsamplingMethod
443-
+ '}';
464+
return Strings.toString(this);
444465
}
445466

446467
@Override
@@ -484,6 +505,9 @@ public XContentBuilder toXContent(
484505
if (downsamplingMethod != null) {
485506
builder.field(DOWNSAMPLING_METHOD_FIELD.getPreferredName(), downsamplingMethod.toString());
486507
}
508+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled() && frozenAfter != null) {
509+
builder.field(FROZEN_AFTER_FIELD.getPreferredName(), frozenAfter.getStringRep());
510+
}
487511
if (rolloverConfiguration != null) {
488512
builder.field(ROLLOVER_FIELD.getPreferredName());
489513
rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1());
@@ -651,7 +675,8 @@ public record Template(
651675
boolean enabled,
652676
ResettableValue<TimeValue> dataRetention,
653677
ResettableValue<List<DataStreamLifecycle.DownsamplingRound>> downsamplingRounds,
654-
ResettableValue<DownsampleConfig.SamplingMethod> downsamplingMethod
678+
ResettableValue<DownsampleConfig.SamplingMethod> downsamplingMethod,
679+
ResettableValue<TimeValue> frozenAfter
655680
) implements ToXContentObject, Writeable {
656681

657682
public Template {
@@ -676,7 +701,8 @@ public record Template(
676701
args[0] == null || (boolean) args[0],
677702
args[1] == null ? ResettableValue.undefined() : (ResettableValue<TimeValue>) args[1],
678703
args[2] == null ? ResettableValue.undefined() : (ResettableValue<List<DataStreamLifecycle.DownsamplingRound>>) args[2],
679-
args[3] == null ? ResettableValue.undefined() : (ResettableValue<DownsampleConfig.SamplingMethod>) args[3]
704+
args[3] == null ? ResettableValue.undefined() : (ResettableValue<DownsampleConfig.SamplingMethod>) args[3],
705+
args[4] == null ? ResettableValue.undefined() : (ResettableValue<TimeValue>) args[4]
680706
)
681707
);
682708

@@ -699,6 +725,14 @@ public record Template(
699725
String value = p.textOrNull();
700726
return value == null ? ResettableValue.reset() : ResettableValue.create(DownsampleConfig.SamplingMethod.fromString(value));
701727
}, DOWNSAMPLING_METHOD_FIELD, ObjectParser.ValueType.STRING_OR_NULL);
728+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled()) {
729+
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
730+
String value = p.textOrNull();
731+
return value == null
732+
? ResettableValue.reset()
733+
: ResettableValue.create(TimeValue.parseTimeValue(value, FROZEN_AFTER_FIELD.getPreferredName()));
734+
}, FROZEN_AFTER_FIELD, ObjectParser.ValueType.STRING_OR_NULL);
735+
}
702736
}
703737

704738
@Override
@@ -725,6 +759,9 @@ public void writeTo(StreamOutput out) throws IOException {
725759
if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) {
726760
ResettableValue.write(out, downsamplingMethod, StreamOutput::writeWriteable);
727761
}
762+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled() && out.getTransportVersion().supports(SEARCHABLE_SNAPSHOTS_DLM_TV)) {
763+
ResettableValue.write(out, frozenAfter, StreamOutput::writeTimeValue);
764+
}
728765
}
729766

730767
/**
@@ -789,7 +826,11 @@ public static Template read(StreamInput in) throws IOException {
789826
.supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)
790827
? ResettableValue.read(in, DownsampleConfig.SamplingMethod::read)
791828
: ResettableValue.undefined();
792-
return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod);
829+
ResettableValue<TimeValue> frozenAfter = DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled()
830+
&& in.getTransportVersion().supports(SEARCHABLE_SNAPSHOTS_DLM_TV)
831+
? ResettableValue.read(in, StreamInput::readTimeValue)
832+
: ResettableValue.undefined();
833+
return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod, frozenAfter);
793834
}
794835

795836
public static Template dataLifecycleTemplatefromXContent(XContentParser parser) throws IOException {
@@ -830,6 +871,9 @@ public XContentBuilder toXContent(
830871
DOWNSAMPLING_METHOD_FIELD.getPreferredName(),
831872
DownsampleConfig.SamplingMethod::toString
832873
);
874+
if (DLM_SEARCHABLE_SNAPSHOTS_FEATURE_FLAG.isEnabled()) {
875+
frozenAfter.toXContent(builder, params, FROZEN_AFTER_FIELD.getPreferredName(), TimeValue::getStringRep);
876+
}
833877
if (rolloverConfiguration != null) {
834878
builder.field(ROLLOVER_FIELD.getPreferredName());
835879
rolloverConfiguration.evaluateAndConvertToXContent(
@@ -843,7 +887,14 @@ public XContentBuilder toXContent(
843887
}
844888

845889
public DataStreamLifecycle toDataStreamLifecycle() {
846-
return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get(), downsamplingMethod.get());
890+
return new DataStreamLifecycle(
891+
lifecycleType,
892+
enabled,
893+
dataRetention.get(),
894+
downsamplingRounds.get(),
895+
downsamplingMethod.get(),
896+
frozenAfter.get()
897+
);
847898
}
848899
}
849900

@@ -880,6 +931,7 @@ public static class Builder {
880931
private ResettableValue<TimeValue> dataRetention = ResettableValue.undefined();
881932
private ResettableValue<List<DownsamplingRound>> downsamplingRounds = ResettableValue.undefined();
882933
private ResettableValue<DownsampleConfig.SamplingMethod> downsamplingMethod = ResettableValue.undefined();
934+
private ResettableValue<TimeValue> frozenAfter = ResettableValue.undefined();
883935

884936
private Builder(LifecycleType lifecycleType) {
885937
this.lifecycleType = lifecycleType;
@@ -891,6 +943,7 @@ private Builder(DataStreamLifecycle.Template template) {
891943
dataRetention = template.dataRetention();
892944
downsamplingRounds = template.downsamplingRounds();
893945
downsamplingMethod = template.downsamplingMethod();
946+
frozenAfter = template.frozenAfter();
894947
}
895948

896949
private Builder(DataStreamLifecycle lifecycle) {
@@ -899,6 +952,7 @@ private Builder(DataStreamLifecycle lifecycle) {
899952
dataRetention = ResettableValue.create(lifecycle.dataRetention());
900953
downsamplingRounds = ResettableValue.create(lifecycle.downsamplingRounds());
901954
downsamplingMethod = ResettableValue.create(lifecycle.downsamplingMethod());
955+
frozenAfter = ResettableValue.create(lifecycle.frozenAfter);
902956
}
903957

904958
public Builder composeTemplate(DataStreamLifecycle.Template template) {
@@ -907,6 +961,7 @@ public Builder composeTemplate(DataStreamLifecycle.Template template) {
907961
dataRetention(template.dataRetention());
908962
downsamplingRounds(template.downsamplingRounds());
909963
downsamplingMethod(template.downsamplingMethod());
964+
frozenAfter(template.frozenAfter());
910965
return this;
911966
}
912967

@@ -951,12 +1006,31 @@ public Builder downsamplingMethod(@Nullable DownsampleConfig.SamplingMethod down
9511006
return this;
9521007
}
9531008

1009+
public Builder frozenAfter(ResettableValue<TimeValue> frozenAfter) {
1010+
if (frozenAfter.isDefined()) {
1011+
this.frozenAfter = frozenAfter;
1012+
}
1013+
return this;
1014+
}
1015+
1016+
public Builder frozenAfter(@Nullable TimeValue frozenAfter) {
1017+
this.frozenAfter = ResettableValue.create(frozenAfter);
1018+
return this;
1019+
}
1020+
9541021
public DataStreamLifecycle build() {
955-
return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get(), downsamplingMethod.get());
1022+
return new DataStreamLifecycle(
1023+
lifecycleType,
1024+
enabled,
1025+
dataRetention.get(),
1026+
downsamplingRounds.get(),
1027+
downsamplingMethod.get(),
1028+
frozenAfter.get()
1029+
);
9561030
}
9571031

9581032
public Template buildTemplate() {
959-
return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod);
1033+
return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod, frozenAfter);
9601034
}
9611035
}
9621036

0 commit comments

Comments
 (0)