(tree) Added new forest data and summary formats for incremental summaries

packages/dds/tree/src/codec/codec.ts

@@ -465,8 +465,14 @@ export function withSchemaValidation<
  * If the need arises, they might be added in the future.
  *
  * @privateRemarks
- * Entries in these enums should document the user facing impact of opting into a particular version.
- * For example, document if there is an encoding efficiency improvement of oping into that version or newer.
+ * The entries in these enums should document the following:
+ * - The user facing impact of opting into a particular version. This will help customers decide if they want to opt into
+ * a new version. For example, document if there is an encoding efficiency improvement of oping into that version or newer.
+ * - Any new data formats that are introduced in that version. This will help developers tell which data formats a given
+ * version will write. For example, document if a new summary or encoding format is added in a version.
+ * - Whether the above features or data formats introduced in a version are enabled by default or require the
+ * {@link minVersionForCollab} option to be set to that particular version.
+ *
  * Versions with no notable impact can be omitted.
  *
  * This scheme assumes a single version will always be enough to communicate compatibility.
@@ -505,38 +511,40 @@ export const FluidClientVersion = {
 	 * Fluid Framework Client 2.43 and newer.
 	 * @remarks
 	 * New formats introduced in 2.43:
-	 * - SchemaFormatVersion.v2
-	 * - MessageFormatVersion.v4
-	 * - EditManagerFormatVersion.v4
-	 * - Sequence format version 3
+	 * - SchemaFormatVersion.v2 - written when minVersionForCollab \>= 2.43
+	 * - MessageFormatVersion.v4 - written when minVersionForCollab \>= 2.43
+	 * - EditManagerFormatVersion.v4 - written when minVersionForCollab \>= 2.43
+	 * - sequence-field/formatV3 - written when minVersionForCollab \>= 2.43
 	 */
 	v2_43: "2.43.0",
 
 	/**
 	 * Fluid Framework Client 2.52 and newer.
 	 * @remarks
 	 * New formats introduced in 2.52:
-	 * - DetachedFieldIndexFormatVersion.v2
+	 * - DetachedFieldIndexFormatVersion.v2 - written when minVersionForCollab \>= 2.52
 	 */
 	v2_52: "2.52.0",
 
 	/**
 	 * Fluid Framework Client 2.73 and newer.
 	 * @remarks
 	 * New formats introduced in 2.73:
-	 * - FieldBatchFormatVersion v2
+	 * - FieldBatchFormatVersion.v2 - written when minVersionForCollab \>= 2.73
 	 */
 	v2_73: "2.73.0",
 
 	/**
 	 * Fluid Framework Client 2.74 and newer.
 	 * @remarks
 	 * New formats introduced in 2.74:
-	 * - SharedTreeSummaryFormatVersion v2
-	 * - DetachedFieldIndexSummaryFormatVersion v2
-	 * - SchemaSummaryFormatVersion v2
-	 * - EditManagerSummaryFormatVersion v2
-	 * - ForestSummaryFormatVersion v2
+	 * - SharedTreeSummaryFormatVersion.v2 - written by default
+	 * - DetachedFieldIndexSummaryFormatVersion.v2 - written by default
+	 * - SchemaSummaryFormatVersion.v2 - written by default
+	 * - EditManagerSummaryFormatVersion.v2 - written by default
+	 * - ForestSummaryFormatVersion.v2 - written by default
+	 * - ForestFormatVersion.v2 - written when minVersionForCollab \>= 2.74
+	 * - ForestSummaryFormatVersion.v3 - written when minVersionForCollab \>= 2.74
 	 */
 	v2_74: "2.74.0",
 } as const satisfies Record<string, MinimumVersionForCollab>;

packages/dds/tree/src/feature-libraries/forest-summary/codec.ts

@@ -5,48 +5,60 @@
 
 import { assert, oob } from "@fluidframework/core-utils/internal";
 import type { MinimumVersionForCollab } from "@fluidframework/runtime-definitions/internal";
+import {
+	getConfigForMinVersionForCollab,
+	lowestMinVersionForCollab,
+} from "@fluidframework/runtime-utils/internal";
 
 import {
 	type CodecTree,
 	type CodecWriteOptions,
+	FluidClientVersion,
 	type IJsonCodec,
 	makeVersionedValidatedCodec,
 } from "../../codec/index.js";
 import type { FieldKey, ITreeCursorSynchronous } from "../../core/index.js";
 import type { FieldBatchCodec, FieldBatchEncodingContext } from "../chunked-forest/index.js";
 
-import { FormatV1, ForestFormatVersion } from "./format.js";
+import {
+	ForestFormatVersion,
+	validVersions,
+	type Format,
+	FormatCommon,
+} from "./formatCommon.js";
 import { brand } from "../../util/index.js";
 
 /**
  * Uses field cursors
  */
 export type FieldSet = ReadonlyMap<FieldKey, ITreeCursorSynchronous>;
-export type ForestCodec = IJsonCodec<FieldSet, FormatV1, FormatV1, FieldBatchEncodingContext>;
+export type ForestCodec = IJsonCodec<FieldSet, Format, Format, FieldBatchEncodingContext>;
 
 /**
  * Convert a MinimumVersionForCollab to a ForestFormatVersion.
  * @param clientVersion - The MinimumVersionForCollab to convert.
  * @returns The ForestFormatVersion that corresponds to the provided MinimumVersionForCollab.
  */
-function clientVersionToForestSummaryFormatVersion(
+export function clientVersionToForestFormatVersion(
 	clientVersion: MinimumVersionForCollab,
 ): ForestFormatVersion {
-	// Currently, forest summary codec only writes in version 1.
-	return brand(ForestFormatVersion.v1);
+	return brand(
+		getConfigForMinVersionForCollab(clientVersion, {
+			[lowestMinVersionForCollab]: ForestFormatVersion.v1,
+			[FluidClientVersion.v2_74]: ForestFormatVersion.v2,
+		}),
+	);
 }
 
 export function makeForestSummarizerCodec(
 	options: CodecWriteOptions,
 	fieldBatchCodec: FieldBatchCodec,
 ): ForestCodec {
 	const inner = fieldBatchCodec;
-	// TODO: AB#41865
-	// This needs to be updated to support multiple versions.
-	// The second version will be used to enable incremental summarization.
-	const writeVersion = clientVersionToForestSummaryFormatVersion(options.minVersionForCollab);
-	return makeVersionedValidatedCodec(options, new Set([ForestFormatVersion.v1]), FormatV1, {
-		encode: (data: FieldSet, context: FieldBatchEncodingContext): FormatV1 => {
+	const writeVersion = clientVersionToForestFormatVersion(options.minVersionForCollab);
+	const formatSchema = FormatCommon(writeVersion);
+	return makeVersionedValidatedCodec(options, validVersions, formatSchema, {
+		encode: (data: FieldSet, context: FieldBatchEncodingContext): Format => {
 			const keys: FieldKey[] = [];
 			const fields: ITreeCursorSynchronous[] = [];
 			for (const [key, value] of data) {
@@ -55,7 +67,7 @@ export function makeForestSummarizerCodec(
 			}
 			return { keys, fields: inner.encode(fields, context), version: writeVersion };
 		},
-		decode: (data: FormatV1, context: FieldBatchEncodingContext): FieldSet => {
+		decode: (data: Format, context: FieldBatchEncodingContext): FieldSet => {
 			const out: Map<FieldKey, ITreeCursorSynchronous> = new Map();
 			const fields = inner.decode(data.fields, context);
 			assert(data.keys.length === fields.length, 0x891 /* mismatched lengths */);
@@ -70,5 +82,5 @@ export function makeForestSummarizerCodec(
 export function getCodecTreeForForestFormat(
 	clientVersion: MinimumVersionForCollab,
 ): CodecTree {
-	return { name: "Forest", version: clientVersionToForestSummaryFormatVersion(clientVersion) };
+	return { name: "Forest", version: clientVersionToForestFormatVersion(clientVersion) };
 }

packages/dds/tree/src/feature-libraries/forest-summary/forestSummarizer.ts

@@ -42,19 +42,26 @@ import {
 	type IncrementalEncodingPolicy,
 } from "../chunked-forest/index.js";
 
-import { type ForestCodec, makeForestSummarizerCodec } from "./codec.js";
+import {
+	clientVersionToForestFormatVersion,
+	type ForestCodec,
+	makeForestSummarizerCodec,
+} from "./codec.js";
 import {
 	ForestIncrementalSummaryBehavior,
 	ForestIncrementalSummaryBuilder,
 } from "./incrementalSummaryBuilder.js";
 import {
-	forestSummaryContentKey,
-	forestSummaryKey,
 	minVersionToForestSummaryFormatVersion,
-	supportedForestSummaryFormatVersions,
-	type ForestSummaryFormatVersion,
+	getForestRootSummaryContentKey,
 } from "./summaryTypes.js";
 import { TreeCompressionStrategy } from "../treeCompressionUtils.js";
+import { ForestFormatVersion } from "./formatCommon.js";
+import {
+	ForestSummaryFormatVersion,
+	forestSummaryKey,
+	supportedForestSummaryFormatVersions,
+} from "./summaryFormatCommon.js";
 
 /**
  * Provides methods for summarizing and loading a forest.
@@ -66,6 +73,7 @@ export class ForestSummarizer
 	private readonly codec: ForestCodec;
 
 	private readonly incrementalSummaryBuilder: ForestIncrementalSummaryBuilder;
+	private readonly forestRootSummaryContentKey: string;
 
 	/**
 	 * @param encoderContext - The schema if provided here must be mutated by the caller to keep it up to date.
@@ -87,11 +95,25 @@ export class ForestSummarizer
 			true /* supportPreVersioningFormat */,
 		);
 
-		// TODO: this should take in CodecWriteOptions, and use it to pick the write version.
 		this.codec = makeForestSummarizerCodec(options, fieldBatchCodec);
+
+		const forestFormatWriteVersion = clientVersionToForestFormatVersion(
+			options.minVersionForCollab,
+		);
+		const summaryFormatWriteVersion = minVersionToForestSummaryFormatVersion(
+			options.minVersionForCollab,
+		);
+		this.forestRootSummaryContentKey = getForestRootSummaryContentKey(
+			summaryFormatWriteVersion,
+		);
+
+		// Incremental summary is supported from ForestFormatVersion.v2 and ForestSummaryFormatVersion.v3 onwards.
+		const enableIncrementalSummary =
+			forestFormatWriteVersion >= ForestFormatVersion.v2 &&
+			summaryFormatWriteVersion >= ForestSummaryFormatVersion.v3 &&
+			encoderContext.encodeType === TreeCompressionStrategy.CompressedIncremental;
 		this.incrementalSummaryBuilder = new ForestIncrementalSummaryBuilder(
-			encoderContext.encodeType ===
-				TreeCompressionStrategy.CompressedIncremental /* enableIncrementalSummary */,
+			enableIncrementalSummary,
 			(cursor: ITreeCursorSynchronous) => this.forest.chunkField(cursor),
 			shouldEncodeIncrementally,
 			initialSequenceNumber,
@@ -153,23 +175,26 @@ export class ForestSummarizer
 
 		this.incrementalSummaryBuilder.completeSummary({
 			incrementalSummaryContext,
-			forestSummaryContent: stringify(encoded),
+			forestSummaryRootContent: stringify(encoded),
+			forestSummaryRootContentKey: this.forestRootSummaryContentKey,
 			builder,
 		});
 	}
 
 	protected async loadInternal(
 		services: IChannelStorageService,
 		parse: SummaryElementParser,
+		version: ForestSummaryFormatVersion | undefined,
 	): Promise<void> {
-		// The contents of the top-level forest must be present under a summary blob named `forestSummaryContentKey`.
+		// Get the key of the summary blob where the top-level forest content is stored based on the summary format version.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.SingleBlob`, this blob will contain all
 		// of forest's contents.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.Incremental`, this blob will contain only
 		// the top-level forest node's contents.
 		// The contents of the incremental chunks will be in separate tree nodes and will be read later during decoding.
+		const forestSummaryRootContentKey = getForestRootSummaryContentKey(version);
 		assert(
-			await services.contains(forestSummaryContentKey),
+			await services.contains(forestSummaryRootContentKey),
 			0xc21 /* Forest summary content missing in snapshot */,
 		);
 
@@ -184,7 +209,7 @@ export class ForestSummarizer
 		// TODO: this code is parsing data without an optional validator, this should be defined in a typebox schema as part of the
 		// forest summary format.
 		const fields = this.codec.decode(
-			await readAndParseSnapshotBlob(forestSummaryContentKey, services, parse),
+			await readAndParseSnapshotBlob(forestSummaryRootContentKey, services, parse),
 			{
 				...this.encoderContext,
 				incrementalEncoderDecoder: this.incrementalSummaryBuilder,

packages/dds/tree/src/feature-libraries/forest-summary/format.ts → packages/dds/tree/src/feature-libraries/forest-summary/formatCommon.ts

@@ -6,21 +6,25 @@
 import { type Static, Type } from "@sinclair/typebox";
 
 import { schemaFormatV1 } from "../../core/index.js";
-import { brand, type Brand } from "../../util/index.js";
+import type { Brand } from "../../util/index.js";
 import { EncodedFieldBatch } from "../chunked-forest/index.js";
 
 /**
  * The format version for the forest.
  */
 export const ForestFormatVersion = {
 	v1: 1,
+	/** This format supports incremental encoding */
+	v2: 2,
 } as const;
 export type ForestFormatVersion = Brand<
 	(typeof ForestFormatVersion)[keyof typeof ForestFormatVersion],
 	"ForestFormatVersion"
 >;
 
-const FormatGeneric = (
+export const validVersions = new Set([...Object.values(ForestFormatVersion)]);
+
+export const FormatCommon = (
 	version: ForestFormatVersion,
 	// Return type is intentionally derived.
 	// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
@@ -33,6 +37,4 @@ const FormatGeneric = (
 		},
 		{ additionalProperties: false },
 	);
-
-export const FormatV1 = FormatGeneric(brand<ForestFormatVersion>(ForestFormatVersion.v1));
-export type FormatV1 = Static<typeof FormatV1>;
+export type Format = Static<ReturnType<typeof FormatCommon>>;

packages/dds/tree/src/feature-libraries/forest-summary/formatV1.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { Static } from "@sinclair/typebox";
+
+import { brand } from "../../util/index.js";
+import { FormatCommon, ForestFormatVersion } from "./formatCommon.js";
+
+export const FormatV1 = FormatCommon(brand<ForestFormatVersion>(ForestFormatVersion.v1));
+export type FormatV1 = Static<typeof FormatV1>;

packages/dds/tree/src/feature-libraries/forest-summary/formatV2.ts

@@ -0,0 +1,12 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { Static } from "@sinclair/typebox";
+
+import { brand } from "../../util/index.js";
+import { FormatCommon, ForestFormatVersion } from "./formatCommon.js";
+
+export const FormatV2 = FormatCommon(brand<ForestFormatVersion>(ForestFormatVersion.v2));
+export type FormatV2 = Static<typeof FormatV2>;