(tree) Added new forest data and summary formats for incremental summaries

packages/dds/tree/src/codec/codec.ts

@@ -532,11 +532,8 @@ export const FluidClientVersion = {
 	 * Fluid Framework Client 2.74 and newer.
 	 * @remarks
 	 * New formats introduced in 2.74:
-	 * - SharedTreeSummaryFormatVersion v2
-	 * - DetachedFieldIndexSummaryFormatVersion v2
-	 * - SchemaSummaryFormatVersion v2
-	 * - EditManagerSummaryFormatVersion v2
-	 * - ForestSummaryFormatVersion v2
+	 * - ForestFormatVersion v2
+	 * - ForestSummaryFormatVersion v3
 	 */
 	v2_74: "2.74.0",
 } as const satisfies Record<string, MinimumVersionForCollab>;

packages/dds/tree/src/feature-libraries/forest-summary/codec.ts

@@ -5,48 +5,55 @@
 
 import { assert, oob } from "@fluidframework/core-utils/internal";
 import type { MinimumVersionForCollab } from "@fluidframework/runtime-definitions/internal";
+import {
+	getConfigForMinVersionForCollab,
+	lowestMinVersionForCollab,
+} from "@fluidframework/runtime-utils/internal";
 
 import {
 	type CodecTree,
 	type CodecWriteOptions,
+	FluidClientVersion,
 	type IJsonCodec,
 	makeVersionedValidatedCodec,
 } from "../../codec/index.js";
 import type { FieldKey, ITreeCursorSynchronous } from "../../core/index.js";
 import type { FieldBatchCodec, FieldBatchEncodingContext } from "../chunked-forest/index.js";
 
-import { FormatV1, ForestFormatVersion } from "./format.js";
+import { ForestFormatVersion, validVersions, type Format, FormatGeneric } from "./format.js";
 import { brand } from "../../util/index.js";
 
 /**
  * Uses field cursors
  */
 export type FieldSet = ReadonlyMap<FieldKey, ITreeCursorSynchronous>;
-export type ForestCodec = IJsonCodec<FieldSet, FormatV1, FormatV1, FieldBatchEncodingContext>;
+export type ForestCodec = IJsonCodec<FieldSet, Format, Format, FieldBatchEncodingContext>;
 
 /**
  * Convert a MinimumVersionForCollab to a ForestFormatVersion.
  * @param clientVersion - The MinimumVersionForCollab to convert.
  * @returns The ForestFormatVersion that corresponds to the provided MinimumVersionForCollab.
  */
-function clientVersionToForestSummaryFormatVersion(
+export function clientVersionToForestFormatVersion(
 	clientVersion: MinimumVersionForCollab,
 ): ForestFormatVersion {
-	// Currently, forest summary codec only writes in version 1.
-	return brand(ForestFormatVersion.v1);
+	return brand(
+		getConfigForMinVersionForCollab(clientVersion, {
+			[lowestMinVersionForCollab]: ForestFormatVersion.v1,
+			[FluidClientVersion.v2_74]: ForestFormatVersion.v2,
+		}),
+	);
 }
 
 export function makeForestSummarizerCodec(
 	options: CodecWriteOptions,
 	fieldBatchCodec: FieldBatchCodec,
 ): ForestCodec {
 	const inner = fieldBatchCodec;
-	// TODO: AB#41865
-	// This needs to be updated to support multiple versions.
-	// The second version will be used to enable incremental summarization.
-	const writeVersion = clientVersionToForestSummaryFormatVersion(options.minVersionForCollab);
-	return makeVersionedValidatedCodec(options, new Set([ForestFormatVersion.v1]), FormatV1, {
-		encode: (data: FieldSet, context: FieldBatchEncodingContext): FormatV1 => {
+	const writeVersion = clientVersionToForestFormatVersion(options.minVersionForCollab);
+	const formatSchema = FormatGeneric(writeVersion);
+	return makeVersionedValidatedCodec(options, validVersions, formatSchema, {
+		encode: (data: FieldSet, context: FieldBatchEncodingContext): Format => {
 			const keys: FieldKey[] = [];
 			const fields: ITreeCursorSynchronous[] = [];
 			for (const [key, value] of data) {
@@ -55,7 +62,7 @@ export function makeForestSummarizerCodec(
 			}
 			return { keys, fields: inner.encode(fields, context), version: writeVersion };
 		},
-		decode: (data: FormatV1, context: FieldBatchEncodingContext): FieldSet => {
+		decode: (data: Format, context: FieldBatchEncodingContext): FieldSet => {
 			const out: Map<FieldKey, ITreeCursorSynchronous> = new Map();
 			const fields = inner.decode(data.fields, context);
 			assert(data.keys.length === fields.length, 0x891 /* mismatched lengths */);
@@ -70,5 +77,5 @@ export function makeForestSummarizerCodec(
 export function getCodecTreeForForestFormat(
 	clientVersion: MinimumVersionForCollab,
 ): CodecTree {
-	return { name: "Forest", version: clientVersionToForestSummaryFormatVersion(clientVersion) };
+	return { name: "Forest", version: clientVersionToForestFormatVersion(clientVersion) };
 }

packages/dds/tree/src/feature-libraries/forest-summary/forestSummarizer.ts

@@ -42,19 +42,24 @@ import {
 	type IncrementalEncodingPolicy,
 } from "../chunked-forest/index.js";
 
-import { type ForestCodec, makeForestSummarizerCodec } from "./codec.js";
+import {
+	clientVersionToForestFormatVersion,
+	type ForestCodec,
+	makeForestSummarizerCodec,
+} from "./codec.js";
 import {
 	ForestIncrementalSummaryBehavior,
 	ForestIncrementalSummaryBuilder,
 } from "./incrementalSummaryBuilder.js";
 import {
-	forestSummaryContentKey,
+	ForestSummaryFormatVersion,
 	forestSummaryKey,
 	minVersionToForestSummaryFormatVersion,
 	supportedForestSummaryFormatVersions,
-	type ForestSummaryFormatVersion,
+	getForestRootSummaryContentKey,
 } from "./summaryTypes.js";
 import { TreeCompressionStrategy } from "../treeCompressionUtils.js";
+import { ForestFormatVersion } from "./format.js";
 
 /**
  * Provides methods for summarizing and loading a forest.
@@ -66,6 +71,7 @@ export class ForestSummarizer
 	private readonly codec: ForestCodec;
 
 	private readonly incrementalSummaryBuilder: ForestIncrementalSummaryBuilder;
+	private readonly summaryFormatWriteVersion: ForestSummaryFormatVersion;
 
 	/**
 	 * @param encoderContext - The schema if provided here must be mutated by the caller to keep it up to date.
@@ -87,11 +93,21 @@ export class ForestSummarizer
 			true /* supportPreVersioningFormat */,
 		);
 
-		// TODO: this should take in CodecWriteOptions, and use it to pick the write version.
 		this.codec = makeForestSummarizerCodec(options, fieldBatchCodec);
+
+		this.summaryFormatWriteVersion = minVersionToForestSummaryFormatVersion(
+			options.minVersionForCollab,
+		);
+		const forestFormatWriteVersion = clientVersionToForestFormatVersion(
+			options.minVersionForCollab,
+		);
+		// Incremental summary is supported from ForestFormatVersion.v2 and ForestSummaryFormatVersion.v3 onwards.
+		const enableIncrementalSummary =
+			forestFormatWriteVersion >= ForestFormatVersion.v2 &&
+			this.summaryFormatWriteVersion >= ForestSummaryFormatVersion.v3 &&
+			encoderContext.encodeType === TreeCompressionStrategy.CompressedIncremental;
 		this.incrementalSummaryBuilder = new ForestIncrementalSummaryBuilder(
-			encoderContext.encodeType ===
-				TreeCompressionStrategy.CompressedIncremental /* enableIncrementalSummary */,
+			enableIncrementalSummary,
 			(cursor: ITreeCursorSynchronous) => this.forest.chunkField(cursor),
 			shouldEncodeIncrementally,
 			initialSequenceNumber,
@@ -153,23 +169,28 @@ export class ForestSummarizer
 
 		this.incrementalSummaryBuilder.completeSummary({
 			incrementalSummaryContext,
-			forestSummaryContent: stringify(encoded),
+			forestSummaryRootContent: stringify(encoded),
+			forestSummaryRootContentKey: getForestRootSummaryContentKey(
+				this.summaryFormatWriteVersion,
+			),
 			builder,
 		});
 	}
 
 	protected async loadInternal(
 		services: IChannelStorageService,
 		parse: SummaryElementParser,
+		version: ForestSummaryFormatVersion | undefined,
 	): Promise<void> {
-		// The contents of the top-level forest must be present under a summary blob named `forestSummaryContentKey`.
+		// Get the key of the summary blob where the top-level forest content based on the summary format version.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.SingleBlob`, this blob will contain all
 		// of forest's contents.
 		// If the summary was generated as `ForestIncrementalSummaryBehavior.Incremental`, this blob will contain only
 		// the top-level forest node's contents.
 		// The contents of the incremental chunks will be in separate tree nodes and will be read later during decoding.
+		const forestSummaryRootContentKey = getForestRootSummaryContentKey(version);
 		assert(
-			await services.contains(forestSummaryContentKey),
+			await services.contains(forestSummaryRootContentKey),
 			0xc21 /* Forest summary content missing in snapshot */,
 		);
 
@@ -184,7 +205,7 @@ export class ForestSummarizer
 		// TODO: this code is parsing data without an optional validator, this should be defined in a typebox schema as part of the
 		// forest summary format.
 		const fields = this.codec.decode(
-			await readAndParseSnapshotBlob(forestSummaryContentKey, services, parse),
+			await readAndParseSnapshotBlob(forestSummaryRootContentKey, services, parse),
 			{
 				...this.encoderContext,
 				incrementalEncoderDecoder: this.incrementalSummaryBuilder,

packages/dds/tree/src/feature-libraries/forest-summary/format.ts

@@ -14,13 +14,17 @@ import { EncodedFieldBatch } from "../chunked-forest/index.js";
  */
 export const ForestFormatVersion = {
 	v1: 1,
+	/** This format supports incremental encoding */
+	v2: 2,
 } as const;
 export type ForestFormatVersion = Brand<
 	(typeof ForestFormatVersion)[keyof typeof ForestFormatVersion],
 	"ForestFormatVersion"
 >;
 
-const FormatGeneric = (
+export const validVersions = new Set([...Object.values(ForestFormatVersion)]);
+
+export const FormatGeneric = (
 	version: ForestFormatVersion,
 	// Return type is intentionally derived.
 	// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
@@ -33,6 +37,10 @@ const FormatGeneric = (
 		},
 		{ additionalProperties: false },
 	);
+export type Format = Static<ReturnType<typeof FormatGeneric>>;
 
 export const FormatV1 = FormatGeneric(brand<ForestFormatVersion>(ForestFormatVersion.v1));
 export type FormatV1 = Static<typeof FormatV1>;
+
+export const FormatV2 = FormatGeneric(brand<ForestFormatVersion>(ForestFormatVersion.v2));
+export type FormatV2 = Static<typeof FormatV2>;

packages/dds/tree/src/feature-libraries/forest-summary/incrementalSummaryBuilder.ts

@@ -27,7 +27,7 @@ import type { ISnapshotTree } from "@fluidframework/driver-definitions/internal"
 import { LoggingError } from "@fluidframework/telemetry-utils/internal";
 import type { IFluidHandle } from "@fluidframework/core-interfaces";
 import type { SummaryElementStringifier } from "../../shared-tree-core/index.js";
-import { chunkContentsBlobKey, forestSummaryContentKey } from "./summaryTypes.js";
+import { summaryContentBlobKeyV3 } from "./summaryTypes.js";
 
 /**
  * State that tells whether a summary is currently being tracked.
@@ -189,7 +189,7 @@ function validateReadyToTrackSummary(
  *
  * An example summary tree with incremental summary:
  *     Forest
- *     ├── ForestTree
+ *     ├── contents
  *     ├── 0
  *     |   ├── contents
  *     |   ├── 1
@@ -202,24 +202,14 @@ function validateReadyToTrackSummary(
  *     |   ├── ...
  *     ├── 5 - "/.../Forest/ForestTree/5"
  * - Forest is a summary tree node added by the shared tree and contains the following:
- *   - The inline portion of the top-level forest content is stored in a summary blob called "ForestTree".
+ *   - The inline portion of the top-level forest content is stored in a summary blob called "contents".
  *     It also contains the {@link ChunkReferenceId}s of the incremental chunks under it.
  *   - The summary for each incremental chunk under it is stored against its {@link ChunkReferenceId}.
  * - For each chunk, the structure of the summary tree is the same as the Forest. It contains the following:
  *   - The inline portion of the chunk content is stored in a blob called "contents".
  *     It also contains the {@link ChunkReferenceId}s of the incremental chunks under it.
  *   - The summary for each incremental chunk under it is stored against its {@link ChunkReferenceId}.
  * - Chunks that do not change between summaries are summarized as handles in the summary tree.
- * @remarks
- * It may seem inconsistent that although the structure for the top-level forest tree is similar to that of
- * an incremental chunk, its content is stored in a summary blob called "ForestTree" while the content for
- * the incremental chunks are stored in a summary blob called "contents".
- * This is to keep this summary backwards compatible with old format (before incremental summaries were added)
- * where the entire forest content was in a summary blob called "ForestTree". So, if incremental summaries were
- * disabled, the forest content will be fully backwards compatible.
- * Note that this limits reusing the root node in a location other than root and a non-root node in the root.
- * We could phase this out by switching to write the top-level contents under "contents" if we want to support
- * the above. However, there is no plan to do that for now.
  *
  * TODO: AB#46752
  * Add strong types for the summary structure to document it better. It will help make it super clear what the actual
@@ -305,7 +295,7 @@ export class ForestIncrementalSummaryBuilder implements IncrementalEncoderDecode
 			// and the value is the snapshot tree for the chunk.
 			for (const [chunkReferenceId, chunkSnapshotTree] of Object.entries(snapshotTree.trees)) {
 				const chunkSubTreePath = `${parentTreeKey}${chunkReferenceId}`;
-				const chunkContentsPath = `${chunkSubTreePath}/${chunkContentsBlobKey}`;
+				const chunkContentsPath = `${chunkSubTreePath}/${summaryContentBlobKeyV3}`;
 				if (!(await args.services.contains(chunkContentsPath))) {
 					throw new LoggingError(
 						`SharedTree: Cannot find contents for incremental chunk ${chunkContentsPath}`,
@@ -422,7 +412,7 @@ export class ForestIncrementalSummaryBuilder implements IncrementalEncoderDecode
 				const chunkSummaryBuilder = new SummaryTreeBuilder();
 				this.trackedSummaryProperties.parentSummaryBuilder = chunkSummaryBuilder;
 				chunkSummaryBuilder.addBlob(
-					chunkContentsBlobKey,
+					summaryContentBlobKeyV3,
 					this.trackedSummaryProperties.stringify(chunkEncoder(chunk)),
 				);
 
@@ -454,26 +444,33 @@ export class ForestIncrementalSummaryBuilder implements IncrementalEncoderDecode
 	 * It clears any tracking state and deletes the tracking properties for summaries that are older than the
 	 * latest successful summary.
 	 * @param incrementalSummaryContext - The context for the incremental summary that contains the sequence numbers.
-	 * If this is undefined, the summary tree will only contain a summary blob for `forestSummaryContent`.
-	 * @param forestSummaryContent - The stringified ForestCodec output of top-level Forest content.
+	 * If this is undefined, the summary tree will only contain a summary blob for `forestSummaryRootContent`.
+	 * @param forestSummaryRootContent - The stringified ForestCodec output of top-level Forest content.
+	 * @param forestSummaryRootContentKey - The key to use for the blob containing `forestSummaryRootContent`.
 	 * @param builder - The summary tree builder to use to add the forest's contents. Note that if tracking an incremental
 	 * summary, this builder will be the same as the one tracked in `trackedSummaryProperties`.
 	 * @returns the Forest's summary tree.
 	 */
 	public completeSummary(args: {
 		incrementalSummaryContext: IExperimentalIncrementalSummaryContext | undefined;
-		forestSummaryContent: string;
+		forestSummaryRootContent: string;
+		forestSummaryRootContentKey: string;
 		builder: SummaryTreeBuilder;
 	}): void {
-		const { incrementalSummaryContext, forestSummaryContent, builder } = args;
+		const {
+			incrementalSummaryContext,
+			forestSummaryRootContent,
+			forestSummaryRootContentKey,
+			builder,
+		} = args;
 		if (!this.enableIncrementalSummary || incrementalSummaryContext === undefined) {
-			builder.addBlob(forestSummaryContentKey, forestSummaryContent);
+			builder.addBlob(forestSummaryRootContentKey, forestSummaryRootContent);
 			return;
 		}
 
 		validateTrackingSummary(this.forestSummaryState, this.trackedSummaryProperties);
 
-		builder.addBlob(forestSummaryContentKey, forestSummaryContent);
+		builder.addBlob(forestSummaryRootContentKey, forestSummaryRootContent);
 
 		// Copy over the entries from the latest summary to the current summary.
 		// In the current summary, there can be fields that haven't changed since the latest summary and the chunks