feat: support structural json mapping (#584)

olavloite · web-flow · commit 70065e426aa0 · 2025-07-31T13:47:32.000+02:00
Add support for using POCOs mapped to JSON columns. Fixes #581
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/SpannerSampleDbContext.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/SpannerSampleDbContext.cs
@@ -209,6 +209,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 entity.Property(e => e.Code).HasMaxLength(10);
 
                 entity.Property(e => e.Name).HasMaxLength(100);
+                entity.OwnsMany(e => e.Descriptions, builder =>
+                {
+                    builder.ToJson();
+                });
             });
             
             modelBuilder.Entity<TicketSales>();
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/VenueDescription.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/VenueDescription.cs
@@ -0,0 +1,23 @@
+// Copyright 2025 Google Inc. All Rights Reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.Model;
+
+public class VenueDescription
+{
+    public string Category { get; set; }
+    public string Description { get; set; }
+    public long Capacity { get; set; }
+    public bool Active { get; set; }
+}
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/Venues.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Model/Venues.cs
@@ -29,6 +29,7 @@ public Venues()
         public bool Active { get; set; }
         public long? Capacity { get; set; }
         public List<Nullable<double>> Ratings { get; set; }
+        public List<VenueDescription> Descriptions { get; set; }
 
         public virtual ICollection<Concerts> Concerts { get; set; }
     }
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/SampleDataModel.sql b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/SampleDataModel.sql
@@ -47,11 +47,12 @@ CREATE TABLE Tracks (
 CREATE UNIQUE INDEX Idx_Tracks_AlbumId_Title ON Tracks (TrackId, Title);
 
 CREATE TABLE Venues (
-  Code      STRING(10) NOT NULL,
-  Name      STRING(100),
-  Active    BOOL NOT NULL,
-  Capacity  INT64,
-  Ratings   ARRAY<FLOAT64>,
+  Code         STRING(10) NOT NULL,
+  Name         STRING(100),
+  Active       BOOL NOT NULL,
+  Capacity     INT64,
+  Ratings      ARRAY<FLOAT64>,
+  Descriptions JSON,
 ) PRIMARY KEY (Code);
 
 CREATE TABLE Concerts (
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TransactionTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TransactionTests.cs
@@ -274,6 +274,10 @@ public async Task CanExecuteStaleRead()
             {
                 Code = venueCode,
                 Name = venueName,
+                Descriptions = [
+                    new () {Active = true, Capacity = 1000, Category = "Concert Hall", Description = "Large concert hall"},
+                    new () {Active = false, Capacity = 1000, Category = "Hall", Description = "Large hall"},
+                ],
             });
             await db.SaveChangesAsync();
             await transaction.CommitAsync();
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/SampleDataModel.sql b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/SampleDataModel.sql
@@ -48,11 +48,12 @@ CREATE TABLE Tracks (
 CREATE UNIQUE INDEX Idx_Tracks_AlbumId_Title ON Tracks (AlbumId, Title);
 
 CREATE TABLE Venues (
-  Code        STRING(10) NOT NULL,
-  Name        STRING(100),
-  Description JSON,
-  Active      BOOL NOT NULL,
-  Version     INT64 NOT NULL,
+  Code         STRING(10) NOT NULL,
+  Name         STRING(100),
+  Description  JSON,
+  Active       BOOL NOT NULL,
+  Descriptions JSON,
+  Version      INT64 NOT NULL,
 ) PRIMARY KEY (Code);
 
 CREATE TABLE Concerts (
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/SpannerSampleDbContext.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/SpannerSampleDbContext.cs
@@ -106,6 +106,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             {
                 entity.HasKey(entity => new { entity.Code });
                 entity.Property(e => e.Version).IsConcurrencyToken();
+                entity.OwnsMany(e => e.Descriptions, builder =>
+                {
+                    builder.ToJson();
+                });
             });
 
             modelBuilder.Entity<Concert>(entity =>
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/Venue.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/Venue.cs
@@ -24,6 +24,7 @@ public class Venue : VersionedEntity
         public string Name { get; set; }
         public JsonDocument Description { get; set; }
         public bool Active { get; set; }
+        public List<VenueDescription> Descriptions { get; set; }
 
         public virtual ICollection<Concert> Concerts { get; set; } = new HashSet<Concert>();
     }
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/VenueDescription.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/SampleModel/VenueDescription.cs
@@ -0,0 +1,23 @@
+// Copyright 2025 Google Inc. All Rights Reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Google.Cloud.EntityFrameworkCore.Spanner.Samples.SampleModel;
+
+public class VenueDescription
+{
+    public string Category { get; set; }
+    public string Description { get; set; }
+    public long Capacity { get; set; }
+    public bool Active { get; set; }
+}
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Snippets/StructuralJsonSample.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Snippets/StructuralJsonSample.cs
@@ -0,0 +1,58 @@
+﻿// Copyright 2025 Google Inc. All Rights Reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using Google.Cloud.EntityFrameworkCore.Spanner.Samples.SampleModel;
+using System;
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+/// <summary>
+/// Simple sample showing how to map a POCO to a JSON column.
+/// 
+/// Run from the command line with `dotnet run StructuralJsonSample`
+/// </summary>
+public static class StructuralJsonSample
+{
+    public static async Task Run(string connectionString)
+    {
+        using var context = new SpannerSampleDbContext(connectionString);
+
+        // Create a new Venue with two VenueDescriptions.
+        // VenueDescriptions are mapped to a single JSON column.
+        await context.Venues.AddAsync(new Venue
+        {
+            Code = "CH",
+            Name = "Concert Hall",
+            Descriptions =
+            [
+                new() { Active = true, Category = "Concert Hall", Capacity = 1000, Description = "Large Concert Hall" },
+                new() { Active = false, Category = "Hall", Capacity = 1000, Description = "Large Hall" }
+            ]
+        });
+        var count = await context.SaveChangesAsync();
+
+        // SaveChangesAsync returns the total number of rows that was inserted/updated/deleted.
+        Console.WriteLine($"Added {count} venue.");
+        
+        // Read back the Venue that we added and iterate over the VenueDescriptions that were serialized as JSON in the database.
+        var venue = await context.Venues.FirstAsync();
+        Console.WriteLine($"Venue has {venue.Descriptions.Count} descriptions.");
+        foreach (var description in venue.Descriptions)
+        {
+            Console.WriteLine($"{JsonSerializer.Serialize(description)}");
+        }
+    }
+}
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/EntityFrameworkMockServerTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/EntityFrameworkMockServerTests.cs
@@ -823,8 +823,8 @@ public async Task ExplicitAndImplicitTransactionIsRetried(bool disableInternalRe
         {
             // Setup results.
             _fixture.SpannerMock.AddOrUpdateStatementResult("SELECT 1", StatementResult.CreateSingleColumnResultSet(new V1.Type { Code = V1.TypeCode.Int64 }, "c", 1));
-            var insertSql = $"INSERT INTO `Venues` (`Code`, `Active`, `Capacity`, `Name`, `Ratings`)" +
-                            $"{Environment.NewLine}VALUES (@p0, @p1, @p2, @p3, @p4)";
+            var insertSql = $"INSERT INTO `Venues` (`Descriptions`, `Code`, `Active`, `Capacity`, `Name`, `Ratings`)" +
+                            $"{Environment.NewLine}VALUES (@p0, @p1, @p2, @p3, @p4, @p5)";
             _fixture.SpannerMock.AddOrUpdateStatementResult(insertSql, StatementResult.CreateUpdateCount(1L));
 
             await using var db = new MockServerSampleDbContext(ConnectionString);
@@ -841,6 +841,10 @@ public async Task ExplicitAndImplicitTransactionIsRetried(bool disableInternalRe
             {
                 Code = "C1",
                 Name = "Concert Hall",
+                Descriptions = [
+                    new() { Category = "Concert Hall", Description = "Big concert hall", Capacity = 1000, Active = true },
+                    new() { Category = "Hall", Description = "Big hall", Capacity = 1000, Active = false },
+                ],
             });
 
             // We can only disable internal retries when using explicit transactions. Otherwise, internal retries
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/EntityFrameworkUsingMutationsMockServerTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/EntityFrameworkUsingMutationsMockServerTests.cs
@@ -166,6 +166,41 @@ public async Task InsertSinger()
                 request => request is ExecuteSqlRequest sqlRequest && sqlRequest.Sql.Trim() == selectFullNameSql.Trim());
         }
 
+        [Fact]
+        public async Task InsertVenue()
+        {
+            await using var db = new MockServerSampleDbContextUsingMutations(ConnectionString);
+            db.Venues.Add(new Venues
+            {
+                Code = "C1",
+                Name = "Concert Hall",
+                Descriptions = [
+                    new() { Category = "Concert Hall", Description = "Big concert hall", Capacity = 1000, Active = true },
+                    new() { Category = "Hall", Description = "Big hall", Capacity = 1000, Active = false },
+                ],
+            });
+            var updateCount = await db.SaveChangesAsync();
+
+            Assert.Equal(1L, updateCount);
+            Assert.Single(_fixture.SpannerMock.Requests.OfType<BeginTransactionRequest>());
+            Assert.Collection(
+                _fixture.SpannerMock.Requests.OfType<CommitRequest>(),
+                request => {
+                    Assert.Single(request.Mutations);
+                    var mutation = request.Mutations[0];
+                    Assert.Equal(Mutation.OperationOneofCase.Insert, mutation.OperationCase);
+                    Assert.Equal("Venues", mutation.Insert.Table);
+                    var row = mutation.Insert.Values[0];
+                    var cols = mutation.Insert.Columns;
+                    Assert.Equal("C1", row.Values[cols.IndexOf("Code")].StringValue);
+                    Assert.Equal("Concert Hall", row.Values[cols.IndexOf("Name")].StringValue);
+                    Assert.Equal(
+                        "[{\"Active\":true,\"Capacity\":1000,\"Category\":\"Concert Hall\",\"Description\":\"Big concert hall\"},{\"Active\":false,\"Capacity\":1000,\"Category\":\"Hall\",\"Description\":\"Big hall\"}]",
+                        row.Values[cols.IndexOf("Descriptions")].StringValue);
+                }
+            );
+        }
+
         [Fact]
         public async Task InsertSingerInTransaction()
         {
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Infrastructure/Internal/SpannerModelValidator.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Infrastructure/Internal/SpannerModelValidator.cs
@@ -143,9 +143,9 @@ INNER JOIN INFORMATION_SCHEMA.INDEX_COLUMNS C ON
                     allIndexedColumns.Add(columns);
                 }
 
-                foreach (var entityType in model.GetEntityTypes())
+                foreach (var entityType in model.GetEntityTypes().Where(t => t.GetTableName() != null && !t.IsOwned()))
                 {
-                    var table = StoreObjectIdentifier.Table(entityType.GetTableName());
+                    var table = StoreObjectIdentifier.Table(entityType.GetTableName()!);
                     var pk = entityType.FindPrimaryKey();
                     if (pk == null)
                     {
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerRelationalAnnotationProvider.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerRelationalAnnotationProvider.cs
@@ -39,6 +39,10 @@ public SpannerRelationalAnnotationProvider(RelationalAnnotationProviderDependenc
         /// <inheritdoc />
         public override IEnumerable<IAnnotation> For(IColumn column, bool designTime)
         {
+            if (column.PropertyMappings.Count == 0)
+            {
+                yield break;
+            }
             var property = column.PropertyMappings.First().Property;
             var primaryKey = property.DeclaringType.ContainingEntityType.FindPrimaryKey();
             if (primaryKey is { Properties.Count: 1 }
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerQuerySqlGenerator.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerQuerySqlGenerator.cs
@@ -18,6 +18,7 @@
 using System;
 using System.Collections;
 using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore.Diagnostics;
 
 namespace Google.Cloud.EntityFrameworkCore.Spanner.Query.Internal
 {
@@ -191,5 +192,17 @@ static bool ValueRequiresParentheses(SqlExpression valueExpression)
                 };
             }
         }
+        
+        protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExpression)
+        {
+            Visit(jsonScalarExpression.Json);
+            var path = jsonScalarExpression.Path;
+            if (path.Count == 0)
+            {
+                return jsonScalarExpression;
+            }
+            throw new ArgumentException("json path expressions are not supported");
+        }
+        
     }
 }
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerStringTypeMapping.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerStringTypeMapping.cs
@@ -17,6 +17,7 @@
 using System;
 using System.Data;
 using System.Data.Common;
+using Microsoft.EntityFrameworkCore.Storage.Json;
 
 namespace Google.Cloud.EntityFrameworkCore.Spanner.Storage.Internal
 {
@@ -34,7 +35,7 @@ public SpannerStringTypeMapping(
             StoreTypePostfix? storeTypePostfix = null)
             : this(
                 new RelationalTypeMappingParameters(
-                    new CoreTypeMappingParameters(typeof(string)),
+                    new CoreTypeMappingParameters(typeof(string), jsonValueReaderWriter: JsonStringReaderWriter.Instance),
                     storeType,
                     storeTypePostfix ?? StoreTypePostfix.Size,
                     GetDbType(unicode, fixedLength),
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerStructuralJsonTypeMapping.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerStructuralJsonTypeMapping.cs
@@ -0,0 +1,75 @@
+// Copyright 2025, Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Data.Common;
+using System.IO;
+using System.Linq.Expressions;
+using System.Reflection;
+using System.Text;
+using System.Text.Json;
+using Google.Cloud.Spanner.Data;
+using Microsoft.EntityFrameworkCore.Storage;
+
+namespace Google.Cloud.EntityFrameworkCore.Spanner.Storage.Internal;
+
+public class SpannerStructuralJsonTypeMapping : JsonTypeMapping
+{
+    private static readonly MethodInfo CreateUtf8StreamMethod
+        = typeof(SpannerStructuralJsonTypeMapping).GetMethod(nameof(CreateUtf8Stream), [typeof(string)])!;
+
+    private static readonly MethodInfo GetStringMethod
+        = typeof(DbDataReader).GetRuntimeMethod(nameof(DbDataReader.GetString), [typeof(int)])!;
+
+    public static SpannerStructuralJsonTypeMapping Default => JsonTypeDefault;
+
+    public static SpannerStructuralJsonTypeMapping JsonTypeDefault { get; } = new("json");
+
+    public SpannerStructuralJsonTypeMapping(string storeType)
+        : base(storeType, typeof(JsonElement), System.Data.DbType.String)
+    {
+    }
+
+    public override MethodInfo GetDataReaderMethod() => GetStringMethod;
+
+    public static MemoryStream CreateUtf8Stream(string json)
+        => json == ""
+            ? throw new InvalidOperationException("json cannot be an empty string.")
+            : new MemoryStream(Encoding.UTF8.GetBytes(json));
+
+    public override Expression CustomizeDataReaderExpression(Expression expression)
+        => Expression.Call(CreateUtf8StreamMethod, expression);
+
+    protected SpannerStructuralJsonTypeMapping(RelationalTypeMappingParameters parameters)
+        : base(parameters)
+    {
+    }
+
+    protected override string GenerateNonNullSqlLiteral(object value)
+    {
+        if (value is string s)
+        {
+            return $"JSON '{s}'";
+        }
+        throw new ArgumentException($"{value} is not valid for database type JSON");
+    }
+
+    protected override RelationalTypeMapping Clone(RelationalTypeMappingParameters parameters)
+        => new SpannerStructuralJsonTypeMapping(parameters);
+
+    protected override void ConfigureParameter(DbParameter parameter)
+    {
+        ((SpannerParameter)parameter).SpannerDbType = SpannerDbType.Json;
+        base.ConfigureParameter(parameter);
+    }}
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerTypeMappingSource.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerTypeMappingSource.cs
diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Update/Internal/SpannerModificationCommandBatch .cs b/Google.Cloud.EntityFrameworkCore.Spanner/Update/Internal/SpannerModificationCommandBatch .cs

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ public Venues()`
`29`	`29`	`public bool Active { get; set; }`
`30`	`30`	`public long? Capacity { get; set; }`
`31`	`31`	`public List<Nullable<double>> Ratings { get; set; }`
	`32`	`+ public List<VenueDescription> Descriptions { get; set; }`
`32`	`33`
`33`	`34`	`public virtual ICollection<Concerts> Concerts { get; set; }`
`34`	`35`	`}`
Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ public class Venue : VersionedEntity`
`24`	`24`	`public string Name { get; set; }`
`25`	`25`	`public JsonDocument Description { get; set; }`
`26`	`26`	`public bool Active { get; set; }`
	`27`	`+ public List<VenueDescription> Descriptions { get; set; }`
`27`	`28`
`28`	`29`	`public virtual ICollection<Concert> Concerts { get; set; } = new HashSet<Concert>();`
`29`	`30`	`}`