diff --git a/build.sbt b/build.sbt index 06345bff6290..a50be2fd26c0 100644 --- a/build.sbt +++ b/build.sbt @@ -1251,6 +1251,8 @@ lazy val `ydoc-server` = project lazy val `persistance` = (project in file("lib/java/persistance")) .settings( version := "0.1", + Test / fork := true, + commands += WithDebugCommand.withDebug, frgaalJavaCompilerSetting, Compile / javacOptions := ((Compile / javacOptions).value), libraryDependencies ++= Seq( diff --git a/engine/runtime-parser/src/test/java/org/enso/compiler/core/IrPersistanceTest.java b/engine/runtime-parser/src/test/java/org/enso/compiler/core/IrPersistanceTest.java index 4db46799d881..1e96c4b8686e 100644 --- a/engine/runtime-parser/src/test/java/org/enso/compiler/core/IrPersistanceTest.java +++ b/engine/runtime-parser/src/test/java/org/enso/compiler/core/IrPersistanceTest.java @@ -33,7 +33,7 @@ public void resetDebris() { @Test public void locationTest() throws Exception { var l = new Location(12, 33); - var n = serde(Location.class, l, 8); + var n = serde(Location.class, l, 16); assertEquals(12, n.start()); assertEquals(33, n.end()); @@ -43,14 +43,14 @@ public void locationTest() throws Exception { @Test public void identifiedLocation() throws Exception { var il = new IdentifiedLocation(new Location(5, 19), null); - var in = serde(IdentifiedLocation.class, il, 12); + var in = serde(IdentifiedLocation.class, il, 20); assertEquals(il, in); } @Test public void identifiedLocationWithUUID() throws Exception { var il = new IdentifiedLocation(new Location(5, 19), UUID.randomUUID()); - var in = serde(IdentifiedLocation.class, il, 33); + var in = serde(IdentifiedLocation.class, il, 41); assertEquals("UUIDs are serialized at the moment", il, in); } @@ -63,7 +63,7 @@ public void identifiedLocationNoUUID() throws Exception { case UUID any -> null; default -> obj; }; - var in = serde(IdentifiedLocation.class, il, 12, fn); + var in = serde(IdentifiedLocation.class, il, 20, fn); var withoutUUID = new IdentifiedLocation(il.location()); assertEquals("UUIDs are no longer serialized", withoutUUID, in); } @@ -107,7 +107,7 @@ public void refHolderNoUUID() throws Exception { case UUID any -> null; default -> obj; }; - var in = serde(IdHolder.class, il, 1, fn); + var in = serde(IdHolder.class, il, 9, fn); var withoutUUID = new IdHolder(null); assertEquals("UUIDs are no longer serialized", withoutUUID, in); } @@ -118,7 +118,7 @@ public void scalaMap() throws Exception { var idLoc1 = new IdentifiedLocation(new Location(1, 5)); var in = scala.collection.immutable.Map$.MODULE$.empty().$plus(new Tuple2("Hi", idLoc1)); - var out = serde(scala.collection.immutable.Map.class, in, 36); + var out = serde(scala.collection.immutable.Map.class, in, 44); assertEquals("One element", 1, out.size()); assertEquals(in, out); @@ -137,7 +137,7 @@ public void scalaImmutableMapIsLazy() throws Exception { .$plus(new Tuple2("World", s2)); LazySeq.forbidden = true; - var out = (scala.collection.immutable.Map) serde(scala.collection.immutable.Map.class, in, 64); + var out = (scala.collection.immutable.Map) serde(scala.collection.immutable.Map.class, in, 72); assertEquals("Two pairs element", 2, out.size()); assertEquals("Two keys", 2, out.keySet().size()); @@ -159,7 +159,7 @@ public void scalaHashMap() throws Exception { (scala.collection.mutable.HashMap) scala.collection.mutable.HashMap$.MODULE$.apply(immutable); - var out = serde(scala.collection.mutable.Map.class, in, 36); + var out = serde(scala.collection.mutable.Map.class, in, 44); assertEquals("One element", 1, out.size()); assertEquals(in, out); @@ -171,7 +171,7 @@ public void scalaSet() throws Exception { var idLoc1 = new IdentifiedLocation(new Location(1, 5)); var in = scala.collection.immutable.Set$.MODULE$.empty().$plus(idLoc1); - var out = serde(scala.collection.immutable.Set.class, in, 24); + var out = serde(scala.collection.immutable.Set.class, in, 32); assertEquals("One element", 1, out.size()); assertEquals(in, out); @@ -183,7 +183,7 @@ public void scalaList() throws Exception { var idLoc2 = new IdentifiedLocation(new Location(2, 4), UUID.randomUUID()); var in = join(idLoc2, join(idLoc1, nil())); - var out = serde(List.class, in, 65); + var out = serde(List.class, in, 73); assertEquals("Two elements", 2, out.size()); assertEquals("UUIDs are serialized at the moment", idLoc2, out.head()); @@ -195,7 +195,7 @@ public void scalaListSharedRef() throws Exception { var idLoc1 = new IdentifiedLocation(new Location(1, 5)); var in = join(idLoc1, join(idLoc1, nil())); - var out = serde(List.class, in, 32); + var out = serde(List.class, in, 40); assertEquals("Two elements", 2, out.size()); assertEquals("Head is equal to original", idLoc1, out.head()); @@ -288,7 +288,7 @@ public void hashMapIsLazy() throws Exception { in.put("World", s2); LazySeq.forbidden = true; - var out = serde(java.util.Map.class, in, 64); + var out = serde(java.util.Map.class, in, 72); assertEquals("Two pairs element", 2, out.size()); assertEquals("Two keys", 2, out.keySet().size()); diff --git a/lib/java/persistance/src/main/java/org/enso/persist/PerBufferReference.java b/lib/java/persistance/src/main/java/org/enso/persist/PerBufferReference.java index 930d957cb6ec..59086e995dda 100644 --- a/lib/java/persistance/src/main/java/org/enso/persist/PerBufferReference.java +++ b/lib/java/persistance/src/main/java/org/enso/persist/PerBufferReference.java @@ -1,5 +1,6 @@ package org.enso.persist; +import java.io.IOException; import org.enso.persist.PerInputImpl.InputCache; import org.enso.persist.Persistance.Reference; @@ -8,14 +9,32 @@ final class PerBufferReference extends Persistance.Reference { private final PerInputImpl.InputCache cache; private final int offset; - private PerBufferReference(Persistance p, PerInputImpl.InputCache buffer, int offset) { + /** + * References can be cached, or loaded again every time. + * + *

If {@code cached} is set to {@code this}, then the caching is disabled and {@link + * #get(Class)} will always load a new instance of the object. This is the mode one gets when + * using an API method {@link Persistance.Input#readReference(Class)}. + * + *

In other cases the {@code cached} value can be {@code null} meaning not yet loaded + * or non-{@code null} holding the cached value to be returned from the {@link #get(Class)} + * method until this reference instance is GCed. + */ + private Object cached; + + private PerBufferReference( + Persistance p, PerInputImpl.InputCache buffer, int offset, boolean allowCaching) { this.p = p; this.cache = buffer; this.offset = offset; + this.cached = allowCaching ? null : this; } @SuppressWarnings(value = "unchecked") - final T readObject(Class clazz) { + final T readObject(Class clazz) throws IOException { + if (cached != this && clazz.isInstance(cached)) { + return clazz.cast(cached); + } if (p != null) { if (clazz.isAssignableFrom(p.clazz)) { clazz = (Class) p.clazz; @@ -23,8 +42,11 @@ final T readObject(Class clazz) { throw new ClassCastException(); } } - org.enso.persist.PerInputImpl in = new PerInputImpl(cache, offset); + var in = new PerInputImpl(cache, offset); T obj = in.readInline(clazz); + if (cached != this) { + cached = obj; + } return obj; } @@ -33,6 +55,10 @@ static Reference from(InputCache buffer, int offset) { } static Reference from(Persistance p, InputCache buffer, int offset) { - return new PerBufferReference<>(p, buffer, offset); + return new PerBufferReference<>(p, buffer, offset, false); + } + + static Reference cached(Persistance p, InputCache buffer, int offset) { + return new PerBufferReference<>(p, buffer, offset, true); } } diff --git a/lib/java/persistance/src/main/java/org/enso/persist/PerGenerator.java b/lib/java/persistance/src/main/java/org/enso/persist/PerGenerator.java index aceebece524c..c63bf543805c 100644 --- a/lib/java/persistance/src/main/java/org/enso/persist/PerGenerator.java +++ b/lib/java/persistance/src/main/java/org/enso/persist/PerGenerator.java @@ -12,12 +12,14 @@ import org.slf4j.Logger; final class PerGenerator { - static final byte[] HEADER = new byte[] {0x0a, 0x0d, 0x03, 0x0f}; + static final byte[] HEADER = new byte[] {0x0a, 0x0d, 0x13, 0x0f}; private final OutputStream main; private final Map knownObjects = new IdentityHashMap<>(); + private int countReferences = 1; + private final Map pendingReferences = new IdentityHashMap<>(); private final Histogram histogram; private final PerMap map; - final Function writeReplace; + private final Function writeReplace; private int position; private PerGenerator( @@ -39,12 +41,11 @@ static byte[] writeObject(Object obj, Function writeReplace) thr data.writeInt(g.versionStamp()); data.write(new byte[4]); // space data.flush(); - var at = g.writeObject(obj); + + var at = g.writeObjectAndReferences(obj); + var arr = out.toByteArray(); - arr[8] = (byte) ((at >> 24) & 0xff); - arr[9] = (byte) ((at >> 16) & 0xff); - arr[10] = (byte) ((at >> 8) & 0xff); - arr[11] = (byte) (at & 0xff); + putIntToArray(arr, 8, at); if (histogram != null) { histogram.dump(PerUtils.LOG, arr.length); @@ -52,6 +53,13 @@ static byte[] writeObject(Object obj, Function writeReplace) thr return arr; } + private static void putIntToArray(byte[] arr, int position, int value) { + arr[position] = (byte) ((value >> 24) & 0xff); + arr[position + 1] = (byte) ((value >> 16) & 0xff); + arr[position + 2] = (byte) ((value >> 8) & 0xff); + arr[position + 3] = (byte) (value & 0xff); + } + final int writeObject(T t) throws IOException { if (t == null) { return -1; @@ -106,6 +114,63 @@ final int versionStamp() { return map.versionStamp; } + private int registerReference(Persistance.Reference ref) { + var obj = ref.get(Object.class); + var existingId = pendingReferences.get(obj); + if (existingId == null) { + var currentSize = countReferences++; + pendingReferences.put(obj, currentSize); + return currentSize; + } else { + return existingId; + } + } + + /** + * Writes an object into the buffer. Writes also all {@link Persistance.Reference} that were left + * pending during the serialization. + * + * @param obj the object to write down + * @return location of the table {@code int size and then int[size]} + */ + private int writeObjectAndReferences(Object obj) throws IOException { + pendingReferences.put(obj, 0); + var objAt = writeObject(obj); + + var refsOut = new ByteArrayOutputStream(); + var refsData = new DataOutputStream(refsOut); + refsData.writeInt(-1); // space for size of references + refsData.writeInt(objAt); // the main object + var count = 1; + for (; ; ) { + var all = new ArrayList<>(pendingReferences.entrySet()); + all.sort( + (e1, e2) -> { + return e1.getValue() - e2.getValue(); + }); + var round = all.subList(count, all.size()); + if (round.isEmpty()) { + break; + } + for (var entry : round) { + count++; + var at = writeObject(entry.getKey()); + assert count == entry.getValue() : "Expecting " + count + " got " + entry.getValue(); + refsData.writeInt(at); + } + } + refsData.flush(); + var arr = refsOut.toByteArray(); + + putIntToArray(arr, 0, count); + + var tableAt = this.position; + this.main.write(arr); + this.position += arr.length; + + return tableAt; + } + private static final class ReferenceOutput extends DataOutputStream implements Persistance.Output { private final PerGenerator generator; @@ -117,6 +182,12 @@ private static final class ReferenceOutput extends DataOutputStream @Override public void writeInline(Class clazz, T t) throws IOException { + if (Persistance.Reference.class == clazz) { + Persistance.Reference ref = (Persistance.Reference) t; + var id = this.generator.registerReference(ref); + writeInt(id); + return; + } var obj = generator.writeReplace.apply(t); var p = generator.map.forType(clazz); p.writeInline(obj, this); diff --git a/lib/java/persistance/src/main/java/org/enso/persist/PerInputImpl.java b/lib/java/persistance/src/main/java/org/enso/persist/PerInputImpl.java index 37d16d2fb611..7d612ed5ce6e 100644 --- a/lib/java/persistance/src/main/java/org/enso/persist/PerInputImpl.java +++ b/lib/java/persistance/src/main/java/org/enso/persist/PerInputImpl.java @@ -31,24 +31,37 @@ static Reference readObject(ByteBuffer buf, Function read } } var version = buf.getInt(4); - var cache = new InputCache(buf, readResolve); - if (version != cache.map().versionStamp) { + var map = PerMap.create(); + if (version != map.versionStamp) { throw new IOException( "Incompatible version " + Integer.toHexString(version) + " != " - + Integer.toHexString(cache.map().versionStamp)); + + Integer.toHexString(map.versionStamp)); } - var at = buf.getInt(8); - return PerBufferReference.from(cache, at); + var tableAt = buf.getInt(8); + buf.position(tableAt); + var count = buf.getInt(); + assert count > 0 : "There is always the main object in the table: " + count; + var refs = new int[count]; + for (var i = 0; i < count; i++) { + refs[i] = buf.getInt(); + } + var cache = new InputCache(buf, readResolve, map, refs); + return cache.getRef(0); } @Override - public T readInline(Class clazz) { + public T readInline(Class clazz) throws IOException { + if (clazz == Persistance.Reference.class) { + var refId = readInt(); + var ref = cache.getRef(refId); + return clazz.cast(ref); + } Persistance p = cache.map().forType(clazz); T res = p.readWith(this); - var resolve = cache.readResolve().apply(res); + var resolve = cache.resolveObject(res); return clazz.cast(resolve); } @@ -193,14 +206,14 @@ static Object readIndirect(InputCache cache, PerMap map, Input in) throws IOExce var id = in.readInt(); var p = map.forId(id); - if (cache.cache().get(at) instanceof Object res) { + if (cache.getObjectAt(at) instanceof Object res) { return p.clazz.cast(res); } var inData = new PerInputImpl(cache, at); var res = p.readWith(inData); - res = cache.readResolve().apply(res); - var prev = cache.cache().put(at, res); + res = cache.resolveObject(res); + var prev = cache.putObjectAt(at, res); if (prev != null) { var bothObjectsAreTheSame = Objects.equals(res, prev); var sb = new StringBuilder(); @@ -241,17 +254,51 @@ static Reference readIndirectAsReference( } } - static final record InputCache( - ByteBuffer buf, - Function readResolve, - Map cache, - PerMap map) { - InputCache(ByteBuffer buf, Function readResolve) { - this( - buf, - readResolve == null ? Function.identity() : readResolve, - new HashMap<>(), - PerMap.create()); + static final class InputCache { + private final Map cache = new HashMap<>(); + private final Function readResolve; + private final PerMap map; + private final ByteBuffer buf; + private final Reference[] refs; + + private InputCache( + ByteBuffer buf, Function readResolve, PerMap map, int[] refs) { + this.buf = buf; + this.readResolve = readResolve; + this.map = map; + this.refs = new Reference[refs.length]; + for (var i = 0; i < refs.length; i++) { + this.refs[i] = PerBufferReference.cached(null, this, refs[i]); + } + } + + final Object resolveObject(Object res) { + if (readResolve != null) { + return readResolve.apply(res); + } else { + return res; + } + } + + final Object getObjectAt(int at) { + return cache.get(at); + } + + final Object putObjectAt(int at, Object obj) { + return cache.put(at, obj); + } + + final PerMap map() { + return map; + } + + final ByteBuffer buf() { + return buf; + } + + @SuppressWarnings("unchecked") + final Reference getRef(int index) { + return refs[index]; } } } diff --git a/lib/java/persistance/src/main/java/org/enso/persist/PerMap.java b/lib/java/persistance/src/main/java/org/enso/persist/PerMap.java index 0cd477063489..d7b4c486e5e2 100644 --- a/lib/java/persistance/src/main/java/org/enso/persist/PerMap.java +++ b/lib/java/persistance/src/main/java/org/enso/persist/PerMap.java @@ -5,7 +5,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import org.enso.persist.Persistance.Reference; import org.openide.util.lookup.Lookups; final class PerMap { @@ -17,7 +16,6 @@ final class PerMap { var loader = PerMap.class.getClassLoader(); var lookup = Lookups.metaInfServices(loader); var all = new ArrayList(); - all.add(ReferencePersitance.INSTANCE); all.addAll(lookup.lookupAll(Persistance.class)); ALL = all; } @@ -100,23 +98,4 @@ final Persistance forId(int id) { } return p; } - - private static class ReferencePersitance extends Persistance { - private static final ReferencePersitance INSTANCE = new ReferencePersitance(); - - private ReferencePersitance() { - super(Reference.class, true, 60941); - } - - @Override - @SuppressWarnings("unchecked") - protected void writeObject(Reference obj, Output out) throws IOException { - out.writeObject(obj.get(Object.class)); - } - - @Override - protected Reference readObject(Input in) throws IOException, ClassNotFoundException { - return in.readReference(Object.class); - } - } } diff --git a/lib/java/persistance/src/main/java/org/enso/persist/Persistance.java b/lib/java/persistance/src/main/java/org/enso/persist/Persistance.java index f9b6db9856f4..8c9d3422e2e5 100644 --- a/lib/java/persistance/src/main/java/org/enso/persist/Persistance.java +++ b/lib/java/persistance/src/main/java/org/enso/persist/Persistance.java @@ -134,7 +134,8 @@ public static interface Input extends DataInput { /** * Reads a reference to an object written down by {@link Output#writeObject(Object)} but without * reading the object itself. The object can then be obtained "later" via the {@link - * Reference#get(Class)} method. + * Reference#get(Class)} method. The object is not cached and is loaded again and again + * whenever the {@link Reference#get(Class)} method is called. * * @param the type to read * @param clazz the expected type of the object to read @@ -236,7 +237,13 @@ public V get(Class expectedType) { var value = switch (this) { case PerMemoryReference m -> m.value(); - case PerBufferReference b -> b.readObject(expectedType); + case PerBufferReference b -> { + try { + yield b.readObject(expectedType); + } catch (IOException e) { + throw raise(RuntimeException.class, e); + } + } }; return expectedType.cast(value); } diff --git a/lib/java/persistance/src/test/java/org/enso/persist/PersistanceTest.java b/lib/java/persistance/src/test/java/org/enso/persist/PersistanceTest.java index 956f1bead9d4..f9795ffc6c8c 100644 --- a/lib/java/persistance/src/test/java/org/enso/persist/PersistanceTest.java +++ b/lib/java/persistance/src/test/java/org/enso/persist/PersistanceTest.java @@ -1,13 +1,14 @@ -package org.enso.compiler.core; +package org.enso.persist; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; import java.io.IOException; import java.util.UUID; import java.util.function.Function; import java.util.function.Supplier; -import org.enso.persist.Persistable; -import org.enso.persist.Persistance; import org.junit.Test; import org.openide.util.lookup.ServiceProvider; @@ -171,5 +172,80 @@ public record IntegerSupply(Supplier supply) {} // @start region="self-annotation" @Persistable(id = 432436) public record ServiceSupply(Service supply) {} + // @end region="self-annotation" + + @Persistable(id = 432437) + public static class SelfLoop { + public Persistance.Reference self; + + public Persistance.Reference self() { + return self; + } + + public SelfLoop(Persistance.Reference self) { + this.self = self; + } + } + + @Test + public void testReferenceLoopsInPersistance() throws Exception { + var obj = new SelfLoop(null); + // make the loop + obj.self = Persistance.Reference.of(obj); + + var loaded = serde(SelfLoop.class, obj, -1); + var next = loaded.self.get(SelfLoop.class); + var next2 = next.self.get(SelfLoop.class); + assertSame("The recreated object again points to itself", next, next2); + assertSame("The recreated object again points to itself", loaded, next); + } + + @Persistable(id = 432439) + public record LongerLoop1(int x, Persistance.Reference y) {} + + @Persistable(id = 432440) + public record LongerLoop2(Persistance.Reference y) {} + + @Persistable(id = 432441) + public static class LongerLoop3 { + public final String a; + public Persistance.Reference y; + + public String a() { + return a; + } + + public Persistance.Reference y() { + return y; + } + + public LongerLoop3(String a, Persistance.Reference y) { + this.a = a; + this.y = y; + } + } + + @Test + public void testLoopsBetweenDifferentTypes() throws Exception { + var obj3 = new LongerLoop3("a", null); + var obj2 = new LongerLoop2(Persistance.Reference.of(obj3)); + var obj1 = new LongerLoop1(1, Persistance.Reference.of(obj2)); + obj3.y = Persistance.Reference.of(obj1); + + var loaded1 = serde(LongerLoop1.class, obj1, -1); + var r2 = loaded1.y().get(LongerLoop2.class); + var r3 = r2.y().get(LongerLoop3.class); + var r1 = r3.y().get(LongerLoop1.class); + + assertSame("The recreated structure contains the loop", loaded1, r1); + + var current = r1; + for (var i = 0; i < 10; i++) { + var next = + loaded1.y().get(LongerLoop2.class).y().get(LongerLoop3.class).y().get(LongerLoop1.class); + assertSame("current points back to itself", current, next); + current = next; + } + } }