wip

works but needs cleanup and rebasing on the _hW branch once that's stable

Author: wmertens

packages/qwik/src/core/shared/qrl/qrl-class.ts

@@ -140,14 +140,19 @@ export const createQRL = <TYPE>(
       return fn;
     }
     return function (this: unknown, ...args: QrlArgs<TYPE>) {
-      let context = tryGetInvokeContext();
-      if (context) {
-        return fn.apply(this, args);
-      }
-      context = newInvokeContext();
+      const context = tryGetInvokeContext() || newInvokeContext();
+      const prevQrl = context.$qrl$;
+      const prevEvent = context.$event$;
+      // used by useLexicalScope
       context.$qrl$ = qrl;
-      context.$event$ = this as Event;
-      return invoke.call(this, context, fn as any, ...args);
+      // TODO possibly remove this, are we using it?
+      context.$event$ ||= this as Event;
+      try {
+        return invoke.call(this, context, fn as any, ...args);
+      } finally {
+        context.$qrl$ = prevQrl;
+        context.$event$ = prevEvent;
+      }
     } as TYPE;
   };
 

packages/qwik/src/core/shared/serialization.md

@@ -37,3 +37,17 @@ To avoid blocking the main thread on wake, we lazily restore the roots, with cac
 The serialized text is first parsed to get an array of encoded root data.
 
 Then, a proxy gets the raw data and returns an array that deserializes properties on demand and caches them. Objects are also lazily restored.
+
+## Out-of-order streaming
+
+when we allow out-of-order streaming but we want interactivity while the page is still streaming, we can't scan the state once for cycles/Promises/references, because later state might refer to earlier state.
+
+Therefore we must make the serializer single-pass. To do this, we could store each object as a root, but that will make references take more bytes. Instead, we could implement sub-paths for references, and when we encounter a reference, we output that instead.
+
+For later references it would be best to have a root reference, so we'd probably add a root for the sub-reference and later we can refer to that.
+
+We'll still need to keep track of Promises. We can do this by waiting, but that halts streaming. Instead, we could write a forward reference id, and when the promise is resolved, we store the result as the next root item. At the end of the stream we can emit a mapping from forward references to root index.
+
+Then later, when we send out-of-order state, it will append to the existing state on the client.
+
+This single-pass approach might just be better in general, because several bugs were found due to differences between the first and second pass code.

packages/qwik/src/core/shared/shared-serialization.ts

@@ -16,6 +16,7 @@ import {
   WrappedSignal,
   isSerializerObj,
   type EffectSubscriptions,
+  type SerializedArg,
 } from '../signal/signal';
 import type { Subscriber } from '../signal/signal-subscriber';
 import {
@@ -291,6 +292,7 @@ const inflate = (
       signal.$effects$ = d.slice(5);
       break;
     }
+    // Inflating a SerializedSignal is the same as inflating a ComputedSignal
     case TypeIds.SerializedSignal:
     case TypeIds.ComputedSignal: {
       const computed = target as ComputedSignal<unknown>;
@@ -840,11 +842,31 @@ export const createSerializationContext = (
         const v =
           obj instanceof WrappedSignal
             ? obj.untrackedValue
-            : obj instanceof ComputedSignal && (obj.$invalid$ || fastSkipSerialize(obj))
+            : obj instanceof ComputedSignal &&
+                !(obj instanceof SerializedSignal) &&
+                (obj.$invalid$ || fastSkipSerialize(obj))
               ? NEEDS_COMPUTATION
               : obj.$untrackedValue$;
         if (v !== NEEDS_COMPUTATION) {
-          discoveredValues.push(v);
+          if (obj instanceof SerializedSignal) {
+            promises.push(
+              (obj.$computeQrl$ as any as QRLInternal<SerializedArg<any, any>>)
+                .resolve()
+                .then((arg) => {
+                  let data;
+                  if (arg.serialize) {
+                    data = arg.serialize(v);
+                  }
+                  if (data === undefined) {
+                    data = NEEDS_COMPUTATION;
+                  }
+                  serializationResults.set(obj, data);
+                  discoveredValues.push(data);
+                })
+            );
+          } else {
+            discoveredValues.push(v);
+          }
         }
         if (obj.$effects$) {
           discoveredValues.push(...obj.$effects$);
@@ -1188,7 +1210,9 @@ function serialize(serializationContext: SerializationContext): void {
        * Special case: when a Signal value is an SSRNode, it always needs to be a DOM ref instead.
        * It can never be meant to become a vNode, because vNodes are internal only.
        */
+      const isSerialized = value instanceof SerializedSignal;
       const v: unknown =
+        !isSerialized &&
         value instanceof ComputedSignal &&
         (value.$invalid$ || fastSkipSerialize(value.$untrackedValue$))
           ? NEEDS_COMPUTATION
@@ -1209,12 +1233,13 @@ function serialize(serializationContext: SerializationContext): void {
           value.$effects$,
         ];
         if (v !== NEEDS_COMPUTATION) {
-          out.push(v);
+          if (isSerialized) {
+            out.push(serializationResults.get(value));
+          } else {
+            out.push(v);
+          }
         }
-        output(
-          value instanceof SerializedSignal ? TypeIds.SerializedSignal : TypeIds.ComputedSignal,
-          out
-        );
+        output(isSerialized ? TypeIds.SerializedSignal : TypeIds.ComputedSignal, out);
       } else {
         output(TypeIds.Signal, [v, ...(value.$effects$ || [])]);
       }

packages/qwik/src/core/shared/shared-serialization.unit.ts

@@ -411,9 +411,10 @@ describe('shared-serialization', () => {
       `);
     });
     it(title(TypeIds.SerializedSignal), async () => {
-      const custom = createSerialized$<MyCustomSerializable, number>(
-        (prev) => new MyCustomSerializable((prev as number) || 3)
-      );
+      const custom = createSerialized$({
+        deserialize: (n?: number) => new MyCustomSerializable(n || 3),
+        serialize: (obj) => obj.n,
+      });
       // Force the value to be created
       custom.value.inc();
       const objs = await serialize(custom);

packages/qwik/src/core/signal/signal-api.ts

@@ -5,8 +5,8 @@ import {
   SerializedSignal,
   Signal as SignalImpl,
   throwIfQRLNotResolved,
-  type ConstructorFn,
   type CustomSerializable,
+  type SerializedArg,
 } from './signal';
 import type { Signal } from './signal.public';
 
@@ -23,8 +23,13 @@ export const createComputedSignal = <T>(qrl: QRL<() => T>): ComputedSignal<T> =>
 
 /** @internal */
 export const createSerializedSignal = <T extends CustomSerializable<T, S>, S>(
-  qrl: QRL<ConstructorFn<T, S>>
+  // We want to also add T as a possible parameter type, but that breaks type inference
+  arg: QRL<{
+    serialize: (data: S | undefined) => T;
+    deserialize: (data: T) => S;
+    initial?: S;
+  }>
 ) => {
-  throwIfQRLNotResolved(qrl);
-  return new SerializedSignal<T>(null, qrl);
+  throwIfQRLNotResolved(arg);
+  return new SerializedSignal<T, S>(null, arg as any as QRLInternal<SerializedArg<T, S>>);
 };

packages/qwik/src/core/signal/signal.public.ts

@@ -1,6 +1,6 @@
 import { implicit$FirstArg } from '../shared/qrl/implicit_dollar';
 import { SerializerSymbol } from '../shared/utils/serialize-utils';
-import type { CustomSerializable } from './signal';
+import type { CustomSerializable, SerializedArg } from './signal';
 import {
   createSignal as _createSignal,
   createComputedSignal as createComputedQrl,
@@ -88,13 +88,9 @@ export { createComputedQrl };
  *
  * @public
  */
-export const createSerialized$: <
-  T extends CustomSerializable<any, S>,
-  S = T extends { [SerializerSymbol]: (obj: any) => infer U } ? U : unknown,
->(
+export const createSerialized$: <T, S>(
   // We want to also add T as a possible parameter type, but that breaks type inference
-  // The
-  qrl: (data: S | undefined) => T
+  arg: SerializedArg<T, S>
 ) => T extends Promise<any> ? never : SerializedSignal<T> = implicit$FirstArg(
   createSerializedQrl as any
 );

packages/qwik/src/core/signal/signal.ts

@@ -11,28 +11,28 @@
  *   - It needs to store a function which needs to re-run.
  *   - It is `Readonly` because it is computed.
  */
+import type { VNode } from '../client/types';
+import { vnode_getProp, vnode_isTextVNode, vnode_isVNode, vnode_setProp } from '../client/vnode';
 import { pad, qwikDebugToString } from '../debug';
+import type { OnRenderFn } from '../shared/component.public';
 import { assertDefined, assertFalse, assertTrue } from '../shared/error/assert';
+import { QError, qError } from '../shared/error/error';
+import type { Props } from '../shared/jsx/jsx-runtime';
 import { type QRLInternal } from '../shared/qrl/qrl-class';
 import type { QRL } from '../shared/qrl/qrl.public';
-import { trackSignal, tryGetInvokeContext } from '../use/use-core';
-import { Task, TaskFlags, isTask } from '../use/use-task';
+import { ChoreType, type NodePropData, type NodePropPayload } from '../shared/scheduler';
+import type { Container, HostElement } from '../shared/types';
 import { ELEMENT_PROPS, OnRenderProp, QSubscribers } from '../shared/utils/markers';
 import { isPromise } from '../shared/utils/promises';
 import { qDev } from '../shared/utils/qdev';
-import type { VNode } from '../client/types';
-import { vnode_getProp, vnode_isTextVNode, vnode_isVNode, vnode_setProp } from '../client/vnode';
-import { ChoreType, type NodePropData, type NodePropPayload } from '../shared/scheduler';
-import type { Container, HostElement } from '../shared/types';
+import { SerializerSymbol } from '../shared/utils/serialize-utils';
 import type { ISsrNode } from '../ssr/ssr-types';
+import { trackSignal, tryGetInvokeContext } from '../use/use-core';
+import { Task, TaskFlags, isTask } from '../use/use-task';
+import { NEEDS_COMPUTATION } from './flags';
+import { Subscriber, isSubscriber } from './signal-subscriber';
 import type { Signal as ISignal, ReadonlySignal } from './signal.public';
 import type { TargetType } from './store';
-import { isSubscriber, Subscriber } from './signal-subscriber';
-import type { Props } from '../shared/jsx/jsx-runtime';
-import type { OnRenderFn } from '../shared/component.public';
-import { NEEDS_COMPUTATION } from './flags';
-import { QError, qError } from '../shared/error/error';
-import { SerializerSymbol } from '../shared/utils/serialize-utils';
 
 const DEBUG = false;
 
@@ -374,7 +374,7 @@ export const triggerEffects = (
   DEBUG && log('done scheduling');
 };
 
-type ComputeQRL<T> = QRLInternal<(prev: T | undefined) => T>;
+type ComputeQRL<T> = QRLInternal<() => T>;
 
 /**
  * A signal which is computed from other signals.
@@ -415,6 +415,7 @@ export class ComputedSignal<T> extends Signal<T> {
    */
   force() {
     this.$invalid$ = true;
+    // TODO shouldn't force be set to true, invalid left alone and the effects scheduled?
     this.$forceRunEffects$ = false;
     triggerEffects(this.$container$, this, this.$effects$);
   }
@@ -439,9 +440,7 @@ export class ComputedSignal<T> extends Signal<T> {
     const previousEffectSubscription = ctx?.$effectSubscriber$;
     ctx && (ctx.$effectSubscriber$ = [this, EffectProperty.VNODE]);
     try {
-      const untrackedValue = computeQrl.getFn(ctx)(
-        this.$untrackedValue$ === NEEDS_COMPUTATION ? undefined : this.$untrackedValue$
-      ) as T;
+      const untrackedValue = computeQrl.getFn(ctx)() as T;
       if (isPromise(untrackedValue)) {
         throw qError(QError.computedNotSync, [
           computeQrl.dev ? computeQrl.dev.file : '',
@@ -554,37 +553,90 @@ export class WrappedSignal<T> extends Signal<T> implements Subscriber {
   }
 }
 
-export type CustomSerializable<T extends { [SerializerSymbol]: (obj: any) => any }, S> = {
-  [SerializerSymbol]: (obj: T) => S;
-};
 /**
- * Called with serialized data to reconstruct an object. If it uses signals or stores, it will be
- * called when these change, and then the argument will be the previously constructed object.
- *
- * The constructed object should provide a `[SerializerSymbol]` method which provides the serialized
- * data.
+ * Serialize and deserialize custom objects.
  *
- * This function may not return a promise.
+ * If you pass a function, it will be used as the `deserialize` function.
  *
  * @public
  */
-export type ConstructorFn<
-  T extends CustomSerializable<T, any>,
-  S = ReturnType<T[typeof SerializerSymbol]>,
-> = ((data: S | undefined) => T) | ((data: S | undefined | T) => T);
+export type SerializedArg<T, S> =
+  | {
+      /**
+       * This function will be called with serialized data to reconstruct an object.
+       *
+       * If it is created for the first time, it will get the `initial` data or `undefined`.
+       *
+       * If it uses signals or stores, it will be called when these change, and then the second
+       * argument will be the previously constructed object.
+       *
+       * This function must not return a promise.
+       */
+      deserialize: (data: S | undefined, previous: T | undefined) => T;
+      /**
+       * This function will be called with the custom object to get the serialized data. You can
+       * return a promise if you need to do async work.
+       *
+       * The result may be anything that Qwik can serialize.
+       *
+       * If you do not provide it, the object will be serialized as `undefined`.
+       */
+      serialize?: (customObject: T) => S | Promise<S>;
+      /** The initial value to use when deserializing. */
+      initial?: S;
+    }
+  | ((data: S | undefined, previous: T | undefined) => T);
 
 /**
  * A signal which provides a non-serializable value. It works like a computed signal, but it is
  * handled slightly differently during serdes.
  *
  * @public
  */
-export class SerializedSignal<T extends CustomSerializable<T, any>> extends ComputedSignal<T> {
-  constructor(container: Container | null, fn: QRL<ConstructorFn<T>>) {
-    super(container, fn as unknown as ComputeQRL<T>);
+export class SerializedSignal<T, S> extends ComputedSignal<T> {
+  constructor(container: Container | null, argQrl: QRLInternal<SerializedArg<T, S>>) {
+    super(container, argQrl as unknown as ComputeQRL<T>);
+  }
+  $didInitialize$: boolean = false;
+
+  $computeIfNeeded$(): boolean {
+    if (!this.$invalid$) {
+      return false;
+    }
+    throwIfQRLNotResolved(this.$computeQrl$);
+    const arg = (this.$computeQrl$ as any as QRLInternal<SerializedArg<T, S>>).resolved!;
+    let deserialize, initial;
+    if (typeof arg === 'function') {
+      deserialize = arg;
+    } else {
+      deserialize = arg.deserialize;
+      initial = arg.initial;
+    }
+    const currentValue =
+      this.$untrackedValue$ === NEEDS_COMPUTATION ? initial : this.$untrackedValue$;
+    const untrackedValue = trackSignal(
+      () =>
+        this.$didInitialize$
+          ? deserialize(undefined, currentValue as T)
+          : deserialize(currentValue as S, undefined),
+      this,
+      EffectProperty.VNODE,
+      this.$container$!
+    );
+    DEBUG && log('SerializedSignal.$compute$', untrackedValue);
+    this.$invalid$ = false;
+    const didChange = untrackedValue !== this.$untrackedValue$;
+    if (didChange) {
+      this.$untrackedValue$ = untrackedValue;
+    }
+    return didChange;
   }
 }
 
+// TODO move to serializer
+export type CustomSerializable<T extends { [SerializerSymbol]: (obj: any) => any }, S> = {
+  [SerializerSymbol]: (obj: T) => S;
+};
 /** @internal */
 export const isSerializerObj = <T extends { [SerializerSymbol]: (obj: any) => any }, S>(
   obj: unknown