Support for dynamic page loading for Multi-WACZs (#399)

Support querying pages dynamically via pages endpoint:
- If loaded pages < total pages, load additional pages via /pages
endpoint if canQueryPages is set to true on item
- If query provided, do `/pages?search=...` to search for pages. Use
pagination of 25 pages for now.

WACZ hash:
- if provided, store waczhash per page, and use for thumbnail, favicon
and top-level replay loading.

Embed: better support for force reloading, call delete with reload
directly in embed

Depends on webrecorder/wabac.js#220

Author: ikreymer

package.json

@@ -1,7 +1,7 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "2.2.5",
+  "version": "2.3.0",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
@@ -18,7 +18,7 @@
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
     "@shoelace-style/shoelace": "~2.15.1",
-    "@webrecorder/wabac": "^2.20.8",
+    "@webrecorder/wabac": "^2.21.0",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^6.3.9",

src/embed.ts

@@ -7,9 +7,9 @@ import {
 } from "lit";
 import { ifDefined } from "lit/directives/if-defined.js";
 
-import { wrapCss, updateFaviconLinks } from "./misc";
+import { wrapCss, updateFaviconLinks, apiPrefix } from "./misc";
 import { SWManager } from "./swmanager";
-import { property } from "lit/decorators.js";
+import { property, query } from "lit/decorators.js";
 import type { FavIconEventDetail } from "./types";
 import type { EmbedReplayData, EmbedReplayEvent } from "./item";
 
@@ -82,6 +82,9 @@ class Embed extends LitElement {
 
   @property({ type: Boolean }) useRuffle = false;
 
+  @query("iframe")
+  private readonly iframe?: HTMLIFrameElement | null;
+
   replayfile = defaultReplayFile;
   mainElementName = "replay-app-main";
   appName = "ReplayWeb.page";
@@ -126,17 +129,28 @@ class Embed extends LitElement {
     }
   }
 
-  fullReload() {
-    const iframe = this.renderRoot.querySelector("iframe");
-    if (iframe?.contentWindow) {
-      iframe.contentWindow.postMessage({ type: "fullReload" });
+  async fullReload() {
+    if (!this.iframe?.contentWindow) {
+      return false;
     }
+
+    const deleteURL = apiPrefix + "/c/" + this.coll + "?reload=1";
+
+    const resp = await this.iframe.contentWindow.fetch(deleteURL, {
+      method: "DELETE",
+    });
+
+    if (resp.status !== 200) {
+      return false;
+    }
+
+    this.iframe.contentWindow.location.reload();
+
+    return true;
   }
 
   handleMessage(event: IframeMessage) {
-    const iframe = this.renderRoot.querySelector("iframe");
-
-    if (iframe && event.source === iframe.contentWindow) {
+    if (this.iframe && event.source === this.iframe.contentWindow) {
       switch (event.data.type) {
         case "urlchange":
           if (this.deepLink) {

src/item.ts

@@ -85,6 +85,7 @@ export type EmbedReplayData = {
   ts?: string;
   title?: string;
   query?: string;
+  waczhash?: string;
 };
 
 export type EmbedReplayEvent = EmbedReplayData & {
@@ -229,16 +230,6 @@ class Item extends LitElement {
 
       this.observer.observe(this);
     }
-
-    if (this.embed) {
-      window.addEventListener("message", (event: MessageEvent) => {
-        if (event.source === window.parent) {
-          if (event.data.type === "fullReload") {
-            void this.deleteFully(true);
-          }
-        }
-      });
-    }
   }
 
   async runUpdateLoop() {
@@ -976,6 +967,7 @@ class Item extends LitElement {
                   sourceUrl="${this.sourceUrl || ""}"
                   url="${this.tabData.url || ""}"
                   ts="${this.tabData.ts || ""}"
+                  waczhash="${this.tabData.waczhash || ""}"
                   @coll-tab-nav="${this.onItemTabNav}"
                   id="replay"
                   @replay-loading="${this.onReplayLoading}"

src/pageentry.ts

@@ -5,7 +5,7 @@ import { property, state } from "lit/decorators.js";
 
 import "keyword-mark-element/lib/keyword-mark.js";
 
-import { getReplayLink } from "./pageutils";
+import { getPageDateTS, getReplayLink } from "./pageutils";
 
 import { wrapCss } from "./misc";
 import type { URLResource } from "./types";
@@ -192,7 +192,7 @@ class PageEntry extends LitElement {
 
   render() {
     const p = this.page!;
-    const date = this.page!.date;
+    const { date } = getPageDateTS(p);
 
     const hasSize = typeof p.size === "number";
 
@@ -245,6 +245,7 @@ class PageEntry extends LitElement {
                     "pages",
                     this.page!.url,
                     this.page!.timestamp!,
+                    this.page!.waczhash,
                   )}"
                 >
                   <p class="is-size-6 has-text-weight-bold has-text-link text">
@@ -298,16 +299,20 @@ class PageEntry extends LitElement {
     `;
   }
 
+  private getReplayPrefix() {
+    const waczhash = this.page!.waczhash ? `:${this.page!.waczhash}/` : "";
+    const ts = this.page!.timestamp || "";
+    return this.replayPrefix + "/" + waczhash + ts + "id_";
+  }
+
   private renderPageIcon() {
     if (!this.thumbnailValid) {
       return this.renderFavicon();
     }
     return html`<img
       class="thumbnail"
       @error=${() => (this.thumbnailValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/urn:thumbnail:${
-        this.page!.url
-      }`}
+      src=${`${this.getReplayPrefix()}/urn:thumbnail:${this.page!.url}`}
       loading="lazy"
     />`;
   }
@@ -319,9 +324,7 @@ class PageEntry extends LitElement {
     return html`<img
       class="favicon"
       @error=${() => (this.iconValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/${
-        this.page!.favIconUrl
-      }`}
+      src=${`${this.getReplayPrefix()}/${this.page!.favIconUrl}`}
       loading="lazy"
     />`;
   }
@@ -377,6 +380,7 @@ class PageEntry extends LitElement {
     const data = {
       url: this.page!.url,
       ts: this.page!.timestamp,
+      waczhash: this.page!.waczhash,
     };
     this.sendChangeEvent(data, reload);
     return false;