App Shell Model¶

// Generates dist/sw.js from src/sw.js with a precache manifest.
// Requires Node.js 20 or later (readdir with { recursive: true }).
import { createHash } from "node:crypto";
import { readdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";

const DIST = path.resolve("dist");
const SW_SOURCE = path.resolve("src/sw.js");

// What goes into the precache: the shell and everything it needs offline.
const INCLUDE = /\.(?:html|js|css|woff2|svg|png|webmanifest)$/;
const EXCLUDE = [
  /^sw\.js$/, // the worker never precaches itself
  /\.map$/, // source maps are for DevTools, not users
  /^img\/content\//, // content images are runtime-cached, not precached
  /^basic\//, // the no-JavaScript fallback is server-rendered
];
// Files whose names contain a content hash, like app.8d41e0c7.js.
const FINGERPRINTED = /\.[0-9a-f]{8,}\.[a-z0-9]+$/;

const sha = (data) => createHash("sha256").update(data).digest("hex");

async function listFiles(dir) {
  const entries = await readdir(dir, { recursive: true, withFileTypes: true });
  return entries
    .filter((entry) => entry.isFile())
    .map((entry) => path.join(entry.parentPath ?? entry.path, entry.name))
    .map((file) => path.relative(dir, file).split(path.sep).join("/"))
    .filter((file) => INCLUDE.test(file) && !EXCLUDE.some((re) => re.test(file)))
    .sort(); // stable order, so the same input always yields the same output
}

const files = await listFiles(DIST);
if (!files.includes("shell.html")) {
  throw new Error("dist/shell.html is missing: the app shell must be precached");
}

// 1. Build ID over every precached file, so any change produces a new worker.
const buildHash = createHash("sha256");
for (const file of files) {
  buildHash.update(file).update(await readFile(path.join(DIST, file)));
}
const BUILD_ID = buildHash.digest("hex").slice(0, 12);

// 2. Stamp the build ID into the shell before computing its revision.
const shellPath = path.join(DIST, "shell.html");
const shellSource = await readFile(shellPath, "utf8");
await writeFile(shellPath, shellSource.replaceAll("%BUILD_ID%", BUILD_ID));

// 3. Manifest entries: fingerprinted files need no revision (null).
const manifest = [];
for (const file of files) {
  const url = `/${file}`;
  const revision = FINGERPRINTED.test(file)
    ? null
    : sha(await readFile(path.join(DIST, file))).slice(0, 16);
  manifest.push({ url, revision });
}

// 4. Inject into the worker source.
const swSource = await readFile(SW_SOURCE, "utf8");
for (const marker of ["self.__SHELL_MANIFEST", "%BUILD_ID%"]) {
  if (!swSource.includes(marker)) throw new Error(`src/sw.js lacks the ${marker} injection point`);
}
const swOutput = swSource
  .replace("self.__SHELL_MANIFEST", JSON.stringify(manifest))
  .replaceAll("%BUILD_ID%", BUILD_ID);
await writeFile(path.join(DIST, "sw.js"), swOutput);

console.log(`sw.js: ${manifest.length} precached files, build ${BUILD_ID}`);

/* App shell service worker. Processed by scripts/build-sw.mjs. */
const BUILD_ID = "%BUILD_ID%";
const MANIFEST = self.__SHELL_MANIFEST; // [{ url, revision }]
const SHELL_URL = "/shell.html";

const PRECACHE = `precache-${BUILD_ID}`;
const API_CACHE = "api-v1";
const IMAGE_CACHE = "images-v1";
const IMAGE_CACHE_MAX_ENTRIES = 200;
const API_TIMEOUT_MS = 3000;
const REVISION_HEADER = "X-Precache-Revision";

const PRECACHED_PATHS = new Set(MANIFEST.map((entry) => entry.url));

// Navigations that must reach the server instead of getting the shell.
// Matched against pathname + search, like Workbox's NavigationRoute.
const SHELL_DENYLIST = [
  /^\/api\//, // JSON opened directly in a tab
  /^\/auth\//, // OAuth redirects and login forms are server-rendered
  /^\/basic\//, // the no-JavaScript version of the app
  /^\/admin(?:\/|$)/, // a separate, server-rendered application
  /\/[^/?]+\.[a-z0-9]+(?:\?|$)/i, // URLs that look like files: /export.csv, /feed.xml
];

// ---------------------------------------------------------------- install

self.addEventListener("install", (event) => {
  event.waitUntil(precache());

  // Chrome 123+ and Safari 27: serve fingerprinted assets straight from the
  // precache without starting this worker. Browsers without static routing
  // fall through to the fetch handler below, which does the same thing.
  if (typeof event.addRoutes === "function") {
    event
      .addRoutes([{ condition: { urlPattern: "/assets/*" }, source: { cacheName: PRECACHE } }])
      .catch((error) => console.warn("Static routes rejected", error));
  }
});

async function precache() {
  const cache = await caches.open(PRECACHE);
  const previous = await Promise.all(
    (await caches.keys())
      .filter((name) => name.startsWith("precache-") && name !== PRECACHE)
      .map((name) => caches.open(name)),
  );

  try {
    await Promise.all(MANIFEST.map((entry) => precacheEntry(cache, previous, entry)));
  } catch (error) {
    // Never leave a half-filled cache behind: a failed install is retried on
    // the next update check, and the old worker keeps serving meanwhile.
    await caches.delete(PRECACHE);
    throw error;
  }
}

async function precacheEntry(cache, previousCaches, { url, revision }) {
  const wanted = String(revision);

  // Reuse an identical copy from the previous release instead of downloading.
  for (const old of previousCaches) {
    const hit = await old.match(url);
    if (hit && hit.headers.get(REVISION_HEADER) === wanted) {
      await cache.put(url, hit);
      return;
    }
  }

  // Unversioned files bypass the HTTP cache ("reload") so a stale copy can
  // never be precached; fingerprinted files are immutable and may use it.
  const response = await fetch(url, { cache: revision === null ? "default" : "reload" });
  if (!response.ok) {
    throw new Error(`Precache request for ${url} failed with ${response.status}`);
  }

  // Re-wrapping the response adds the revision header and drops the
  // "redirected" flag: a redirected response cannot answer a navigation.
  const headers = new Headers(response.headers);
  headers.set(REVISION_HEADER, wanted);
  await cache.put(
    url,
    new Response(response.body, {
      status: response.status,
      statusText: response.statusText,
      headers,
    }),
  );
}

// --------------------------------------------------------------- activate

self.addEventListener("activate", (event) => {
  event.waitUntil(
    (async () => {
      // Navigations are answered from the cache, so a navigation preload
      // request would be wasted. The setting persists on the registration,
      // so switch it off explicitly in case an earlier version enabled it.
      if (self.registration.navigationPreload) {
        await self.registration.navigationPreload.disable();
      }
      const names = await caches.keys();
      await Promise.all(
        names
          .filter((name) => name.startsWith("precache-") && name !== PRECACHE)
          .map((name) => caches.delete(name)),
      );
    })(),
  );
});

// ------------------------------------------------------------------ fetch

self.addEventListener("fetch", (event) => {
  const { request } = event;
  if (request.method !== "GET") return; // mutations always go to the network

  const url = new URL(request.url);

  if (request.mode === "navigate") {
    const sameOrigin = url.origin === self.location.origin;
    const denied = SHELL_DENYLIST.some((re) => re.test(url.pathname + url.search));
    if (sameOrigin && !denied) {
      event.respondWith(serveShell(request));
    }
    return; // denied navigations get the browser's default network handling
  }

  if (url.origin !== self.location.origin) return; // third parties: not our business

  if (PRECACHED_PATHS.has(url.pathname)) {
    event.respondWith(fromPrecache(request, url.pathname));
  } else if (url.pathname.startsWith("/api/")) {
    event.respondWith(networkFirst(event, API_CACHE, API_TIMEOUT_MS));
  } else if (request.destination === "image") {
    event.respondWith(cacheFirst(event, IMAGE_CACHE, IMAGE_CACHE_MAX_ENTRIES));
  }
  // Everything else falls through to the network.
});

async function serveShell(request) {
  const cache = await caches.open(PRECACHE);
  const shell = await cache.match(SHELL_URL);
  if (shell) return shell;

  // The shell should always be there; if it is not (caches cleared by hand,
  // a cleanup bug), the server's own SPA fallback is the next best thing.
  try {
    return await fetch(request);
  } catch {
    return new Response(
      "<!doctype html><meta charset=utf-8><title>Offline</title><h1>You are offline</h1>",
      { status: 503, headers: { "Content-Type": "text/html; charset=utf-8" } },
    );
  }
}

async function fromPrecache(request, pathname) {
  const cache = await caches.open(PRECACHE);
  // Query strings are not part of precache keys (for example ?utm_source).
  return (await cache.match(pathname)) ?? fetch(request);
}

async function networkFirst(event, cacheName, timeoutMs) {
  const cache = await caches.open(cacheName);
  const network = fetch(event.request);
  // Write successful responses to the cache without delaying the page: the
  // clone is taken in the first reaction to the response, before the page can
  // read the body. API data is per user: clear this cache on logout (see the
  // "LOGOUT" message below). A failed write (quota) must not fail the request.
  const update = network
    .then((response) => (response.ok ? cache.put(event.request, response.clone()) : undefined))
    .catch(() => undefined);
  // Keep the worker alive until the cache write settles, even when the cached
  // copy wins the race below, so the cache is refreshed for next time.
  event.waitUntil(update);

  let timer;
  const timeout = new Promise((resolve) => {
    timer = setTimeout(resolve, timeoutMs, null);
  });
  try {
    const winner = await Promise.race([network, timeout]);
    if (winner) return winner;
    // Slow network: answer from the cache if possible, otherwise keep waiting.
    return (await cache.match(event.request)) ?? (await network);
  } catch (error) {
    const cached = await cache.match(event.request);
    if (cached) return cached;
    throw error; // the page's fetch() rejects, and the router shows its offline state
  } finally {
    clearTimeout(timer);
  }
}

async function cacheFirst(event, cacheName, maxEntries) {
  const cache = await caches.open(cacheName);
  const cached = await cache.match(event.request);
  if (cached) return cached;

  const response = await fetch(event.request);
  if (response.ok) {
    event.waitUntil(
      cache.put(event.request, response.clone()).then(() => trimCache(cache, maxEntries)),
    );
  }
  return response;
}

async function trimCache(cache, maxEntries) {
  const keys = await cache.keys(); // insertion order: oldest first
  const excess = keys.length - maxEntries;
  if (excess > 0) {
    await Promise.all(keys.slice(0, excess).map((key) => cache.delete(key)));
  }
}

// --------------------------------------------------------------- messages

self.addEventListener("message", (event) => {
  switch (event.data?.type) {
    case "SKIP_WAITING": // sent by the page when the user accepts an update
      self.skipWaiting();
      break;
    case "LOGOUT": // per-user data must not survive a logout
      event.waitUntil(caches.delete(API_CACHE));
      break;
    case "GET_BUILD_ID":
      event.ports[0]?.postMessage(BUILD_ID);
      break;
  }
});

// Router and views for the app shell. Bundled to /assets/app.<hash>.js.
const view = document.getElementById("view");
let inflight = null; // AbortController of the route currently loading
let firstRender = true;

const routes = [
  {
    pattern: /^\/$/,
    load: (params, signal) => fetchJSON("/api/notes?limit=20", signal),
    render: renderNoteList,
  },
  {
    pattern: /^\/notes\/([\w-]+)$/,
    load: ([id], signal) => fetchJSON(`/api/notes/${encodeURIComponent(id)}`, signal),
    render: renderNote,
  },
  {
    pattern: /^\/settings$/,
    load: async () => null, // purely local view: nothing to fetch
    render: renderSettings,
  },
];

class HttpError extends Error {
  constructor(status) {
    super(`HTTP ${status}`);
    this.status = status;
  }
}

async function fetchJSON(url, signal) {
  const response = await fetch(url, { signal, headers: { Accept: "application/json" } });
  if (!response.ok) throw new HttpError(response.status);
  return response.json();
}

const escapeHTML = (value) =>
  String(value).replace(/[&<>"']/g, (c) => `&#${c.charCodeAt(0)};`);

// ------------------------------------------------------------------ views

function renderNoteList(notes) {
  return {
    title: "Notes",
    html: `<h1>Notes</h1>
      <ul class="note-list">
        ${notes
          .map(
            (n) => `<li class="list-row"><a href="/notes/${encodeURIComponent(n.id)}">
              ${escapeHTML(n.title)}</a></li>`,
          )
          .join("")}
      </ul>`,
  };
}

function renderNote(note) {
  // On the initial render the hero image is the likely LCP element: give it
  // high priority. Later (soft) navigations keep the default priority.
  const priority = firstRender ? ' fetchpriority="high"' : "";
  // Number() keeps API-supplied dimensions from injecting markup; the explicit
  // width and height reserve the image's box and prevent a layout shift.
  const w = Number(note.image?.width);
  const h = Number(note.image?.height);
  const size = w > 0 && h > 0 ? ` width="${w}" height="${h}"` : "";
  const hero = note.image
    ? `<img src="${escapeHTML(note.image.src)}"${size}
         alt="${escapeHTML(note.image.alt)}"${priority}>`
    : "";
  return {
    title: note.title,
    html: `<article><h1>${escapeHTML(note.title)}</h1>${hero}
      <div class="note-body">${escapeHTML(note.text)}</div></article>`,
  };
}

function renderSettings() {
  return { title: "Settings", html: `<h1>Settings</h1><p>Settings are stored on this device.</p>` };
}

function renderProblem(error) {
  if (error instanceof HttpError && error.status === 404) {
    return { title: "Not found", html: `<h1>Not found</h1><p><a href="/">Back to your notes</a></p>` };
  }
  if (!navigator.onLine || error instanceof TypeError) {
    // fetch() rejects with a TypeError on network failure (and the service
    // worker found nothing in its cache either).
    return {
      title: "Offline",
      html: `<h1>You are offline</h1><p>This note has not been saved for offline use yet.</p>
        <button type="button" data-action="retry">Try again</button>`,
    };
  }
  return { title: "Something went wrong", html: `<h1>Something went wrong</h1>
    <button type="button" data-action="retry">Try again</button>` };
}

// ----------------------------------------------------------------- router

async function render(pathname) {
  inflight?.abort(); // a newer navigation supersedes the pending one
  const controller = new AbortController();
  inflight = controller;

  const match = routes
    .map((route) => ({ route, params: pathname.match(route.pattern) }))
    .find(({ params }) => params);

  view.setAttribute("aria-busy", "true");
  let result;
  try {
    if (!match) throw new HttpError(404);
    const data = await match.route.load(match.params.slice(1), controller.signal);
    result = match.route.render(data);
  } catch (error) {
    if (error.name === "AbortError") return; // superseded: render nothing
    console.error("Route failed", pathname, error);
    result = renderProblem(error);
  }

  view.innerHTML = result.html;
  view.setAttribute("aria-busy", "false");
  document.title = `${result.title} · Acme Notes`;
  performance.mark("content-rendered", { detail: { route: match?.route.pattern.source ?? "404" } });

  if (!firstRender) {
    // Soft navigation: move focus to the new content for assistive technology.
    view.focus({ preventScroll: true });
    window.scrollTo(0, 0);
  }
  firstRender = false;
}

function navigate(url, { replace = false } = {}) {
  if (replace) history.replaceState(null, "", url);
  else history.pushState(null, "", url);
  render(location.pathname);
}

document.addEventListener("click", (event) => {
  const retry = event.target.closest("[data-action=retry]");
  if (retry) {
    render(location.pathname);
    return;
  }
  const link = event.target.closest("a[href]");
  if (
    !link ||
    event.defaultPrevented ||
    event.button !== 0 ||
    event.metaKey || event.ctrlKey || event.shiftKey || event.altKey || // new tab/window
    link.target || link.hasAttribute("download") ||
    link.origin !== location.origin
  ) {
    return; // let the browser handle it
  }
  const url = new URL(link.href);
  if (url.pathname === location.pathname && url.search === location.search) {
    event.preventDefault();
    return;
  }
  // Paths the worker deny-lists are server territory: full navigation. Keep
  // this in sync with SHELL_DENYLIST in sw.js (sections and file-like URLs).
  if (
    /^\/(?:api|auth|basic|admin)(?:\/|$)/.test(url.pathname) ||
    /\/[^/]+\.[a-z0-9]+$/i.test(url.pathname)
  ) {
    return;
  }
  event.preventDefault();
  navigate(url.href);
});

addEventListener("popstate", () => render(location.pathname));

render(location.pathname);

// ------------------------------------------------------ worker registration

if ("serviceWorker" in navigator) {
  // Register after load so a first-time visitor's precaching does not compete
  // with the page's own requests for bandwidth.
  addEventListener("load", () => {
    navigator.serviceWorker
      .register("/sw.js", { scope: "/" })
      .catch((error) => console.error("Service worker registration failed", error));
  });
}

self.addEventListener("install", (event) => {
  event.waitUntil(precache());
  if (typeof event.addRoutes === "function") {
    event
      .addRoutes([
        // Exact-URL cache lookups in this release's precache; a miss goes to
        // the network, never to the fetch handler.
        { condition: { urlPattern: "/assets/*" }, source: { cacheName: PRECACHE } },
        // Uncached content images: no reason to wake the worker.
        { condition: { urlPattern: "/img/content/*" }, source: "network" },
      ])
      .catch((error) => console.warn("Static routes rejected", error));
  }
});

// Streaming shell: cached head + network fragment + cached foot.
const PRECACHE = "precache-%BUILD_ID%";
const FRAGMENT_HEADER_VALUE = "fragment";

self.addEventListener("activate", (event) => {
  event.waitUntil(
    (async () => {
      if (self.registration.navigationPreload) {
        await self.registration.navigationPreload.enable();
        // The server returns only the <main> fragment when it sees this value.
        await self.registration.navigationPreload.setHeaderValue(FRAGMENT_HEADER_VALUE);
      }
    })(),
  );
});

self.addEventListener("fetch", (event) => {
  const { request } = event;
  if (request.mode !== "navigate" || request.method !== "GET") return;
  const url = new URL(request.url);
  if (url.origin !== self.location.origin || /^\/(?:api|auth|admin)(?:\/|$)/.test(url.pathname)) {
    return; // must not fall through with preload enabled: see note below
  }
  event.respondWith(streamPage(event));
});

async function streamPage(event) {
  const cache = await caches.open(PRECACHE);
  const [head, foot] = await Promise.all([
    cache.match("/partials/head.html"),
    cache.match("/partials/foot.html"),
  ]);
  if (!head || !foot) {
    // Partials missing: fall back to a normal full page from the network.
    // The preload response cannot be used here, because the server answered
    // it with a bare fragment. Let it settle so Chromium does not log a
    // cancelled-preload warning, then request the full page.
    event.waitUntil(Promise.resolve(event.preloadResponse).catch(() => undefined));
    return fetch(event.request);
  }

  const fragment = contentFragment(event, cache); // starts now, awaited later
  const { readable, writable } = new TransformStream();

  const pump = (async () => {
    try {
      await head.body.pipeTo(writable, { preventClose: true });
      const body = await fragment;
      await body.body.pipeTo(writable, { preventClose: true });
      await foot.body.pipeTo(writable, { preventClose: true });
      await writable.close();
    } catch (error) {
      // The head is already on screen; aborting ends the document early.
      console.error("Streaming failed", error);
      await writable.abort(error).catch(() => undefined);
    }
  })();
  event.waitUntil(pump); // keep the worker alive until the stream is complete

  return new Response(readable, {
    headers: { "Content-Type": "text/html; charset=utf-8" },
  });
}

async function contentFragment(event, cache) {
  try {
    // Navigation preload started this request in parallel with worker start-up.
    const preloaded = await event.preloadResponse;
    if (preloaded) return preloaded;
    // No preload (unsupported or disabled): ask for the fragment explicitly.
    return await fetch(event.request.url, {
      headers: { "Service-Worker-Navigation-Preload": FRAGMENT_HEADER_VALUE },
      credentials: "same-origin",
    });
  } catch {
    // Offline: an offline fragment keeps the shell usable.
    return (
      (await cache.match("/partials/offline.html")) ??
      new Response("<main id=view><h1>You are offline</h1></main>", {
        headers: { "Content-Type": "text/html; charset=utf-8" },
      })
    );
  }
}

// Express 5 middleware: one handler, two representations per URL.
import { renderFragment, renderFullPage } from "./render.mjs";

export async function pages(req, res, next) {
  if (req.method !== "GET" || req.path.startsWith("/api/")) return next();
  try {
    const wantsFragment = req.get("Service-Worker-Navigation-Preload") === "fragment";
    const { status, html } = wantsFragment
      ? await renderFragment(req.path, req.user)
      : await renderFullPage(req.path, req.user);
    res
      .status(status)
      .set("Vary", "Service-Worker-Navigation-Preload")
      .set("Cache-Control", "private, no-cache")
      .type("html")
      .send(html);
  } catch (error) {
    next(error);
  }
}