docs: OpenStream benchmark reports, published per release (#150)

Adds a reports section to the OpenStream spec so its claims rest on a
reproducible measurement rather than an assertion. Each report is a run of
the envelope over a defined corpus on real hardware: proof that
decompression restores every byte, that an incompressible input costs only
the framing overhead, that a compressible one saves what it claims against
the complete wire size, and how long each codec takes.

- docs/openstream/reports/ holds a machine-readable <id>.json (canonical,
  with a versioned schema) and a rendered <id>.md per report, plus a README
  on the shape and on submitting one. The seed report is nixamp 0.17.1 over
  the synthetic corpus, labelled synthetic so no one reads a padded-fixture
  number as production.
- The site renders them at /docs/openstream/reports (index) and
  /docs/openstream/reports/<id> (one report), under the dynamic /docs/[slug]
  tree so the reports routes never shadow a spec's own doc page. A small
  lib/reports.ts reads the JSON at build time; REPORTED_SPECS keeps the
  route surface explicit. sitemap includes the index and every report.
- The spec doc gains a Benchmark reports section linking there, and repeats
  the honest caveats: OpenStream frames Zstandard and gzip rather than being
  a new algorithm, synthetic padding flatters a codec, an efficient real
  feed saves little, and round-trip exactness is the one pass/fail.

The report format is produced by `nixamp compression benchmark` (in the
nixamp repo); a release runs it and commits the two files here.


Claude-Session: https://claude.ai/code/session_01MxNif5tsYq4LczgG7aE8Jp

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Anthony Ettinger 2026-09-12 06:23:46 -07:00 • committed by GitHub
parent 692bfa0a2a
commit 9f42ce222a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1145 additions and 1 deletions

View file

@ -0,0 +1,75 @@
import Link from "next/link";
import { notFound } from "next/navigation";
import type { ReactNode } from "react";
import type { Metadata } from "next";
import { marked } from "marked";
import { hasReports, readReportJson, readReportMarkdown, reportIds, REPORTED_SPECS } from "@/lib/reports";
import { SiteShell } from "@/components/site-shell";
import { sanitizeRenderedHtml } from "@/lib/html";
// Statically generate every published report of every reported spec.
export function generateStaticParams(): Array<{ slug: string; id: string }> {
const out: Array<{ slug: string; id: string }> = [];
for (const slug of REPORTED_SPECS) {
for (const id of reportIds(slug)) out.push({ slug, id });
}
return out;
}
export const dynamicParams = false;
export async function generateMetadata({
params,
}: {
params: Promise<{ slug: string; id: string }>;
}): Promise<Metadata> {
const { slug, id } = await params;
const r = readReportJson(slug, id);
if (!r) return { title: "Not found · LogicSRC" };
return {
title: `${r.implementation.name} ${r.implementation.version} benchmark · ${slug} · LogicSRC`,
description: `Reproducible ${slug} benchmark: ${r.environment.runtime} on ${r.environment.os}, generated ${r.generatedAt}.`,
alternates: { canonical: `/docs/${slug}/reports/${id}` },
};
}
export default async function ReportPage({
params,
}: {
params: Promise<{ slug: string; id: string }>;
}): Promise<ReactNode> {
const { slug, id } = await params;
if (!hasReports(slug)) notFound();
const md = readReportMarkdown(slug, id);
const json = readReportJson(slug, id);
if (!md || !json) notFound();
const rawHtml = await marked.parse(md);
const html = sanitizeRenderedHtml(rawHtml);
const repoJson = `https://github.com/profullstack/logicsrc/blob/master/docs/${slug}/reports/${id}.json`;
return (
<SiteShell active="Docs">
<article className="band" style={{ maxWidth: "48rem" }}>
<p style={{ marginBottom: "1.5rem" }}>
<Link href={`/docs/${slug}/reports`} style={{ color: "#5b6b7a", textDecoration: "none" }}>
← Benchmark reports
</Link>
</p>
<div
className="blog-content"
style={{ lineHeight: 1.7 }}
dangerouslySetInnerHTML={{ __html: html }}
/>
<p style={{ marginTop: "2rem", color: "#5b6b7a" }}>
Machine-readable source:{" "}
<a href={repoJson} rel="noreferrer">
{id}.json
</a>{" "}
(schema {json.schema}). Reproduce it by running the reference
implementation&apos;s benchmark and comparing on your own hardware.
</p>
</article>
</SiteShell>
);
}

View file

@ -0,0 +1,80 @@
import Link from "next/link";
import { notFound } from "next/navigation";
import type { ReactNode } from "react";
import type { Metadata } from "next";
import { docTitle, readDoc } from "@/lib/docs";
import { hasReports, listReports, REPORTED_SPECS } from "@/lib/reports";
import { SiteShell } from "@/components/site-shell";
// One reports index per spec that has one. Kept under the dynamic /docs/[slug]
// tree so it never shadows the spec's own doc page.
export function generateStaticParams(): Array<{ slug: string }> {
return REPORTED_SPECS.map((slug) => ({ slug }));
}
export const dynamicParams = false;
export async function generateMetadata({
params,
}: {
params: Promise<{ slug: string }>;
}): Promise<Metadata> {
const { slug } = await params;
return {
title: `Benchmark reports · ${slug} · LogicSRC`,
description: `Reproducible benchmark reports published with each release of the ${slug} reference implementation.`,
alternates: { canonical: `/docs/${slug}/reports` },
};
}
export default async function ReportsIndex({
params,
}: {
params: Promise<{ slug: string }>;
}): Promise<ReactNode> {
const { slug } = await params;
if (!hasReports(slug)) notFound();
const md = readDoc(slug);
const specTitle = md ? docTitle(md, slug) : slug;
const reports = listReports(slug);
return (
<SiteShell active="Docs">
<div className="band" style={{ maxWidth: "48rem" }}>
<p style={{ marginBottom: "1.5rem" }}>
<Link href={`/docs/${slug}`} style={{ color: "#5b6b7a", textDecoration: "none" }}>
← {specTitle}
</Link>
</p>
<div className="section-head">
<h2>Benchmark reports</h2>
<p>
Each report is a reproducible run of the {specTitle} benchmark over a
defined corpus, published with a release so the standard&apos;s claims
rest on a measurement rather than an assertion. Anyone can reproduce
one; the machine-readable JSON and the run&apos;s environment travel
with every report.
</p>
</div>
{reports.length === 0 ? (
<p style={{ color: "#41505d" }}>No reports published yet.</p>
) : (
<ul style={{ listStyle: "none", margin: 0, padding: 0 }}>
{reports.map((r) => (
<li key={r.id} style={{ padding: "1.25rem 0", borderTop: "1px solid #e3e6e0" }}>
<Link href={`/docs/${slug}/reports/${r.id}`} style={{ color: "inherit", textDecoration: "none" }}>
<h3 style={{ margin: "0 0 0.35rem", fontSize: "1.15rem", color: "#101418" }}>
{r.implementation}
</h3>
</Link>
<p style={{ color: "#41505d", margin: 0 }}>
{new Date(r.generatedAt).toISOString().slice(0, 10)} · {r.headline}
</p>
</li>
))}
</ul>
)}
</div>
</SiteShell>
);
}

View file

@ -1,6 +1,7 @@
import type { MetadataRoute } from "next";
import { publicClient } from "@/lib/supabase";
import { DOC_SLUGS } from "@/lib/docs";
import { REPORTED_SPECS, reportIds } from "@/lib/reports";
export const dynamic = "force-dynamic";
@ -49,6 +50,15 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
priority: 0.6,
}));
// Benchmark reports: the index per reported spec, and each published report.
const reportEntries: MetadataRoute.Sitemap = [];
for (const slug of REPORTED_SPECS) {
reportEntries.push({ url: `${base}/docs/${slug}/reports`, changeFrequency: "monthly", priority: 0.5 });
for (const id of reportIds(slug)) {
reportEntries.push({ url: `${base}/docs/${slug}/reports/${id}`, changeFrequency: "yearly", priority: 0.4 });
}
}
let postEntries: MetadataRoute.Sitemap = [];
try {
const supabase = publicClient();
@ -68,5 +78,5 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
postEntries = [];
}
return [...staticEntries, ...docEntries, ...postEntries];
return [...staticEntries, ...docEntries, ...reportEntries, ...postEntries];
}

View file

@ -0,0 +1,116 @@
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
// Benchmark reports published alongside a spec, at docs/<spec>/reports/.
// Each report is a machine-readable <id>.json (the canonical artifact,
// produced by a reference implementation and committed on a release) and a
// rendered <id>.md that this site displays. Read at build time, so the
// deployed image has no runtime filesystem dependency.
const DOCS_DIR = resolve(process.cwd(), "../../docs");
// Specs that carry a reports section. Kept explicit so a stray directory
// never becomes a route.
export const REPORTED_SPECS = ["openstream"] as const;
export type ReportedSpec = (typeof REPORTED_SPECS)[number];
export function hasReports(spec: string): spec is ReportedSpec {
return (REPORTED_SPECS as readonly string[]).includes(spec);
}
function reportsDir(spec: string): string {
return resolve(DOCS_DIR, spec, "reports");
}
const ID = /^[a-z0-9][a-z0-9._-]{0,80}$/;
/** The report ids published for a spec, newest first by filename. */
export function reportIds(spec: string): string[] {
if (!hasReports(spec)) return [];
const dir = reportsDir(spec);
if (!existsSync(dir)) return [];
const ids = new Set<string>();
for (const name of readdirSync(dir)) {
const m = /^(.+)\.json$/.exec(name);
if (m && ID.test(m[1] as string)) ids.add(m[1] as string);
}
return [...ids].sort().reverse();
}
export interface ReportEnvironment {
runtime: string;
zstd: string;
zlib: string;
os: string;
arch: string;
cpu: string;
cores: number;
memoryGiB: number;
}
export interface ReportModeSummary {
mode: string;
level: number;
wireBytes: number;
savingsPercent: number;
roundTrip: boolean;
encodeMs: number;
decodeMs: number;
}
export interface Report {
schema: number;
spec: string;
specVersion: string;
generatedAt: string;
implementation: { name: string; version: string };
environment: ReportEnvironment;
summary: { corpusBytes: number; byMode: ReportModeSummary[] };
caveats: string[];
// Other fields (samples, policy, envelope) are present in the JSON but not
// needed for the listing; the detail page renders the committed Markdown.
[key: string]: unknown;
}
export function readReportJson(spec: string, id: string): Report | null {
if (!hasReports(spec) || !ID.test(id)) return null;
try {
return JSON.parse(readFileSync(resolve(reportsDir(spec), `${id}.json`), "utf8")) as Report;
} catch {
return null;
}
}
export function readReportMarkdown(spec: string, id: string): string | null {
if (!hasReports(spec) || !ID.test(id)) return null;
try {
return readFileSync(resolve(reportsDir(spec), `${id}.md`), "utf8");
} catch {
return null;
}
}
export interface ReportSummary {
id: string;
generatedAt: string;
implementation: string;
/** The best round-tripping saving on the corpus, for the listing line. */
headline: string;
}
export function listReports(spec: string): ReportSummary[] {
const out: ReportSummary[] = [];
for (const id of reportIds(spec)) {
const r = readReportJson(spec, id);
if (!r) continue;
const best = r.summary?.byMode
?.filter((m) => m.roundTrip && m.mode !== "stored")
.sort((a, b) => b.savingsPercent - a.savingsPercent)[0];
out.push({
id,
generatedAt: r.generatedAt,
implementation: `${r.implementation.name} ${r.implementation.version}`,
headline: best ? `${best.mode} saved ${best.savingsPercent}% across the corpus` : "no codec beat stored on this corpus",
});
}
return out;
}