Enforce OpenContext HTTP byte limit while streaming (#136)

packages/opencontext/src/adapters/http.test.ts

@@ -0,0 +1,33 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { httpAdapter } from "./http.js";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("http adapter response limits", () => {
+  it("enforces the byte limit for multibyte responses without a content-length header", async () => {
+    const content = "é".repeat(3 * 1024 * 1024);
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(content)));
+
+    await expect(
+      httpAdapter.load("https://example.com/context.md", {
+        dir: process.cwd(),
+        offline: false,
+        config: {}
+      })
+    ).rejects.toThrow(/response exceeds the 5242880 byte limit/);
+  });
+
+  it("decodes responses within the byte limit", async () => {
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("café")));
+
+    await expect(
+      httpAdapter.load("https://example.com/context.md", {
+        dir: process.cwd(),
+        offline: false,
+        config: {}
+      })
+    ).resolves.toMatchObject({ content: "café", trust: "untrusted" });
+  });
+});

packages/opencontext/src/adapters/http.ts

@@ -20,6 +20,38 @@ export class OfflineError extends Error {
 const DEFAULT_TIMEOUT_MS = 10_000;
 const MAX_BYTES = 5 * 1024 * 1024;
 
+async function readBody(response: Response, uri: string): Promise<string> {
+  if (!response.body) return "";
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      totalBytes += value.byteLength;
+      if (totalBytes > MAX_BYTES) {
+        await reader.cancel().catch(() => undefined);
+        throw new Error(`Refusing to load ${uri}: response exceeds the ${MAX_BYTES} byte limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(bytes);
+}
+
 export const httpAdapter: Adapter = {
   name: "http",
   schemes: ["http", "https"],
@@ -39,6 +71,7 @@ export const httpAdapter: Adapter = {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), timeoutMs);
 
+    try {
       let response: Response;
       try {
         response = await fetch(url, {
@@ -50,8 +83,6 @@ export const httpAdapter: Adapter = {
         });
       } catch (error) {
         throw new Error(`Failed to fetch ${uri}: ${(error as Error).message}`);
-    } finally {
-      clearTimeout(timer);
       }
 
       if (!response.ok) {
@@ -63,10 +94,7 @@ export const httpAdapter: Adapter = {
         throw new Error(`Refusing to load ${uri}: ${declaredLength} bytes exceeds the ${MAX_BYTES} byte limit.`);
       }
 
-    const content = await response.text();
-    if (content.length > MAX_BYTES) {
-      throw new Error(`Refusing to load ${uri}: response exceeds the ${MAX_BYTES} byte limit.`);
-    }
+      const content = await readBody(response, uri);
 
       return {
         content,
@@ -75,5 +103,8 @@ export const httpAdapter: Adapter = {
         retrievedAt: new Date().toISOString(),
         trust: (ctx.config.trust as AdapterResult["trust"]) ?? "untrusted"
       };
+    } finally {
+      clearTimeout(timer);
+    }
   }
 };