Add feed discovery plugin

plugins/feed-discovery/src/url-safety.ts

@@ -0,0 +1,89 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+const BLOCKED_HOSTS = new Set(["localhost", "metadata.google.internal"]);
+
+export async function assertSafeHttpUrl(input: string) {
+  const url = new URL(input);
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`Unsupported URL protocol: ${url.protocol}`);
+  }
+
+  const hostname = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1");
+  if (BLOCKED_HOSTS.has(hostname) || hostname.endsWith(".localhost")) {
+    throw new Error(`Blocked internal hostname: ${hostname}`);
+  }
+
+  if (isBlockedIp(hostname)) {
+    throw new Error(`Blocked internal IP address: ${hostname}`);
+  }
+
+  const addresses = await lookup(hostname, { all: true, verbatim: true });
+  for (const address of addresses) {
+    if (isBlockedIp(address.address)) {
+      throw new Error(`Blocked internal resolved address: ${address.address}`);
+    }
+  }
+
+  return url;
+}
+
+export function canonicalizeUrl(input: string, base?: string) {
+  const url = new URL(input, base);
+  url.hash = "";
+
+  for (const param of [...url.searchParams.keys()]) {
+    if (/^(utm_|fbclid$|gclid$|mc_cid$|mc_eid$)/i.test(param)) {
+      url.searchParams.delete(param);
+    }
+  }
+
+  if ((url.protocol === "https:" && url.port === "443") || (url.protocol === "http:" && url.port === "80")) {
+    url.port = "";
+  }
+
+  if (url.pathname !== "/" && url.pathname.endsWith("/")) {
+    url.pathname = url.pathname.slice(0, -1);
+  }
+
+  return url.toString();
+}
+
+function isBlockedIp(value: string) {
+  const kind = isIP(value);
+  if (kind === 4) {
+    const parts = value.split(".").map((part) => Number(part));
+    const [a, b] = parts;
+    return (
+      a === 0 ||
+      a === 10 ||
+      a === 127 ||
+      (a === 169 && b === 254) ||
+      (a === 172 && b >= 16 && b <= 31) ||
+      (a === 192 && b === 168) ||
+      (a === 100 && b >= 64 && b <= 127)
+    );
+  }
+
+  if (kind === 6) {
+    const normalized = value.toLowerCase();
+    const mappedDottedIpv4 = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(normalized);
+    if (mappedDottedIpv4) {
+      return isBlockedIp(mappedDottedIpv4[1]);
+    }
+    const mappedHexIpv4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(normalized);
+    if (mappedHexIpv4) {
+      const high = Number.parseInt(mappedHexIpv4[1], 16);
+      const low = Number.parseInt(mappedHexIpv4[2], 16);
+      return isBlockedIp(`${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`);
+    }
+    return (
+      normalized === "::1" ||
+      normalized.startsWith("fc") ||
+      normalized.startsWith("fd") ||
+      normalized.startsWith("fe80")
+    );
+  }
+
+  return false;
+}