Add feed discovery plugin

plugins/feed-discovery/src/providers/index.ts

@@ -0,0 +1,45 @@
+import type { FeedDiscoveryConfig, FeedDiscoveryProvider, FeedProviderManifest } from "../types.js";
+import { ITunesPodcastProvider } from "./itunes-podcast.js";
+import { ManualCuratedProvider } from "./manual-curated.js";
+import { OpmlDirectoryProvider } from "./opml-directory.js";
+import { PodcastIndexProvider } from "./podcastindex.js";
+import { WebCandidateFeedProbeProvider } from "./web-feed-probe.js";
+
+export function createDefaultFeedProviders(config: FeedDiscoveryConfig): FeedDiscoveryProvider[] {
+  return [
+    new ManualCuratedProvider(),
+    new OpmlDirectoryProvider(config.opmlPaths),
+    new WebCandidateFeedProbeProvider(config),
+    new ITunesPodcastProvider(config),
+    new PodcastIndexProvider(config)
+  ];
+}
+
+export function providerManifests(config: FeedDiscoveryConfig): FeedProviderManifest[] {
+  return createDefaultFeedProviders(config).map((provider) => ({
+    id: provider.id,
+    name: provider.name,
+    type: provider.id.includes("podcast") || provider.id.includes("itunes") ? "podcast" : "all",
+    requiresApiKey: provider.requiresApiKey,
+    enabledByDefault: provider.enabledByDefault && (!provider.requiresApiKey || hasProviderKey(provider.id, config)),
+    description: describeProvider(provider.id)
+  }));
+}
+
+function hasProviderKey(id: string, config: FeedDiscoveryConfig) {
+  if (id === "podcastindex") {
+    return Boolean(config.podcastIndexApiKey && config.podcastIndexApiSecret);
+  }
+  return true;
+}
+
+function describeProvider(id: string) {
+  const descriptions: Record<string, string> = {
+    "manual-curated": "Searches locally curated high-trust feed sources.",
+    "opml-directory": "Searches configured local OPML files.",
+    "web-feed-probe": "Probes configured candidate homepages and direct URL queries for feeds.",
+    "itunes-podcast": "Searches the public iTunes podcast directory.",
+    podcastindex: "Searches PodcastIndex when API credentials are configured."
+  };
+  return descriptions[id] ?? "Feed discovery provider.";
+}

plugins/feed-discovery/src/providers/itunes-podcast.ts

@@ -0,0 +1,64 @@
+import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
+
+interface ITunesPodcast {
+  collectionName?: string;
+  artistName?: string;
+  feedUrl?: string;
+  collectionViewUrl?: string;
+  artworkUrl600?: string;
+  primaryGenreName?: string;
+  releaseDate?: string;
+}
+
+export class ITunesPodcastProvider implements FeedDiscoveryProvider {
+  id = "itunes-podcast";
+  name = "iTunes Podcast Search";
+  enabledByDefault = true;
+  requiresApiKey = false;
+
+  constructor(private readonly config: Pick<FeedDiscoveryConfig, "requestTimeoutMs" | "userAgent">) {}
+
+  async search(query: FeedDiscoveryQuery) {
+    if (query.type && query.type !== "all" && query.type !== "podcast") {
+      return [];
+    }
+
+    const url = new URL("https://itunes.apple.com/search");
+    url.searchParams.set("term", query.q);
+    url.searchParams.set("entity", "podcast");
+    url.searchParams.set("limit", String(Math.min(query.limit ?? 25, 50)));
+    if (query.locale) {
+      url.searchParams.set("country", query.locale.toUpperCase());
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
+    try {
+      const response = await fetch(url, {
+        signal: controller.signal,
+        headers: { "user-agent": this.config.userAgent }
+      });
+      if (!response.ok) {
+        throw new Error(`iTunes HTTP ${response.status}`);
+      }
+      const body = await response.json() as { results?: ITunesPodcast[] };
+      return (body.results ?? [])
+        .filter((item) => item.feedUrl)
+        .map((item): DiscoveredFeed => ({
+          title: item.collectionName ?? item.feedUrl ?? "Untitled podcast",
+          description: item.artistName,
+          homepageUrl: item.collectionViewUrl,
+          feedUrl: item.feedUrl as string,
+          kind: "podcast",
+          provider: this.id,
+          imageUrl: item.artworkUrl600,
+          lastPublishedAt: item.releaseDate,
+          score: 0,
+          confidence: 0.85,
+          tags: [item.primaryGenreName, "podcast"].filter((tag): tag is string => Boolean(tag))
+        }));
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+}

plugins/feed-discovery/src/providers/manual-curated.ts

@@ -0,0 +1,59 @@
+import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
+
+const CURATED_FEEDS: DiscoveredFeed[] = [
+  {
+    title: "Indie Hackers",
+    description: "Stories and discussions from founders building profitable internet businesses.",
+    homepageUrl: "https://www.indiehackers.com",
+    feedUrl: "https://www.indiehackers.com/feed",
+    kind: "blog",
+    provider: "manual-curated",
+    score: 0,
+    confidence: 0.8,
+    tags: ["indie", "startup", "microsaas", "saas"]
+  },
+  {
+    title: "Hacker News",
+    description: "Technology, startup, software, and open-source discussion.",
+    homepageUrl: "https://news.ycombinator.com",
+    feedUrl: "https://news.ycombinator.com/rss",
+    kind: "news",
+    provider: "manual-curated",
+    score: 0,
+    confidence: 0.75,
+    tags: ["startups", "software", "open-source", "technology"]
+  },
+  {
+    title: "GitHub Blog",
+    description: "GitHub product, open-source, and developer ecosystem news.",
+    homepageUrl: "https://github.blog",
+    feedUrl: "https://github.blog/feed/",
+    kind: "github",
+    provider: "manual-curated",
+    score: 0,
+    confidence: 0.7,
+    tags: ["github", "open-source", "software", "developers"]
+  }
+];
+
+export class ManualCuratedProvider implements FeedDiscoveryProvider {
+  id = "manual-curated";
+  name = "Manual Curated";
+  enabledByDefault = true;
+  requiresApiKey = false;
+
+  async search(query: FeedDiscoveryQuery) {
+    const terms = normalize(query.q);
+    return CURATED_FEEDS.filter((feed) => {
+      if (query.type && query.type !== "all" && feed.kind !== query.type) {
+        return false;
+      }
+      const haystack = normalize([feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" "));
+      return terms.length === 0 || terms.some((term) => haystack.includes(term));
+    }).map((feed) => ({ ...feed, provider: this.id }));
+  }
+}
+
+function normalize(value: string) {
+  return value.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
+}

plugins/feed-discovery/src/providers/opml-directory.ts

@@ -0,0 +1,92 @@
+import { readFile } from "node:fs/promises";
+import { XMLParser } from "fast-xml-parser";
+import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
+
+const parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: "",
+  textNodeName: "text"
+});
+
+export class OpmlDirectoryProvider implements FeedDiscoveryProvider {
+  id = "opml-directory";
+  name = "OPML Directory";
+  enabledByDefault = true;
+  requiresApiKey = false;
+
+  constructor(private readonly paths: string[] = []) {}
+
+  async search(query: FeedDiscoveryQuery) {
+    const feeds: DiscoveredFeed[] = [];
+    for (const path of this.paths) {
+      const content = await readFile(path, "utf8");
+      const parsed = parser.parse(content) as unknown;
+      for (const outline of collectOutlines(parsed)) {
+        const feedUrl = stringValue(outline.xmlUrl);
+        if (!feedUrl) {
+          continue;
+        }
+        const title = stringValue(outline.title) || stringValue(outline.text) || feedUrl;
+        const tags = [stringValue(outline.category)].filter((tag): tag is string => Boolean(tag));
+        const feed: DiscoveredFeed = {
+          title,
+          description: stringValue(outline.description),
+          homepageUrl: stringValue(outline.htmlUrl),
+          feedUrl,
+          kind: inferKind(tags.join(" "), query.type),
+          provider: this.id,
+          score: 0,
+          confidence: 0.7,
+          tags
+        };
+        if (matches(feed, query)) {
+          feeds.push(feed);
+        }
+      }
+    }
+    return feeds;
+  }
+}
+
+function collectOutlines(value: unknown): Array<Record<string, unknown>> {
+  if (!isRecord(value)) {
+    return [];
+  }
+  const current = "xmlUrl" in value ? [value] : [];
+  return [
+    ...current,
+    ...Object.values(value).flatMap((entry) => {
+      if (Array.isArray(entry)) {
+        return entry.flatMap(collectOutlines);
+      }
+      return collectOutlines(entry);
+    })
+  ];
+}
+
+function matches(feed: DiscoveredFeed, query: FeedDiscoveryQuery) {
+  if (query.type && query.type !== "all" && feed.kind !== query.type) {
+    return false;
+  }
+  const terms = query.q.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
+  const haystack = [feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" ").toLowerCase();
+  return terms.length === 0 || terms.some((term) => haystack.includes(term));
+}
+
+function inferKind(tags: string, requested?: FeedDiscoveryQuery["type"]) {
+  if (requested && requested !== "all") {
+    return requested;
+  }
+  if (tags.includes("podcast")) {
+    return "podcast";
+  }
+  return "opml";
+}
+
+function stringValue(value: unknown) {
+  return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

plugins/feed-discovery/src/providers/podcastindex.ts

@@ -0,0 +1,76 @@
+import { createHash } from "node:crypto";
+import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
+
+interface PodcastIndexFeed {
+  title?: string;
+  description?: string;
+  url?: string;
+  link?: string;
+  image?: string;
+  language?: string;
+  newestItemPublishTime?: number;
+  categories?: Record<string, string>;
+}
+
+export class PodcastIndexProvider implements FeedDiscoveryProvider {
+  id = "podcastindex";
+  name = "PodcastIndex";
+  enabledByDefault = false;
+  requiresApiKey = true;
+
+  constructor(private readonly config: FeedDiscoveryConfig) {}
+
+  async search(query: FeedDiscoveryQuery) {
+    if (!this.config.podcastIndexApiKey || !this.config.podcastIndexApiSecret) {
+      return [];
+    }
+    if (query.type && query.type !== "all" && query.type !== "podcast") {
+      return [];
+    }
+
+    const authDate = Math.floor(Date.now() / 1000).toString();
+    const auth = createHash("sha1")
+      .update(this.config.podcastIndexApiKey + this.config.podcastIndexApiSecret + authDate)
+      .digest("hex");
+
+    const url = new URL("https://api.podcastindex.org/api/1.0/search/byterm");
+    url.searchParams.set("q", query.q);
+    url.searchParams.set("max", String(Math.min(query.limit ?? 25, 100)));
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
+    try {
+      const response = await fetch(url, {
+        signal: controller.signal,
+        headers: {
+          "user-agent": this.config.userAgent,
+          "X-Auth-Date": authDate,
+          "X-Auth-Key": this.config.podcastIndexApiKey,
+          Authorization: auth
+        }
+      });
+      if (!response.ok) {
+        throw new Error(`PodcastIndex HTTP ${response.status}`);
+      }
+      const body = await response.json() as { feeds?: PodcastIndexFeed[] };
+      return (body.feeds ?? [])
+        .filter((feed) => feed.url)
+        .map((feed): DiscoveredFeed => ({
+          title: feed.title ?? feed.url ?? "Untitled podcast",
+          description: feed.description,
+          homepageUrl: feed.link,
+          feedUrl: feed.url as string,
+          kind: "podcast",
+          provider: this.id,
+          language: feed.language,
+          imageUrl: feed.image,
+          lastPublishedAt: feed.newestItemPublishTime ? new Date(feed.newestItemPublishTime * 1000).toISOString() : undefined,
+          score: 0,
+          confidence: 0.9,
+          tags: [...Object.values(feed.categories ?? {}), "podcast"]
+        }));
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+}

plugins/feed-discovery/src/providers/web-feed-probe.ts

@@ -0,0 +1,42 @@
+import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
+import { probeSite } from "../probe-site.js";
+
+export class WebCandidateFeedProbeProvider implements FeedDiscoveryProvider {
+  id = "web-feed-probe";
+  name = "Web Candidate Feed Probe";
+  enabledByDefault = true;
+  requiresApiKey = false;
+
+  constructor(private readonly config: FeedDiscoveryConfig) {}
+
+  async search(query: FeedDiscoveryQuery) {
+    const candidates = candidateUrls(query, this.config.candidateUrls).slice(0, this.config.maxProbes);
+    const feeds: DiscoveredFeed[] = [];
+
+    for (const candidate of candidates) {
+      const result = await probeSite(candidate, this.config);
+      feeds.push(...result.feeds.map((feed) => ({ ...feed, provider: this.id })));
+    }
+
+    return feeds.filter((feed) => !query.type || query.type === "all" || feed.kind === query.type);
+  }
+}
+
+function candidateUrls(query: FeedDiscoveryQuery, configured: string[]) {
+  const candidates = new Set<string>();
+  if (/^https?:\/\//i.test(query.q)) {
+    candidates.add(query.q);
+  }
+
+  for (const entry of configured) {
+    const [url, ...keywords] = entry.split("|").map((part) => part.trim());
+    if (!url) {
+      continue;
+    }
+    if (keywords.length === 0 || keywords.some((keyword) => query.q.toLowerCase().includes(keyword.toLowerCase()))) {
+      candidates.add(url);
+    }
+  }
+
+  return [...candidates];
+}