diff --git a/apps/commandboard-api/package.json b/apps/commandboard-api/package.json index 999cf6b..bad9548 100644 --- a/apps/commandboard-api/package.json +++ b/apps/commandboard-api/package.json @@ -15,6 +15,7 @@ "@logicsrc/plugin-core": "file:../../packages/plugin-core", "@logicsrc/plugin-c0mpute": "file:../../plugins/c0mpute", "@logicsrc/plugin-coinpay": "file:../../plugins/coinpay", + "@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery", "@logicsrc/plugin-sh1pt": "file:../../plugins/sh1pt", "@logicsrc/plugin-ugig": "file:../../plugins/ugig", "@logicsrc/validators": "file:../../packages/validators" diff --git a/apps/commandboard-api/src/contract.test.ts b/apps/commandboard-api/src/contract.test.ts index 2c20969..c6bc7b0 100644 --- a/apps/commandboard-api/src/contract.test.ts +++ b/apps/commandboard-api/src/contract.test.ts @@ -47,7 +47,7 @@ describe("CommandBoard API contracts", () => { }; expect(response.status).toBe(200); - expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute"]); + expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute", "feed-discovery"]); expect(body.plugins.find((plugin) => plugin.id === "sh1pt")).toMatchObject({ enabled: true, capabilities: expect.arrayContaining(["projects.sync", "actions.publish", "deployments.status"]) @@ -58,6 +58,28 @@ describe("CommandBoard API contracts", () => { }); expect(body.capabilities["actions.publish"]).toEqual(["sh1pt"]); expect(body.capabilities["compute.jobs.dispatch"]).toEqual(["c0mpute"]); + expect(body.capabilities["feeds.discover"]).toEqual(["feed-discovery"]); + }); + + it("exposes feed discovery plugin endpoints", async () => { + const providersResponse = await fetch(`${baseUrl}/api/feeds/providers`); + const providersBody = await providersResponse.json() as { providers: Array<{ id: string; enabledByDefault: boolean }> }; + const discoverResponse = await fetch(`${baseUrl}/api/feeds/discover?q=microsaas&providers=manual-curated&includeUnvalidated=true`); + const discoverBody = await discoverResponse.json() as { query: string; results: Array<{ feedUrl: string; provider: string }> }; + const rssResponse = await fetch(`${baseUrl}/rss/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`); + const rssBody = await rssResponse.text(); + const opmlResponse = await fetch(`${baseUrl}/opml/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`); + const opmlBody = await opmlResponse.text(); + + expect(providersResponse.status).toBe(200); + expect(providersBody.providers.map((provider) => provider.id)).toContain("manual-curated"); + expect(discoverResponse.status).toBe(200); + expect(discoverBody.query).toBe("microsaas"); + expect(discoverBody.results[0]).toMatchObject({ provider: "manual-curated" }); + expect(rssResponse.status).toBe(200); + expect(rssBody).toContain(" { diff --git a/apps/commandboard-api/src/index.ts b/apps/commandboard-api/src/index.ts index 1e23416..41b2f43 100644 --- a/apps/commandboard-api/src/index.ts +++ b/apps/commandboard-api/src/index.ts @@ -3,11 +3,12 @@ import { pathToFileURL } from "node:url"; import { createPluginRegistry } from "@logicsrc/plugin-core"; import { c0mputePlugin } from "@logicsrc/plugin-c0mpute"; import { coinPayPlugin } from "@logicsrc/plugin-coinpay"; +import { discoverFeeds, feedDiscoveryPlugin, listFeedProviders, renderAtom, renderJsonFeed, renderOpml, renderRss, type FeedKind } from "@logicsrc/plugin-feed-discovery"; import { sh1ptPlugin } from "@logicsrc/plugin-sh1pt"; import { uGigPlugin } from "@logicsrc/plugin-ugig"; import { schemas, validate } from "@logicsrc/validators"; -const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin]); +const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin, feedDiscoveryPlugin]); const boards = [ { path: "/general", title: "General", description: "CommandBoard.run general discussion." }, @@ -69,7 +70,7 @@ async function route(request: IncomingMessage, response: ServerResponse) { json(response, 200, { ok: true, service: "commandboard-api", - endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas"] + endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas", "/api/feeds/discover", "/api/feeds/providers", "/rss/discover/:keyword.xml", "/opml/discover/:keyword.xml", "/atom/discover/:keyword.xml", "/json-feed/discover/:keyword.json"] }); return; } @@ -113,6 +114,53 @@ async function route(request: IncomingMessage, response: ServerResponse) { return; } + if (request.method === "GET" && (url.pathname === "/api/feeds/providers" || url.pathname === "/api/rss/providers")) { + json(response, 200, { providers: listFeedProviders() }); + return; + } + + if (request.method === "GET" && (url.pathname === "/api/feeds/discover" || url.pathname === "/api/rss/discover")) { + const query = url.searchParams.get("q"); + if (!query) { + json(response, 422, { error: "Expected q query parameter" }); + return; + } + const result = await discoverFeeds({ + q: query, + type: (url.searchParams.get("type") ?? "all") as FeedKind | "all", + limit: numberParam(url.searchParams.get("limit")), + providers: listParam(url.searchParams.get("providers")), + includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true" + }); + json(response, 200, result); + return; + } + + const formattedDiscover = matchFormattedDiscoverPath(url.pathname); + if (request.method === "GET" && formattedDiscover) { + const result = await discoverFeeds({ + q: formattedDiscover.keyword, + type: "all", + limit: numberParam(url.searchParams.get("limit")), + providers: listParam(url.searchParams.get("providers")), + includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true" + }); + if (formattedDiscover.format === "rss") { + text(response, 200, "application/rss+xml; charset=utf-8", renderRss(result)); + return; + } + if (formattedDiscover.format === "opml") { + text(response, 200, "text/x-opml; charset=utf-8", renderOpml(result)); + return; + } + if (formattedDiscover.format === "atom") { + text(response, 200, "application/atom+xml; charset=utf-8", renderAtom(result)); + return; + } + text(response, 200, "application/feed+json; charset=utf-8", renderJsonFeed(result)); + return; + } + if (request.method === "GET" && url.pathname === "/api/plugins/sh1pt/projects") { json(response, 200, { projects: sh1ptProjects }); return; @@ -194,6 +242,11 @@ function json(response: ServerResponse, status: number, data: unknown) { response.end(JSON.stringify(data, null, 2)); } +function text(response: ServerResponse, status: number, contentType: string, body: string) { + response.writeHead(status, { "content-type": contentType }); + response.end(body); +} + async function readJson(request: IncomingMessage) { const chunks: Buffer[] = []; for await (const chunk of request) { @@ -207,6 +260,41 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +function numberParam(value: string | null) { + if (!value) { + return undefined; + } + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function listParam(value: string | null) { + return value + ?.split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function matchFormattedDiscoverPath(pathname: string) { + const rss = /^\/rss\/discover\/(.+)\.xml$/.exec(pathname); + if (rss) { + return { format: "rss" as const, keyword: decodeURIComponent(rss[1]) }; + } + const opml = /^\/opml\/discover\/(.+)\.xml$/.exec(pathname) ?? /^\/rss\/discover\/(.+)\.opml$/.exec(pathname); + if (opml) { + return { format: "opml" as const, keyword: decodeURIComponent(opml[1]) }; + } + const atom = /^\/atom\/discover\/(.+)\.xml$/.exec(pathname); + if (atom) { + return { format: "atom" as const, keyword: decodeURIComponent(atom[1]) }; + } + const jsonFeed = /^\/json-feed\/discover\/(.+)\.json$/.exec(pathname); + if (jsonFeed) { + return { format: "json-feed" as const, keyword: decodeURIComponent(jsonFeed[1]) }; + } + return undefined; +} + export function startCommandBoardServer(port = Number(process.env.PORT ?? 4010)) { const server = createCommandBoardServer(); server.listen(port, () => { diff --git a/package-lock.json b/package-lock.json index 81d6fda..c36e273 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@logicsrc/plugin-c0mpute": "file:../../plugins/c0mpute", "@logicsrc/plugin-coinpay": "file:../../plugins/coinpay", "@logicsrc/plugin-core": "file:../../packages/plugin-core", + "@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery", "@logicsrc/plugin-sh1pt": "file:../../plugins/sh1pt", "@logicsrc/plugin-ugig": "file:../../plugins/ugig", "@logicsrc/validators": "file:../../packages/validators" @@ -1447,6 +1448,10 @@ "resolved": "packages/plugin-core", "link": true }, + "node_modules/@logicsrc/plugin-feed-discovery": { + "resolved": "plugins/feed-discovery", + "link": true + }, "node_modules/@logicsrc/plugin-sh1pt": { "resolved": "plugins/sh1pt", "link": true @@ -1679,6 +1684,18 @@ "node": ">= 10" } }, + "node_modules/@nodable/entities": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.1.tgz", + "integrity": "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@oxc-project/types": { "version": "0.133.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", @@ -2551,6 +2568,13 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/rss": { + "version": "0.0.32", + "resolved": "https://registry.npmjs.org/@types/rss/-/rss-0.0.32.tgz", + "integrity": "sha512-2oKNqKyUY4RSdvl5eZR1n2Q9yvw3XTe3mQHsFPn9alaNBxfPnbXBtGP8R0SV8pK1PrVnLul0zx7izbm5/gF5Qw==", + "dev": true, + "license": "MIT" + }, "node_modules/@vitejs/plugin-react": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz", @@ -2776,6 +2800,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/browserslist": { "version": "4.28.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", @@ -2877,6 +2907,48 @@ "node": ">=18" } }, + "node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/client-only": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", @@ -2969,6 +3041,34 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/csstype": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", @@ -3011,6 +3111,61 @@ "node": ">=8" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -3046,6 +3201,43 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -3272,6 +3464,44 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", + "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.5.0", + "xml-naming": "^0.1.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.8.0.tgz", + "integrity": "sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^2.1.0", + "fast-xml-builder": "^1.2.0", + "path-expression-matcher": "^1.5.0", + "strnum": "^2.3.0", + "xml-naming": "^0.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3442,6 +3672,37 @@ "node": ">=16.9.0" } }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -4044,6 +4305,18 @@ "node": ">=18" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -4100,6 +4373,55 @@ "wrappy": "1" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -4109,6 +4431,21 @@ "node": ">= 0.8" } }, + "node_modules/path-expression-matcher": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", + "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4426,6 +4763,37 @@ "node": ">= 18" } }, + "node_modules/rss": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/rss/-/rss-1.2.2.tgz", + "integrity": "sha512-xUhRTgslHeCBeHAqaWSbOYTydN2f0tAzNXvzh3stjz7QDhQMzdgHf3pfgNIngeytQflrFPfy6axHilTETr6gDg==", + "license": "MIT", + "dependencies": { + "mime-types": "2.1.13", + "xml": "1.0.1" + } + }, + "node_modules/rss/node_modules/mime-db": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.25.0.tgz", + "integrity": "sha512-5k547tI4Cy+Lddr/hdjNbBEWBwSl8EBc5aSdKvedav8DReADgWJzcYiktaRIw3GtGC1jjwldXtTzvqJZmtvC7w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/rss/node_modules/mime-types": { + "version": "2.1.13", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.13.tgz", + "integrity": "sha512-ryBDp1Z/6X90UvjUK3RksH0IBPM137T7cmg4OgD5wQBojlAiUwuok0QeELkim/72EtcYuNlmbkrcGuxj3Kl0YQ==", + "license": "MIT", + "dependencies": { + "mime-db": "~1.25.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -4656,6 +5024,15 @@ "dev": true, "license": "ISC" }, + "node_modules/slugify": { + "version": "1.6.9", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz", + "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -4688,6 +5065,18 @@ "dev": true, "license": "MIT" }, + "node_modules/strnum": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz", + "integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/styled-jsx": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", @@ -5290,6 +5679,15 @@ "node": ">=14.17" } }, + "node_modules/undici": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.27.2.tgz", + "integrity": "sha512-uZsKNuzQxDMUY6M3pIMvy5tvlGmtq8XJ2oLAkfRKGNu+1VQAIvLy2xIVG5ATZl5wDXl/tddByAWCizRbOme+TA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", @@ -5512,6 +5910,40 @@ } } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5550,6 +5982,27 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", + "license": "MIT" + }, + "node_modules/xml-naming": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", + "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", @@ -5595,6 +6048,7 @@ "dependencies": { "@logicsrc/plugin-coinpay": "file:../../plugins/coinpay", "@logicsrc/plugin-core": "file:../plugin-core", + "@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery", "@logicsrc/plugin-ugig": "file:../../plugins/ugig", "@logicsrc/tui": "file:../tui", "@logicsrc/validators": "file:../validators", @@ -5688,6 +6142,22 @@ "vitest": "^4.0.8" } }, + "plugins/feed-discovery": { + "name": "@logicsrc/plugin-feed-discovery", + "version": "0.1.0", + "dependencies": { + "@logicsrc/plugin-core": "file:../../packages/plugin-core", + "cheerio": "^1.0.0", + "fast-xml-parser": "^5.2.5", + "rss": "^1.2.2", + "slugify": "^1.6.6", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/rss": "^0.0.32", + "vitest": "^4.0.8" + } + }, "plugins/sh1pt": { "name": "@logicsrc/plugin-sh1pt", "version": "0.1.0", diff --git a/package.json b/package.json index 17eda3c..9a9cd93 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "apps/*" ], "scripts": { - "build": "npm --workspace @logicsrc/schemas run build && npm --workspace @logicsrc/validators run build && npm --workspace @logicsrc/sdk run build && npm --workspace @logicsrc/plugin-core run build && npm --workspace @logicsrc/plugin-coinpay run build && npm --workspace @logicsrc/plugin-ugig run build && npm --workspace @logicsrc/plugin-sh1pt run build && npm --workspace @logicsrc/plugin-c0mpute run build && npm --workspace @logicsrc/tui run build && npm --workspace @logicsrc/cli run build && npm --workspace @profullstack/logicsrc-mcp run build && npm --workspace @logicsrc/commandboard-api run build && npm --workspace @logicsrc/commandboard-web run build && npm --workspace @logicsrc/web run build", + "build": "npm --workspace @logicsrc/schemas run build && npm --workspace @logicsrc/validators run build && npm --workspace @logicsrc/sdk run build && npm --workspace @logicsrc/plugin-core run build && npm --workspace @logicsrc/plugin-coinpay run build && npm --workspace @logicsrc/plugin-ugig run build && npm --workspace @logicsrc/plugin-sh1pt run build && npm --workspace @logicsrc/plugin-c0mpute run build && npm --workspace @logicsrc/plugin-feed-discovery run build && npm --workspace @logicsrc/tui run build && npm --workspace @logicsrc/cli run build && npm --workspace @profullstack/logicsrc-mcp run build && npm --workspace @logicsrc/commandboard-api run build && npm --workspace @logicsrc/commandboard-web run build && npm --workspace @logicsrc/web run build", "start": "npm --workspace @logicsrc/web run start", "test": "npm run test --workspaces --if-present", "check": "npm run build && npm run test", diff --git a/packages/cli/package.json b/packages/cli/package.json index d033486..d75eb68 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -16,6 +16,7 @@ "dependencies": { "@logicsrc/plugin-core": "file:../plugin-core", "@logicsrc/plugin-coinpay": "file:../../plugins/coinpay", + "@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery", "@logicsrc/plugin-ugig": "file:../../plugins/ugig", "@logicsrc/tui": "file:../tui", "@logicsrc/validators": "file:../validators", diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index b1d9110..4555a35 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -7,6 +7,7 @@ describe("CLI registry", () => { expect(ids).toContain("coinpay"); expect(ids).toContain("ugig"); + expect(ids).toContain("feed-discovery"); expect(ids).not.toContain("sh1pt"); }); }); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 45bd4c8..acfa839 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import { readFileSync } from "node:fs"; import { Command } from "commander"; +import { discoverFeeds, listFeedProviders, probeSite, renderDiscoveryOutput, validateFeed, type FeedKind, type FeedOutputFormat } from "@logicsrc/plugin-feed-discovery"; import { renderArcadeList, renderPluginStatus, renderTui, runArcadeSession, type TaskSnapshot } from "@logicsrc/tui"; import { assertSchemaKind, parseDocument, validate } from "@logicsrc/validators"; import { getConfigValue, readConfig, setConfigValue, writeConfig } from "./config.js"; @@ -295,6 +296,76 @@ openspec outDir: options.out }), "json")); +const feeds = program.command("feeds").description("Discover, validate, probe, and export feed sources."); + +feeds + .command("discover") + .argument("", "Keyword, phrase, or homepage URL") + .option("--type ", "Feed kind or all", "all") + .option("--format ", "json, opml, rss, atom, or json-feed", "json") + .option("--limit ", "Maximum results", "25") + .option("--freshness-days ", "Freshness window for callers that need it") + .option("--include-dead-feeds", "Include feeds that fail validation") + .option("--include-unvalidated", "Return provider candidates without validation") + .option("--providers ", "Comma-separated provider ids") + .description("Discover canonical feed URLs by keyword.") + .action(async (keyword, options) => { + const response = await discoverFeeds({ + q: keyword, + type: options.type as FeedKind | "all", + limit: Number(options.limit), + freshnessDays: options.freshnessDays ? Number(options.freshnessDays) : undefined, + includeDeadFeeds: Boolean(options.includeDeadFeeds), + includeUnvalidated: Boolean(options.includeUnvalidated), + providers: splitOption(options.providers) + }); + console.log(renderDiscoveryOutput(response, options.format as FeedOutputFormat)); + }); + +feeds + .command("validate") + .argument("", "Feed URL") + .option("--format ", "json, table, or markdown", "json") + .description("Validate a feed URL with SSRF protections.") + .action(async (feedUrl, options) => { + const result = await validateFeed(feedUrl); + print(result, options.format as OutputFormat); + if (!result.ok) { + process.exitCode = 1; + } + }); + +feeds + .command("probe") + .argument("", "Homepage URL") + .option("--format ", "json, table, or markdown", "json") + .description("Probe a homepage for alternate feed links and common feed paths.") + .action(async (homepageUrl, options) => { + const result = await probeSite(homepageUrl); + print(result, options.format as OutputFormat); + if (result.feeds.length === 0) { + process.exitCode = 1; + } + }); + +feeds + .command("providers") + .option("--format ", "table, json, or markdown", "table") + .description("List feed discovery providers.") + .action((options) => { + print(listFeedProviders(), options.format as OutputFormat); + }); + +feeds + .command("export-opml") + .argument("", "Keyword or phrase") + .option("--limit ", "Maximum results", "100") + .description("Discover feeds and print OPML.") + .action(async (keyword, options) => { + const response = await discoverFeeds({ q: keyword, limit: Number(options.limit) }); + console.log(renderDiscoveryOutput(response, "opml")); + }); + program.command("plugins").option("--format ", "table, json, or markdown", "table").description("Show plugin status.").action((options) => { const snapshot = defaultPluginRegistry().snapshot(); print(snapshot.plugins, options.format as OutputFormat); @@ -395,6 +466,13 @@ function resolveArcadeGame(options: { arcade?: string | boolean; waitingArcade?: return undefined; } +function splitOption(value: string | undefined) { + return value + ?.split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} + async function runYoloArcade(game: string, repo?: string) { const task: TaskSnapshot = { id: "agentswarm_yolo", diff --git a/packages/cli/src/registry.ts b/packages/cli/src/registry.ts index 3771c1b..6081ee8 100644 --- a/packages/cli/src/registry.ts +++ b/packages/cli/src/registry.ts @@ -1,7 +1,8 @@ import { createPluginRegistry } from "@logicsrc/plugin-core"; import { coinPayPlugin } from "@logicsrc/plugin-coinpay"; +import { feedDiscoveryPlugin } from "@logicsrc/plugin-feed-discovery"; import { uGigPlugin } from "@logicsrc/plugin-ugig"; export function defaultPluginRegistry() { - return createPluginRegistry([coinPayPlugin, uGigPlugin]); + return createPluginRegistry([coinPayPlugin, uGigPlugin, feedDiscoveryPlugin]); } diff --git a/plugins/feed-discovery/README.md b/plugins/feed-discovery/README.md new file mode 100644 index 0000000..7eb0c21 --- /dev/null +++ b/plugins/feed-discovery/README.md @@ -0,0 +1,101 @@ +# LogicSRC Feed Discovery Plugin + +Discover RSS, Atom, JSON Feed, podcast, and feed-like sources by keyword using a provider-based LogicSRC plugin. + +## Install + +This repository includes the plugin as a normal LogicSRC workspace plugin: + +```bash +npm --workspace @logicsrc/plugin-feed-discovery run build +``` + +Consumers can import the runtime API: + +```ts +import { discoverFeeds, renderRss } from "@logicsrc/plugin-feed-discovery"; + +const result = await discoverFeeds({ q: "microsaas", limit: 25 }); +console.log(renderRss(result)); +``` + +## CLI + +```bash +logicsrc feeds discover "microsaas" --format json +logicsrc feeds discover "microsaas" --format opml +logicsrc feeds discover "microsaas" --format rss +logicsrc feeds discover "ai agents" --type podcast +logicsrc feeds validate https://example.com/feed.xml +logicsrc feeds probe https://example.com +logicsrc feeds providers +``` + +Validation is enabled by default. Use `--include-unvalidated` to inspect raw provider candidates without network validation. + +## Providers + +MVP providers: + +- `manual-curated`: local high-trust starter feeds. +- `opml-directory`: local OPML files configured with `LOGICSRC_FEEDS_OPML_PATHS`. +- `web-feed-probe`: probes direct URL queries and configured candidate homepages from `LOGICSRC_FEEDS_CANDIDATE_URLS`. +- `itunes-podcast`: public iTunes podcast search. +- `podcastindex`: optional PodcastIndex search when `PODCASTINDEX_API_KEY` and `PODCASTINDEX_API_SECRET` are set. + +Provider failures are isolated and returned as `providerErrors`; one failed provider does not fail the whole discovery request. + +## HTTP Reference API + +The CommandBoard reference API exposes: + +```http +GET /api/feeds/discover?q=microsaas&type=all&limit=50 +GET /api/feeds/providers +GET /rss/discover/microsaas.xml +GET /api/rss/discover?q=microsaas +``` + +BitTorrented can consume the plugin package directly and map its public routes to the same runtime calls: + +- `/api/rss/discover?q=:keyword` -> `discoverFeeds()` +- `/rss/discover/:keyword.xml` -> `discoverFeeds()` plus `renderRss()` +- `/rss/discover/:keyword.opml` -> `discoverFeeds()` plus `renderOpml()` + +## Configuration + +```bash +LOGICSRC_FEEDS_CACHE_TTL_SECONDS=86400 +LOGICSRC_FEEDS_MAX_PROVIDERS=10 +LOGICSRC_FEEDS_MAX_PROBES=50 +LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS=8000 +LOGICSRC_FEEDS_MAX_BODY_BYTES=1000000 +LOGICSRC_FEEDS_USER_AGENT="LogicSrcFeedDiscovery/0.1" +LOGICSRC_FEEDS_OPML_PATHS="./data/feeds.opml,./data/podcasts.opml" +LOGICSRC_FEEDS_CANDIDATE_URLS="https://example.com|microsaas,https://another.example|ai agents" + +PODCASTINDEX_API_KEY= +PODCASTINDEX_API_SECRET= +``` + +## Security + +`validateFeed()` and `probeSite()` use guarded fetches: + +- Only `http` and `https` URLs are allowed. +- `localhost`, loopback, private, link-local, carrier-grade NAT, and common metadata targets are blocked. +- DNS-resolved private/internal addresses are blocked. +- Redirects are limited and checked through the same guard. +- Request timeouts and response body size limits are enforced. + +## Database + +Supabase-compatible schema SQL is included at `src/db/schema.sql`. The current LogicSRC repo has no shared Supabase migration convention, so the plugin ships the schema for host applications to apply. + +## Deferred v0.2 Work + +- YouTube, Reddit, GitHub, and RSSHub adapters. +- Persistent cache and refresh jobs. +- Atom and JSON Feed public endpoints in host apps. +- Provider health dashboard and admin tooling. +- Paid provider adapters such as RSS.app, Feedly, Inoreader, Twingly, and Listen Notes. diff --git a/plugins/feed-discovery/package.json b/plugins/feed-discovery/package.json new file mode 100644 index 0000000..51f8048 --- /dev/null +++ b/plugins/feed-discovery/package.json @@ -0,0 +1,24 @@ +{ + "name": "@logicsrc/plugin-feed-discovery", + "version": "0.1.0", + "description": "LogicSRC feed discovery plugin for RSS, Atom, JSON Feed, podcasts, OPML directories, and probed sites.", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "vitest run src --passWithNoTests" + }, + "dependencies": { + "@logicsrc/plugin-core": "file:../../packages/plugin-core", + "cheerio": "^1.0.0", + "fast-xml-parser": "^5.2.5", + "rss": "^1.2.2", + "slugify": "^1.6.6", + "zod": "^4.1.13" + }, + "devDependencies": { + "@types/rss": "^0.0.32", + "vitest": "^4.0.8" + } +} diff --git a/plugins/feed-discovery/src/config.ts b/plugins/feed-discovery/src/config.ts new file mode 100644 index 0000000..60445a7 --- /dev/null +++ b/plugins/feed-discovery/src/config.ts @@ -0,0 +1,34 @@ +import type { FeedDiscoveryConfig } from "./types.js"; + +const DEFAULT_TIMEOUT_MS = 8_000; +const DEFAULT_MAX_BODY_BYTES = 1_000_000; + +export function readFeedDiscoveryConfig(env: NodeJS.ProcessEnv = process.env): FeedDiscoveryConfig { + return { + cacheTtlSeconds: readNumber(env.LOGICSRC_FEEDS_CACHE_TTL_SECONDS, 86_400), + maxProviders: readNumber(env.LOGICSRC_FEEDS_MAX_PROVIDERS, 10), + maxProbes: readNumber(env.LOGICSRC_FEEDS_MAX_PROBES, 50), + requestTimeoutMs: readNumber(env.LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS, DEFAULT_TIMEOUT_MS), + maxBodyBytes: readNumber(env.LOGICSRC_FEEDS_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES), + userAgent: env.LOGICSRC_FEEDS_USER_AGENT || "LogicSrcFeedDiscovery/0.1", + opmlPaths: splitList(env.LOGICSRC_FEEDS_OPML_PATHS), + candidateUrls: splitList(env.LOGICSRC_FEEDS_CANDIDATE_URLS), + podcastIndexApiKey: env.PODCASTINDEX_API_KEY, + podcastIndexApiSecret: env.PODCASTINDEX_API_SECRET + }; +} + +function readNumber(value: string | undefined, fallback: number) { + if (!value) { + return fallback; + } + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function splitList(value: string | undefined) { + return (value ?? "") + .split(",") + .map((entry) => entry.trim()) + .filter(Boolean); +} diff --git a/plugins/feed-discovery/src/db/schema.sql b/plugins/feed-discovery/src/db/schema.sql new file mode 100644 index 0000000..169a8f3 --- /dev/null +++ b/plugins/feed-discovery/src/db/schema.sql @@ -0,0 +1,62 @@ +create table if not exists feed_sources ( + id uuid primary key default gen_random_uuid(), + title text not null, + description text, + homepage_url text, + feed_url text not null unique, + canonical_feed_url text, + kind text not null default 'unknown', + provider text not null, + language text, + image_url text, + score numeric not null default 0, + confidence numeric not null default 0, + freshness_score numeric not null default 0, + keyword_score numeric not null default 0, + provider_score numeric not null default 0, + last_published_at timestamptz, + last_checked_at timestamptz, + is_valid boolean default true, + created_at timestamptz default now(), + updated_at timestamptz default now() +); + +create table if not exists feed_discovery_queries ( + id uuid primary key default gen_random_uuid(), + query text not null, + normalized_query text not null, + type text default 'all', + result_count integer not null default 0, + created_at timestamptz default now() +); + +create table if not exists feed_discovery_results ( + query_id uuid references feed_discovery_queries(id) on delete cascade, + source_id uuid references feed_sources(id) on delete cascade, + rank integer not null, + score numeric not null, + created_at timestamptz default now(), + primary key (query_id, source_id) +); + +create table if not exists feed_items ( + id uuid primary key default gen_random_uuid(), + source_id uuid references feed_sources(id) on delete cascade, + title text not null, + url text not null unique, + description text, + published_at timestamptz, + guid text, + created_at timestamptz default now() +); + +create table if not exists feed_provider_logs ( + id uuid primary key default gen_random_uuid(), + provider text not null, + query text not null, + status text not null, + result_count integer default 0, + error_message text, + duration_ms integer, + created_at timestamptz default now() +); diff --git a/plugins/feed-discovery/src/dedupe.ts b/plugins/feed-discovery/src/dedupe.ts new file mode 100644 index 0000000..0bf47dc --- /dev/null +++ b/plugins/feed-discovery/src/dedupe.ts @@ -0,0 +1,38 @@ +import type { DiscoveredFeed } from "./types.js"; +import { canonicalizeUrl } from "./url-safety.js"; + +export function dedupeFeeds(feeds: DiscoveredFeed[]) { + const byKey = new Map(); + + for (const feed of feeds) { + const key = dedupeKey(feed); + const existing = byKey.get(key); + if (!existing || feed.score > existing.score || feed.confidence > existing.confidence) { + byKey.set(key, mergeFeed(existing, feed)); + } + } + + return [...byKey.values()].sort((a, b) => b.score - a.score || b.confidence - a.confidence); +} + +function dedupeKey(feed: DiscoveredFeed) { + const feedUrl = feed.canonicalFeedUrl || feed.feedUrl; + try { + return `feed:${canonicalizeUrl(feedUrl)}`; + } catch { + return `feed:${feedUrl.toLowerCase()}`; + } +} + +function mergeFeed(existing: DiscoveredFeed | undefined, next: DiscoveredFeed): DiscoveredFeed { + if (!existing) { + return next; + } + + return { + ...existing, + ...next, + tags: [...new Set([...existing.tags, ...next.tags])], + sampleItems: next.sampleItems?.length ? next.sampleItems : existing.sampleItems + }; +} diff --git a/plugins/feed-discovery/src/discovery.ts b/plugins/feed-discovery/src/discovery.ts new file mode 100644 index 0000000..19f1e26 --- /dev/null +++ b/plugins/feed-discovery/src/discovery.ts @@ -0,0 +1,120 @@ +import slugify from "slugify"; +import { readFeedDiscoveryConfig } from "./config.js"; +import { dedupeFeeds } from "./dedupe.js"; +import { createDefaultFeedProviders } from "./providers/index.js"; +import { scoreFeed } from "./scoring.js"; +import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery, FeedDiscoveryResponse } from "./types.js"; +import { canonicalizeUrl } from "./url-safety.js"; +import { validateFeed } from "./validate-feed.js"; + +export async function discoverFeeds(query: FeedDiscoveryQuery, options: { providers?: FeedDiscoveryProvider[]; config?: Partial } = {}): Promise { + const config = { ...readFeedDiscoveryConfig(), ...options.config }; + const normalizedQuery = normalizeQuery(query.q); + const resolvedQuery: FeedDiscoveryQuery = { + ...query, + q: query.q.trim(), + type: query.type ?? "all", + limit: clampLimit(query.limit), + includeUnvalidated: query.includeUnvalidated ?? false, + includeDeadFeeds: query.includeDeadFeeds ?? false + }; + + const providers = (options.providers ?? createDefaultFeedProviders(config)) + .filter((provider) => provider.enabledByDefault || provider.id === "podcastindex") + .filter((provider) => !resolvedQuery.providers?.length || resolvedQuery.providers.includes(provider.id)) + .slice(0, config.maxProviders); + + const settled = await Promise.all(providers.map(async (provider) => { + try { + return { + provider: provider.id, + results: await provider.search(resolvedQuery), + error: undefined + }; + } catch (error) { + return { + provider: provider.id, + results: [], + error: error instanceof Error ? error.message : String(error) + }; + } + })); + + const providerErrors: FeedDiscoveryResponse["providerErrors"] = []; + const candidates: DiscoveredFeed[] = []; + + for (const result of settled) { + if (result.error) { + providerErrors.push({ provider: result.provider, error: result.error }); + continue; + } + candidates.push(...result.results.map((feed) => ({ ...feed, provider: feed.provider || result.provider }))); + } + + const validated = await validateCandidates(candidates.slice(0, Math.max((resolvedQuery.limit ?? 25) * 2, 25)), resolvedQuery, config); + const scored = validated + .filter((feed) => !resolvedQuery.type || resolvedQuery.type === "all" || feed.kind === resolvedQuery.type) + .map((feed) => scoreFeed(feed, resolvedQuery)); + const results = dedupeFeeds(scored).slice(0, resolvedQuery.limit ?? 25); + + return { + query: query.q, + normalizedQuery, + count: results.length, + providerErrors, + results + }; +} + +async function validateCandidates(candidates: DiscoveredFeed[], query: FeedDiscoveryQuery, config: FeedDiscoveryConfig) { + if (query.includeUnvalidated) { + return candidates.map((feed) => ({ + ...feed, + canonicalFeedUrl: safeCanonical(feed.feedUrl), + isValid: feed.isValid ?? false, + validationScore: feed.validationScore ?? 0.4 + })); + } + + const feeds: DiscoveredFeed[] = []; + for (const candidate of candidates) { + const validation = await validateFeed(candidate.feedUrl, config); + if (!validation.ok && !query.includeDeadFeeds) { + continue; + } + feeds.push({ + ...candidate, + title: validation.title ?? candidate.title, + description: validation.description ?? candidate.description, + homepageUrl: validation.homepageUrl ?? candidate.homepageUrl, + canonicalFeedUrl: validation.canonicalFeedUrl ?? safeCanonical(candidate.feedUrl), + kind: validation.ok && validation.kind !== "unknown" ? validation.kind : candidate.kind, + language: validation.language ?? candidate.language, + imageUrl: validation.imageUrl ?? candidate.imageUrl, + lastPublishedAt: validation.lastPublishedAt ?? candidate.lastPublishedAt, + sampleItems: validation.sampleItems.length ? validation.sampleItems : candidate.sampleItems, + isValid: validation.ok, + validationScore: validation.ok ? 1 : 0.2 + }); + } + return feeds; +} + +function normalizeQuery(query: string) { + return slugify(query.trim().toLowerCase(), { lower: true, strict: true }) || "feeds"; +} + +function clampLimit(limit: number | undefined) { + if (!limit) { + return 25; + } + return Math.min(Math.max(Math.trunc(limit), 1), 100); +} + +function safeCanonical(url: string) { + try { + return canonicalizeUrl(url); + } catch { + return url; + } +} diff --git a/plugins/feed-discovery/src/feed-discovery.test.ts b/plugins/feed-discovery/src/feed-discovery.test.ts new file mode 100644 index 0000000..3efd069 --- /dev/null +++ b/plugins/feed-discovery/src/feed-discovery.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { dedupeFeeds } from "./dedupe.js"; +import { discoverFeeds } from "./discovery.js"; +import { parseFeedDocument } from "./feed-parsing.js"; +import { renderDiscoveryOutput } from "./output/index.js"; +import { extractAlternateFeedLinks } from "./probe-site.js"; +import { scoreFeed } from "./scoring.js"; +import type { DiscoveredFeed, FeedDiscoveryProvider } from "./types.js"; +import { assertSafeHttpUrl } from "./url-safety.js"; + +const baseFeed: DiscoveredFeed = { + title: "MicroSaaS Ideas", + description: "Ideas for indie SaaS founders", + homepageUrl: "https://example.com", + feedUrl: "https://example.com/feed.xml", + kind: "blog", + provider: "manual-curated", + score: 0, + confidence: 0.5, + tags: ["microsaas", "saas"] +}; + +describe("feed parsing", () => { + it("parses RSS feeds with sample items", () => { + const result = parseFeedDocument("https://example.com/rss.xml", `MicroSaaS Ideashttps://example.comLaunch tiny productshttps://example.com/postTue, 09 Jun 2026 00:00:00 GMT`); + + expect(result.ok).toBe(true); + expect(result.title).toBe("MicroSaaS Ideas"); + expect(result.sampleItems[0]).toMatchObject({ title: "Launch tiny products", url: "https://example.com/post" }); + }); + + it("parses Atom and JSON Feed documents", () => { + const atom = parseFeedDocument("https://example.com/atom.xml", `Atom FeedEntry2026-06-09T00:00:00Z`); + const json = parseFeedDocument("https://example.com/feed.json", JSON.stringify({ version: "https://jsonfeed.org/version/1.1", title: "JSON Feed", items: [{ title: "Item", url: "https://example.com/item" }] }), "application/feed+json"); + + expect(atom.ok).toBe(true); + expect(json.ok).toBe(true); + }); +}); + +describe("site probing helpers", () => { + it("extracts alternate feed links", () => { + const links = extractAlternateFeedLinks( + ``, + "https://example.com/blog" + ); + + expect(links).toEqual(["https://example.com/rss.xml"]); + }); + + it("blocks private SSRF targets", async () => { + await expect(assertSafeHttpUrl("http://127.0.0.1/feed")).rejects.toThrow(/Blocked internal/); + await expect(assertSafeHttpUrl("http://[::ffff:192.168.1.10]/feed")).rejects.toThrow(/Blocked internal/); + await expect(assertSafeHttpUrl("file:///etc/passwd")).rejects.toThrow(/Unsupported URL protocol/); + }); +}); + +describe("scoring, dedupe, and output", () => { + it("scores relevant fresh feeds and dedupes canonical URLs", () => { + const scored = scoreFeed({ ...baseFeed, lastPublishedAt: new Date().toISOString() }, { q: "microsaas" }); + const duplicate = { ...scored, feedUrl: "https://example.com/feed.xml?utm_source=test", score: scored.score - 0.1 }; + + expect(scored.score).toBeGreaterThan(0.5); + expect(dedupeFeeds([duplicate, scored])).toHaveLength(1); + }); + + it("renders JSON, OPML, and RSS outputs", () => { + const response = { query: "microsaas", normalizedQuery: "microsaas", count: 1, providerErrors: [], results: [scoreFeed(baseFeed, { q: "microsaas" })] }; + + expect(JSON.parse(renderDiscoveryOutput(response, "json"))).toMatchObject({ count: 1 }); + expect(renderDiscoveryOutput(response, "opml")).toContain(" { + it("isolates provider failures", async () => { + const okProvider: FeedDiscoveryProvider = { + id: "ok-provider", + name: "OK", + enabledByDefault: true, + requiresApiKey: false, + async search() { + return [baseFeed]; + } + }; + const failingProvider: FeedDiscoveryProvider = { + id: "broken-provider", + name: "Broken", + enabledByDefault: true, + requiresApiKey: false, + async search() { + throw new Error("provider unavailable"); + } + }; + + const response = await discoverFeeds({ q: "microsaas", includeUnvalidated: true }, { providers: [failingProvider, okProvider] }); + + expect(response.providerErrors).toEqual([{ provider: "broken-provider", error: "provider unavailable" }]); + expect(response.results).toHaveLength(1); + }); +}); diff --git a/plugins/feed-discovery/src/feed-parsing.ts b/plugins/feed-discovery/src/feed-parsing.ts new file mode 100644 index 0000000..92fd4d7 --- /dev/null +++ b/plugins/feed-discovery/src/feed-parsing.ts @@ -0,0 +1,204 @@ +import { XMLParser } from "fast-xml-parser"; +import type { FeedKind, FeedSampleItem, ValidationResult } from "./types.js"; + +const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "", + textNodeName: "text", + cdataPropName: "text", + removeNSPrefix: true +}); + +export function parseFeedDocument(feedUrl: string, body: string, contentType = ""): ValidationResult { + const trimmed = body.trim(); + if (!trimmed) { + return invalid(feedUrl, "Empty feed body"); + } + + if (contentType.includes("json") || trimmed.startsWith("{")) { + return parseJsonFeed(feedUrl, trimmed); + } + + try { + const document = parser.parse(trimmed) as unknown; + if (!isRecord(document)) { + return invalid(feedUrl, "Feed did not parse as an object"); + } + + if (isRecord(document.rss)) { + return parseRss(feedUrl, document.rss); + } + if (isRecord(document.feed)) { + return parseAtom(feedUrl, document.feed); + } + return invalid(feedUrl, "Document is not RSS, Atom, or JSON Feed"); + } catch (error) { + return invalid(feedUrl, error instanceof Error ? error.message : String(error)); + } +} + +function parseJsonFeed(feedUrl: string, body: string): ValidationResult { + try { + const document = JSON.parse(body) as unknown; + if (!isRecord(document) || typeof document.version !== "string" || !document.version.includes("jsonfeed")) { + return invalid(feedUrl, "JSON document is not JSON Feed"); + } + + const items = asArray(document.items).filter(isRecord); + const sampleItems = items.slice(0, 5).map((item) => ({ + title: stringValue(item.title) || stringValue(item.url) || "Untitled item", + url: stringValue(item.url), + publishedAt: stringValue(item.date_published), + description: stringValue(item.summary) || stringValue(item.content_text) + })); + + const title = stringValue(document.title); + if (!title && sampleItems.length === 0) { + return invalid(feedUrl, "Feed has no title or items"); + } + + return { + ok: true, + feedUrl, + canonicalFeedUrl: feedUrl, + title: title || "Untitled JSON Feed", + description: stringValue(document.description), + homepageUrl: stringValue(document.home_page_url), + kind: "blog", + imageUrl: stringValue(document.icon) || stringValue(document.favicon), + lastPublishedAt: newestDate(sampleItems.map((item) => item.publishedAt)), + sampleItems + }; + } catch (error) { + return invalid(feedUrl, error instanceof Error ? error.message : String(error)); + } +} + +function parseRss(feedUrl: string, rss: Record): ValidationResult { + const channel = isRecord(rss.channel) ? rss.channel : rss; + const items = asArray(channel.item).filter(isRecord); + const sampleItems = items.slice(0, 5).map((item) => ({ + title: stringValue(item.title) || stringValue(item.guid) || "Untitled item", + url: linkValue(item.link) || stringValue(item.guid), + publishedAt: stringValue(item.pubDate) || stringValue(item.published) || stringValue(item.updated), + description: stringValue(item.description) + })); + + const title = stringValue(channel.title); + if (!title && sampleItems.length === 0) { + return invalid(feedUrl, "RSS feed has no title or items"); + } + + return { + ok: true, + feedUrl, + canonicalFeedUrl: feedUrl, + title: title || "Untitled RSS Feed", + description: stringValue(channel.description), + homepageUrl: linkValue(channel.link), + kind: detectRssKind(channel), + language: stringValue(channel.language), + imageUrl: imageValue(channel.image), + lastPublishedAt: newestDate([stringValue(channel.lastBuildDate), stringValue(channel.pubDate), ...sampleItems.map((item) => item.publishedAt)]), + sampleItems + }; +} + +function parseAtom(feedUrl: string, feed: Record): ValidationResult { + const entries = asArray(feed.entry).filter(isRecord); + const sampleItems = entries.slice(0, 5).map((entry) => ({ + title: stringValue(entry.title) || "Untitled item", + url: atomLink(entry.link), + publishedAt: stringValue(entry.published) || stringValue(entry.updated), + description: stringValue(entry.summary) || stringValue(entry.content) + })); + + const title = stringValue(feed.title); + if (!title && sampleItems.length === 0) { + return invalid(feedUrl, "Atom feed has no title or entries"); + } + + return { + ok: true, + feedUrl, + canonicalFeedUrl: feedUrl, + title: title || "Untitled Atom Feed", + description: stringValue(feed.subtitle), + homepageUrl: atomLink(feed.link), + kind: "blog", + language: stringValue(feed.lang), + lastPublishedAt: newestDate([stringValue(feed.updated), ...sampleItems.map((item) => item.publishedAt)]), + sampleItems + }; +} + +function detectRssKind(channel: Record): FeedKind { + if (channel.itunes || channel["itunes:author"] || channel.enclosure) { + return "podcast"; + } + return "blog"; +} + +function imageValue(value: unknown) { + if (isRecord(value)) { + return stringValue(value.url); + } + return undefined; +} + +function atomLink(value: unknown) { + const links = asArray(value).filter(isRecord); + const alternate = links.find((link) => stringValue(link.rel) === "alternate") ?? links[0]; + return alternate ? stringValue(alternate.href) : stringValue(value); +} + +function linkValue(value: unknown) { + if (isRecord(value)) { + return stringValue(value.href) || stringValue(value.text); + } + return stringValue(value); +} + +function stringValue(value: unknown) { + if (typeof value === "string") { + return value.trim() || undefined; + } + if (typeof value === "number") { + return String(value); + } + if (isRecord(value) && typeof value.text === "string") { + return value.text.trim() || undefined; + } + return undefined; +} + +function newestDate(values: Array) { + const timestamps = values + .map((value) => (value ? Date.parse(value) : NaN)) + .filter((value) => Number.isFinite(value)); + if (timestamps.length === 0) { + return undefined; + } + return new Date(Math.max(...timestamps)).toISOString(); +} + +function invalid(feedUrl: string, error: string): ValidationResult { + return { + ok: false, + feedUrl, + kind: "unknown", + sampleItems: [], + error + }; +} + +function asArray(value: unknown): unknown[] { + if (Array.isArray(value)) { + return value; + } + return value === undefined || value === null ? [] : [value]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/plugins/feed-discovery/src/http.ts b/plugins/feed-discovery/src/http.ts new file mode 100644 index 0000000..3126e84 --- /dev/null +++ b/plugins/feed-discovery/src/http.ts @@ -0,0 +1,80 @@ +import type { FeedDiscoveryConfig } from "./types.js"; +import { assertSafeHttpUrl } from "./url-safety.js"; + +export interface BoundedFetchResult { + url: string; + contentType: string; + body: string; + status: number; +} + +export async function fetchTextWithGuards(input: string, config: Pick, redirects = 3): Promise { + const url = await assertSafeHttpUrl(input); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs); + + try { + const response = await fetch(url, { + redirect: "manual", + signal: controller.signal, + headers: { + "user-agent": config.userAgent, + accept: "application/rss+xml, application/atom+xml, application/feed+json, application/json, text/xml, application/xml, text/html;q=0.8, */*;q=0.5" + } + }); + + if (isRedirect(response.status)) { + if (redirects <= 0) { + throw new Error("Too many redirects"); + } + const location = response.headers.get("location"); + if (!location) { + throw new Error("Redirect without location"); + } + return fetchTextWithGuards(new URL(location, url).toString(), config, redirects - 1); + } + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const body = await readLimitedText(response, config.maxBodyBytes); + return { + url: response.url || url.toString(), + contentType: response.headers.get("content-type") ?? "", + body, + status: response.status + }; + } finally { + clearTimeout(timeout); + } +} + +async function readLimitedText(response: Response, maxBodyBytes: number) { + if (!response.body) { + return response.text(); + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let received = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + received += value.byteLength; + if (received > maxBodyBytes) { + await reader.cancel(); + throw new Error(`Response exceeded ${maxBodyBytes} bytes`); + } + chunks.push(value); + } + + return new TextDecoder().decode(Buffer.concat(chunks)); +} + +function isRedirect(status: number) { + return status === 301 || status === 302 || status === 303 || status === 307 || status === 308; +} diff --git a/plugins/feed-discovery/src/index.ts b/plugins/feed-discovery/src/index.ts new file mode 100644 index 0000000..28b4a79 --- /dev/null +++ b/plugins/feed-discovery/src/index.ts @@ -0,0 +1,47 @@ +import type { PluginDefinition } from "@logicsrc/plugin-core"; +import { readFeedDiscoveryConfig } from "./config.js"; +import { discoverFeeds } from "./discovery.js"; +import { feedDiscoveryManifest } from "./manifest.js"; +import { probeSite } from "./probe-site.js"; +import { providerManifests } from "./providers/index.js"; +import { validateFeed } from "./validate-feed.js"; + +export const feedDiscoveryPlugin: PluginDefinition = { + manifest: feedDiscoveryManifest, + configDefaults: { + enabled: true, + cache_ttl_seconds: "${LOGICSRC_FEEDS_CACHE_TTL_SECONDS}", + max_providers: "${LOGICSRC_FEEDS_MAX_PROVIDERS}", + max_probes: "${LOGICSRC_FEEDS_MAX_PROBES}", + request_timeout_ms: "${LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS}", + user_agent: "${LOGICSRC_FEEDS_USER_AGENT}" + }, + routes: [ + { method: "GET", path: "/api/feeds/discover", capability: "feeds.discover" }, + { method: "GET", path: "/api/feeds/providers", capability: "feeds.providers.list" }, + { method: "GET", path: "/rss/discover/:keyword.xml", capability: "feeds.export.rss" } + ], + permissions: ["feeds:discover", "feeds:validate", "feeds:probe", "feeds:export"], + tuiPanels: [{ id: "feed-discovery-status", title: "Feed Discovery" }] +}; + +export function listFeedProviders() { + return providerManifests(readFeedDiscoveryConfig()); +} + +export { discoverFeeds, feedDiscoveryManifest, probeSite, readFeedDiscoveryConfig, validateFeed }; +export type { + DiscoveredFeed, + FeedDiscoveryConfig, + FeedDiscoveryProvider, + FeedDiscoveryQuery, + FeedDiscoveryResponse, + FeedKind, + FeedOutputFormat, + FeedProviderManifest, + FeedSampleItem, + ProbeResult, + ValidationResult +} from "./types.js"; +export { renderAtom, renderDiscoveryOutput, renderJsonFeed, renderOpml, renderRss } from "./output/index.js"; +export { createDefaultFeedProviders, providerManifests } from "./providers/index.js"; diff --git a/plugins/feed-discovery/src/manifest.ts b/plugins/feed-discovery/src/manifest.ts new file mode 100644 index 0000000..26a6163 --- /dev/null +++ b/plugins/feed-discovery/src/manifest.ts @@ -0,0 +1,27 @@ +import type { PluginManifest } from "@logicsrc/plugin-core"; + +export const feedDiscoveryManifest: PluginManifest = { + id: "feed-discovery", + name: "Feed Discovery", + version: "0.1.0", + type: ["feeds", "rss", "discovery", "content"], + default: true, + capabilities: [ + "feeds.discover", + "feeds.validate", + "feeds.probe", + "feeds.export.opml", + "feeds.export.rss", + "feeds.providers.list" + ], + commands: ["feeds"], + env: [ + "LOGICSRC_FEEDS_CACHE_TTL_SECONDS", + "LOGICSRC_FEEDS_MAX_PROVIDERS", + "LOGICSRC_FEEDS_MAX_PROBES", + "LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS", + "LOGICSRC_FEEDS_USER_AGENT", + "PODCASTINDEX_API_KEY", + "PODCASTINDEX_API_SECRET" + ] +}; diff --git a/plugins/feed-discovery/src/output/atom.ts b/plugins/feed-discovery/src/output/atom.ts new file mode 100644 index 0000000..bd5d9c2 --- /dev/null +++ b/plugins/feed-discovery/src/output/atom.ts @@ -0,0 +1,26 @@ +import type { FeedDiscoveryResponse } from "../types.js"; +import { escapeXml } from "./opml.js"; + +export function renderAtom(response: FeedDiscoveryResponse) { + const updated = new Date().toISOString(); + const entries = response.results + .map((result) => { + const id = result.canonicalFeedUrl ?? result.feedUrl; + return ` + ${escapeXml(id)} + ${escapeXml(result.title)} + + ${escapeXml(result.lastPublishedAt ?? updated)} + ${escapeXml(result.description ?? `Discovered ${result.kind} feed from ${result.provider}`)} + `; + }) + .join("\n"); + + return ` + + logicsrc:feed-discovery:${escapeXml(response.normalizedQuery)} + LogicSRC feed discovery: ${escapeXml(response.query)} + ${updated} +${entries} +`; +} diff --git a/plugins/feed-discovery/src/output/index.ts b/plugins/feed-discovery/src/output/index.ts new file mode 100644 index 0000000..ed6e9ee --- /dev/null +++ b/plugins/feed-discovery/src/output/index.ts @@ -0,0 +1,23 @@ +import type { FeedDiscoveryResponse, FeedOutputFormat } from "../types.js"; +import { renderAtom } from "./atom.js"; +import { renderJsonFeed } from "./json-feed.js"; +import { renderOpml } from "./opml.js"; +import { renderRss } from "./rss.js"; + +export function renderDiscoveryOutput(response: FeedDiscoveryResponse, format: FeedOutputFormat) { + if (format === "opml") { + return renderOpml(response); + } + if (format === "rss") { + return renderRss(response); + } + if (format === "atom") { + return renderAtom(response); + } + if (format === "json-feed") { + return renderJsonFeed(response); + } + return JSON.stringify(response, null, 2); +} + +export { renderAtom, renderJsonFeed, renderOpml, renderRss }; diff --git a/plugins/feed-discovery/src/output/json-feed.ts b/plugins/feed-discovery/src/output/json-feed.ts new file mode 100644 index 0000000..e59404a --- /dev/null +++ b/plugins/feed-discovery/src/output/json-feed.ts @@ -0,0 +1,22 @@ +import type { FeedDiscoveryResponse } from "../types.js"; + +export function renderJsonFeed(response: FeedDiscoveryResponse) { + return JSON.stringify( + { + version: "https://jsonfeed.org/version/1.1", + title: `LogicSRC feed discovery: ${response.query}`, + home_page_url: "https://bittorrented.com/rss", + feed_url: `https://bittorrented.com/json-feed/discover/${encodeURIComponent(response.normalizedQuery)}.json`, + items: response.results.map((result) => ({ + id: result.canonicalFeedUrl ?? result.feedUrl, + url: result.homepageUrl ?? result.feedUrl, + title: result.title, + summary: result.description ?? `Discovered ${result.kind} feed from ${result.provider}`, + date_published: result.lastPublishedAt, + tags: [result.kind, result.provider, ...result.tags] + })) + }, + null, + 2 + ); +} diff --git a/plugins/feed-discovery/src/output/opml.ts b/plugins/feed-discovery/src/output/opml.ts new file mode 100644 index 0000000..f54d500 --- /dev/null +++ b/plugins/feed-discovery/src/output/opml.ts @@ -0,0 +1,27 @@ +import type { DiscoveredFeed, FeedDiscoveryResponse } from "../types.js"; + +export function renderOpml(response: FeedDiscoveryResponse) { + const outlines = response.results.map(renderOutline).join("\n"); + return ` + + + LogicSRC feed discovery: ${escapeXml(response.query)} + + +${outlines} + +`; +} + +function renderOutline(feed: DiscoveredFeed) { + return ` `; +} + +export function escapeXml(value: string) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} diff --git a/plugins/feed-discovery/src/output/rss.ts b/plugins/feed-discovery/src/output/rss.ts new file mode 100644 index 0000000..9524171 --- /dev/null +++ b/plugins/feed-discovery/src/output/rss.ts @@ -0,0 +1,25 @@ +import RSS from "rss"; +import type { FeedDiscoveryResponse } from "../types.js"; + +export function renderRss(response: FeedDiscoveryResponse) { + const feed = new RSS({ + title: `LogicSRC feed discovery: ${response.query}`, + description: `Discovered feed sources for ${response.query}`, + feed_url: `https://bittorrented.com/rss/discover/${encodeURIComponent(response.normalizedQuery)}.xml`, + site_url: "https://bittorrented.com/rss", + language: "en" + }); + + for (const result of response.results) { + feed.item({ + title: result.title, + description: result.description ?? `Discovered ${result.kind} feed from ${result.provider}`, + url: result.homepageUrl ?? result.feedUrl, + guid: result.canonicalFeedUrl ?? result.feedUrl, + categories: [result.kind, result.provider, ...result.tags], + date: result.lastPublishedAt ? new Date(result.lastPublishedAt) : new Date() + }); + } + + return feed.xml({ indent: true }); +} diff --git a/plugins/feed-discovery/src/probe-site.ts b/plugins/feed-discovery/src/probe-site.ts new file mode 100644 index 0000000..40ebdbb --- /dev/null +++ b/plugins/feed-discovery/src/probe-site.ts @@ -0,0 +1,93 @@ +import * as cheerio from "cheerio"; +import { readFeedDiscoveryConfig } from "./config.js"; +import { fetchTextWithGuards } from "./http.js"; +import type { DiscoveredFeed, FeedDiscoveryConfig, ProbeResult } from "./types.js"; +import { canonicalizeUrl } from "./url-safety.js"; +import { validateFeed } from "./validate-feed.js"; + +const COMMON_FEED_PATHS = [ + "/feed", + "/rss", + "/rss.xml", + "/atom.xml", + "/index.xml", + "/feed.xml", + "/blog/feed", + "/blog/rss.xml", + "/news/feed", + "/posts/feed" +]; + +const FEED_MIME_PATTERN = /(rss|atom|feed\+json|xml)/i; + +export async function probeSite(homepageUrl: string, config: Partial = {}): Promise { + const resolvedConfig = { ...readFeedDiscoveryConfig(), ...config }; + const errors: string[] = []; + const candidates = new Set(); + const canonicalHomepage = canonicalizeUrl(homepageUrl); + + try { + const response = await fetchTextWithGuards(canonicalHomepage, resolvedConfig); + for (const href of extractAlternateFeedLinks(response.body, response.url || canonicalHomepage)) { + candidates.add(href); + } + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + + const homepage = new URL(canonicalHomepage); + for (const path of COMMON_FEED_PATHS) { + candidates.add(new URL(path, homepage.origin).toString()); + } + + const feeds: DiscoveredFeed[] = []; + for (const candidate of [...candidates].slice(0, resolvedConfig.maxProbes)) { + const validation = await validateFeed(candidate, resolvedConfig); + if (!validation.ok) { + if (validation.error) { + errors.push(`${candidate}: ${validation.error}`); + } + continue; + } + + feeds.push({ + title: validation.title ?? candidate, + description: validation.description, + homepageUrl: validation.homepageUrl ?? canonicalHomepage, + feedUrl: candidate, + canonicalFeedUrl: validation.canonicalFeedUrl, + kind: validation.kind, + provider: "web-feed-probe", + language: validation.language, + imageUrl: validation.imageUrl, + lastPublishedAt: validation.lastPublishedAt, + score: 0, + confidence: 0.8, + tags: [], + sampleItems: validation.sampleItems, + isValid: true, + validationScore: 1 + }); + } + + return { + homepageUrl: canonicalHomepage, + feeds, + errors + }; +} + +export function extractAlternateFeedLinks(html: string, baseUrl: string) { + const $ = cheerio.load(html); + const links: string[] = []; + + $("link[rel~='alternate']").each((_, element) => { + const type = String($(element).attr("type") ?? ""); + const href = $(element).attr("href"); + if (href && FEED_MIME_PATTERN.test(type)) { + links.push(canonicalizeUrl(href, baseUrl)); + } + }); + + return [...new Set(links)]; +} diff --git a/plugins/feed-discovery/src/providers/index.ts b/plugins/feed-discovery/src/providers/index.ts new file mode 100644 index 0000000..d194c38 --- /dev/null +++ b/plugins/feed-discovery/src/providers/index.ts @@ -0,0 +1,45 @@ +import type { FeedDiscoveryConfig, FeedDiscoveryProvider, FeedProviderManifest } from "../types.js"; +import { ITunesPodcastProvider } from "./itunes-podcast.js"; +import { ManualCuratedProvider } from "./manual-curated.js"; +import { OpmlDirectoryProvider } from "./opml-directory.js"; +import { PodcastIndexProvider } from "./podcastindex.js"; +import { WebCandidateFeedProbeProvider } from "./web-feed-probe.js"; + +export function createDefaultFeedProviders(config: FeedDiscoveryConfig): FeedDiscoveryProvider[] { + return [ + new ManualCuratedProvider(), + new OpmlDirectoryProvider(config.opmlPaths), + new WebCandidateFeedProbeProvider(config), + new ITunesPodcastProvider(config), + new PodcastIndexProvider(config) + ]; +} + +export function providerManifests(config: FeedDiscoveryConfig): FeedProviderManifest[] { + return createDefaultFeedProviders(config).map((provider) => ({ + id: provider.id, + name: provider.name, + type: provider.id.includes("podcast") || provider.id.includes("itunes") ? "podcast" : "all", + requiresApiKey: provider.requiresApiKey, + enabledByDefault: provider.enabledByDefault && (!provider.requiresApiKey || hasProviderKey(provider.id, config)), + description: describeProvider(provider.id) + })); +} + +function hasProviderKey(id: string, config: FeedDiscoveryConfig) { + if (id === "podcastindex") { + return Boolean(config.podcastIndexApiKey && config.podcastIndexApiSecret); + } + return true; +} + +function describeProvider(id: string) { + const descriptions: Record = { + "manual-curated": "Searches locally curated high-trust feed sources.", + "opml-directory": "Searches configured local OPML files.", + "web-feed-probe": "Probes configured candidate homepages and direct URL queries for feeds.", + "itunes-podcast": "Searches the public iTunes podcast directory.", + podcastindex: "Searches PodcastIndex when API credentials are configured." + }; + return descriptions[id] ?? "Feed discovery provider."; +} diff --git a/plugins/feed-discovery/src/providers/itunes-podcast.ts b/plugins/feed-discovery/src/providers/itunes-podcast.ts new file mode 100644 index 0000000..bf91aa1 --- /dev/null +++ b/plugins/feed-discovery/src/providers/itunes-podcast.ts @@ -0,0 +1,64 @@ +import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js"; + +interface ITunesPodcast { + collectionName?: string; + artistName?: string; + feedUrl?: string; + collectionViewUrl?: string; + artworkUrl600?: string; + primaryGenreName?: string; + releaseDate?: string; +} + +export class ITunesPodcastProvider implements FeedDiscoveryProvider { + id = "itunes-podcast"; + name = "iTunes Podcast Search"; + enabledByDefault = true; + requiresApiKey = false; + + constructor(private readonly config: Pick) {} + + async search(query: FeedDiscoveryQuery) { + if (query.type && query.type !== "all" && query.type !== "podcast") { + return []; + } + + const url = new URL("https://itunes.apple.com/search"); + url.searchParams.set("term", query.q); + url.searchParams.set("entity", "podcast"); + url.searchParams.set("limit", String(Math.min(query.limit ?? 25, 50))); + if (query.locale) { + url.searchParams.set("country", query.locale.toUpperCase()); + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs); + try { + const response = await fetch(url, { + signal: controller.signal, + headers: { "user-agent": this.config.userAgent } + }); + if (!response.ok) { + throw new Error(`iTunes HTTP ${response.status}`); + } + const body = await response.json() as { results?: ITunesPodcast[] }; + return (body.results ?? []) + .filter((item) => item.feedUrl) + .map((item): DiscoveredFeed => ({ + title: item.collectionName ?? item.feedUrl ?? "Untitled podcast", + description: item.artistName, + homepageUrl: item.collectionViewUrl, + feedUrl: item.feedUrl as string, + kind: "podcast", + provider: this.id, + imageUrl: item.artworkUrl600, + lastPublishedAt: item.releaseDate, + score: 0, + confidence: 0.85, + tags: [item.primaryGenreName, "podcast"].filter((tag): tag is string => Boolean(tag)) + })); + } finally { + clearTimeout(timeout); + } + } +} diff --git a/plugins/feed-discovery/src/providers/manual-curated.ts b/plugins/feed-discovery/src/providers/manual-curated.ts new file mode 100644 index 0000000..7dd13aa --- /dev/null +++ b/plugins/feed-discovery/src/providers/manual-curated.ts @@ -0,0 +1,59 @@ +import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js"; + +const CURATED_FEEDS: DiscoveredFeed[] = [ + { + title: "Indie Hackers", + description: "Stories and discussions from founders building profitable internet businesses.", + homepageUrl: "https://www.indiehackers.com", + feedUrl: "https://www.indiehackers.com/feed", + kind: "blog", + provider: "manual-curated", + score: 0, + confidence: 0.8, + tags: ["indie", "startup", "microsaas", "saas"] + }, + { + title: "Hacker News", + description: "Technology, startup, software, and open-source discussion.", + homepageUrl: "https://news.ycombinator.com", + feedUrl: "https://news.ycombinator.com/rss", + kind: "news", + provider: "manual-curated", + score: 0, + confidence: 0.75, + tags: ["startups", "software", "open-source", "technology"] + }, + { + title: "GitHub Blog", + description: "GitHub product, open-source, and developer ecosystem news.", + homepageUrl: "https://github.blog", + feedUrl: "https://github.blog/feed/", + kind: "github", + provider: "manual-curated", + score: 0, + confidence: 0.7, + tags: ["github", "open-source", "software", "developers"] + } +]; + +export class ManualCuratedProvider implements FeedDiscoveryProvider { + id = "manual-curated"; + name = "Manual Curated"; + enabledByDefault = true; + requiresApiKey = false; + + async search(query: FeedDiscoveryQuery) { + const terms = normalize(query.q); + return CURATED_FEEDS.filter((feed) => { + if (query.type && query.type !== "all" && feed.kind !== query.type) { + return false; + } + const haystack = normalize([feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" ")); + return terms.length === 0 || terms.some((term) => haystack.includes(term)); + }).map((feed) => ({ ...feed, provider: this.id })); + } +} + +function normalize(value: string) { + return value.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); +} diff --git a/plugins/feed-discovery/src/providers/opml-directory.ts b/plugins/feed-discovery/src/providers/opml-directory.ts new file mode 100644 index 0000000..d517ebe --- /dev/null +++ b/plugins/feed-discovery/src/providers/opml-directory.ts @@ -0,0 +1,92 @@ +import { readFile } from "node:fs/promises"; +import { XMLParser } from "fast-xml-parser"; +import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js"; + +const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: "", + textNodeName: "text" +}); + +export class OpmlDirectoryProvider implements FeedDiscoveryProvider { + id = "opml-directory"; + name = "OPML Directory"; + enabledByDefault = true; + requiresApiKey = false; + + constructor(private readonly paths: string[] = []) {} + + async search(query: FeedDiscoveryQuery) { + const feeds: DiscoveredFeed[] = []; + for (const path of this.paths) { + const content = await readFile(path, "utf8"); + const parsed = parser.parse(content) as unknown; + for (const outline of collectOutlines(parsed)) { + const feedUrl = stringValue(outline.xmlUrl); + if (!feedUrl) { + continue; + } + const title = stringValue(outline.title) || stringValue(outline.text) || feedUrl; + const tags = [stringValue(outline.category)].filter((tag): tag is string => Boolean(tag)); + const feed: DiscoveredFeed = { + title, + description: stringValue(outline.description), + homepageUrl: stringValue(outline.htmlUrl), + feedUrl, + kind: inferKind(tags.join(" "), query.type), + provider: this.id, + score: 0, + confidence: 0.7, + tags + }; + if (matches(feed, query)) { + feeds.push(feed); + } + } + } + return feeds; + } +} + +function collectOutlines(value: unknown): Array> { + if (!isRecord(value)) { + return []; + } + const current = "xmlUrl" in value ? [value] : []; + return [ + ...current, + ...Object.values(value).flatMap((entry) => { + if (Array.isArray(entry)) { + return entry.flatMap(collectOutlines); + } + return collectOutlines(entry); + }) + ]; +} + +function matches(feed: DiscoveredFeed, query: FeedDiscoveryQuery) { + if (query.type && query.type !== "all" && feed.kind !== query.type) { + return false; + } + const terms = query.q.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean); + const haystack = [feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" ").toLowerCase(); + return terms.length === 0 || terms.some((term) => haystack.includes(term)); +} + +function inferKind(tags: string, requested?: FeedDiscoveryQuery["type"]) { + if (requested && requested !== "all") { + return requested; + } + if (tags.includes("podcast")) { + return "podcast"; + } + return "opml"; +} + +function stringValue(value: unknown) { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/plugins/feed-discovery/src/providers/podcastindex.ts b/plugins/feed-discovery/src/providers/podcastindex.ts new file mode 100644 index 0000000..fcb9084 --- /dev/null +++ b/plugins/feed-discovery/src/providers/podcastindex.ts @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js"; + +interface PodcastIndexFeed { + title?: string; + description?: string; + url?: string; + link?: string; + image?: string; + language?: string; + newestItemPublishTime?: number; + categories?: Record; +} + +export class PodcastIndexProvider implements FeedDiscoveryProvider { + id = "podcastindex"; + name = "PodcastIndex"; + enabledByDefault = false; + requiresApiKey = true; + + constructor(private readonly config: FeedDiscoveryConfig) {} + + async search(query: FeedDiscoveryQuery) { + if (!this.config.podcastIndexApiKey || !this.config.podcastIndexApiSecret) { + return []; + } + if (query.type && query.type !== "all" && query.type !== "podcast") { + return []; + } + + const authDate = Math.floor(Date.now() / 1000).toString(); + const auth = createHash("sha1") + .update(this.config.podcastIndexApiKey + this.config.podcastIndexApiSecret + authDate) + .digest("hex"); + + const url = new URL("https://api.podcastindex.org/api/1.0/search/byterm"); + url.searchParams.set("q", query.q); + url.searchParams.set("max", String(Math.min(query.limit ?? 25, 100))); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs); + try { + const response = await fetch(url, { + signal: controller.signal, + headers: { + "user-agent": this.config.userAgent, + "X-Auth-Date": authDate, + "X-Auth-Key": this.config.podcastIndexApiKey, + Authorization: auth + } + }); + if (!response.ok) { + throw new Error(`PodcastIndex HTTP ${response.status}`); + } + const body = await response.json() as { feeds?: PodcastIndexFeed[] }; + return (body.feeds ?? []) + .filter((feed) => feed.url) + .map((feed): DiscoveredFeed => ({ + title: feed.title ?? feed.url ?? "Untitled podcast", + description: feed.description, + homepageUrl: feed.link, + feedUrl: feed.url as string, + kind: "podcast", + provider: this.id, + language: feed.language, + imageUrl: feed.image, + lastPublishedAt: feed.newestItemPublishTime ? new Date(feed.newestItemPublishTime * 1000).toISOString() : undefined, + score: 0, + confidence: 0.9, + tags: [...Object.values(feed.categories ?? {}), "podcast"] + })); + } finally { + clearTimeout(timeout); + } + } +} diff --git a/plugins/feed-discovery/src/providers/web-feed-probe.ts b/plugins/feed-discovery/src/providers/web-feed-probe.ts new file mode 100644 index 0000000..16d27ef --- /dev/null +++ b/plugins/feed-discovery/src/providers/web-feed-probe.ts @@ -0,0 +1,42 @@ +import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js"; +import { probeSite } from "../probe-site.js"; + +export class WebCandidateFeedProbeProvider implements FeedDiscoveryProvider { + id = "web-feed-probe"; + name = "Web Candidate Feed Probe"; + enabledByDefault = true; + requiresApiKey = false; + + constructor(private readonly config: FeedDiscoveryConfig) {} + + async search(query: FeedDiscoveryQuery) { + const candidates = candidateUrls(query, this.config.candidateUrls).slice(0, this.config.maxProbes); + const feeds: DiscoveredFeed[] = []; + + for (const candidate of candidates) { + const result = await probeSite(candidate, this.config); + feeds.push(...result.feeds.map((feed) => ({ ...feed, provider: this.id }))); + } + + return feeds.filter((feed) => !query.type || query.type === "all" || feed.kind === query.type); + } +} + +function candidateUrls(query: FeedDiscoveryQuery, configured: string[]) { + const candidates = new Set(); + if (/^https?:\/\//i.test(query.q)) { + candidates.add(query.q); + } + + for (const entry of configured) { + const [url, ...keywords] = entry.split("|").map((part) => part.trim()); + if (!url) { + continue; + } + if (keywords.length === 0 || keywords.some((keyword) => query.q.toLowerCase().includes(keyword.toLowerCase()))) { + candidates.add(url); + } + } + + return [...candidates]; +} diff --git a/plugins/feed-discovery/src/scoring.ts b/plugins/feed-discovery/src/scoring.ts new file mode 100644 index 0000000..9036a3e --- /dev/null +++ b/plugins/feed-discovery/src/scoring.ts @@ -0,0 +1,95 @@ +import type { DiscoveredFeed, FeedDiscoveryQuery } from "./types.js"; + +const PROVIDER_SCORES: Record = { + "manual-curated": 1, + podcastindex: 0.9, + "itunes-podcast": 0.85, + "opml-directory": 0.75, + "web-feed-probe": 0.7, + rsshub: 0.7, + reddit: 0.65, + github: 0.65, + youtube: 0.65, + unknown: 0.3 +}; + +export function scoreFeed(feed: DiscoveredFeed, query: FeedDiscoveryQuery): DiscoveredFeed { + const keywordScore = feed.keywordScore ?? calculateKeywordScore(feed, query.q); + const freshnessScore = feed.freshnessScore ?? calculateFreshnessScore(feed.lastPublishedAt); + const providerScore = feed.providerScore ?? PROVIDER_SCORES[feed.provider] ?? PROVIDER_SCORES.unknown; + const validationScore = feed.validationScore ?? (feed.isValid === false ? 0.2 : 1); + const score = keywordScore * 0.4 + freshnessScore * 0.25 + providerScore * 0.2 + validationScore * 0.15; + + return { + ...feed, + score: round(score), + confidence: round(Math.max(feed.confidence, score * validationScore)), + freshnessScore: round(freshnessScore), + keywordScore: round(keywordScore), + providerScore: round(providerScore), + validationScore: round(validationScore) + }; +} + +export function calculateKeywordScore(feed: DiscoveredFeed, query: string) { + const terms = normalizeTerms(query); + if (terms.length === 0) { + return 0.1; + } + + const weightedText = [ + [feed.title, 0.35], + [feed.description, 0.2], + [feed.homepageUrl, 0.1], + [feed.feedUrl, 0.05], + [feed.tags.join(" "), 0.15], + [feed.sampleItems?.map((item) => item.title).join(" "), 0.15] + ] as const; + + let score = 0; + for (const [value, weight] of weightedText) { + const text = String(value ?? "").toLowerCase(); + if (!text) { + continue; + } + const matches = terms.filter((term) => text.includes(term)).length; + score += weight * (matches / terms.length); + } + + return Math.min(1, Math.max(0.05, score)); +} + +export function calculateFreshnessScore(lastPublishedAt?: string) { + if (!lastPublishedAt) { + return 0.1; + } + const ageDays = (Date.now() - Date.parse(lastPublishedAt)) / 86_400_000; + if (!Number.isFinite(ageDays) || ageDays < 0) { + return 0.1; + } + if (ageDays <= 7) { + return 1; + } + if (ageDays <= 30) { + return 0.8; + } + if (ageDays <= 90) { + return 0.6; + } + if (ageDays <= 365) { + return 0.3; + } + return 0.1; +} + +function normalizeTerms(query: string) { + return query + .toLowerCase() + .split(/[^a-z0-9]+/) + .map((term) => term.trim()) + .filter(Boolean); +} + +function round(value: number) { + return Math.round(value * 1000) / 1000; +} diff --git a/plugins/feed-discovery/src/types.ts b/plugins/feed-discovery/src/types.ts new file mode 100644 index 0000000..b5abaa4 --- /dev/null +++ b/plugins/feed-discovery/src/types.ts @@ -0,0 +1,113 @@ +export type FeedKind = + | "blog" + | "news" + | "podcast" + | "youtube" + | "reddit" + | "github" + | "torrent" + | "opml" + | "rsshub" + | "unknown"; + +export type FeedOutputFormat = "json" | "opml" | "rss" | "atom" | "json-feed"; + +export interface FeedDiscoveryQuery { + q: string; + type?: FeedKind | "all"; + limit?: number; + locale?: string; + freshnessDays?: number; + includeDeadFeeds?: boolean; + includeUnvalidated?: boolean; + providers?: string[]; +} + +export interface DiscoveredFeed { + id?: string; + title: string; + description?: string; + homepageUrl?: string; + feedUrl: string; + canonicalFeedUrl?: string; + kind: FeedKind; + provider: string; + language?: string; + imageUrl?: string; + lastPublishedAt?: string; + score: number; + confidence: number; + freshnessScore?: number; + keywordScore?: number; + providerScore?: number; + validationScore?: number; + tags: string[]; + sampleItems?: FeedSampleItem[]; + isValid?: boolean; +} + +export interface FeedSampleItem { + title: string; + url?: string; + publishedAt?: string; + description?: string; +} + +export interface FeedDiscoveryProvider { + id: string; + name: string; + enabledByDefault: boolean; + requiresApiKey: boolean; + search(query: FeedDiscoveryQuery): Promise; +} + +export interface FeedProviderManifest { + id: string; + name: string; + type: FeedKind | "all"; + requiresApiKey: boolean; + enabledByDefault: boolean; + description: string; +} + +export interface FeedDiscoveryResponse { + query: string; + normalizedQuery: string; + count: number; + providerErrors: Array<{ provider: string; error: string }>; + results: DiscoveredFeed[]; +} + +export interface FeedDiscoveryConfig { + cacheTtlSeconds: number; + maxProviders: number; + maxProbes: number; + requestTimeoutMs: number; + maxBodyBytes: number; + userAgent: string; + opmlPaths: string[]; + candidateUrls: string[]; + podcastIndexApiKey?: string; + podcastIndexApiSecret?: string; +} + +export interface ValidationResult { + ok: boolean; + feedUrl: string; + canonicalFeedUrl?: string; + title?: string; + description?: string; + homepageUrl?: string; + kind: FeedKind; + language?: string; + imageUrl?: string; + lastPublishedAt?: string; + sampleItems: FeedSampleItem[]; + error?: string; +} + +export interface ProbeResult { + homepageUrl: string; + feeds: DiscoveredFeed[]; + errors: string[]; +} diff --git a/plugins/feed-discovery/src/url-safety.ts b/plugins/feed-discovery/src/url-safety.ts new file mode 100644 index 0000000..c2e6d25 --- /dev/null +++ b/plugins/feed-discovery/src/url-safety.ts @@ -0,0 +1,89 @@ +import { lookup } from "node:dns/promises"; +import { isIP } from "node:net"; + +const BLOCKED_HOSTS = new Set(["localhost", "metadata.google.internal"]); + +export async function assertSafeHttpUrl(input: string) { + const url = new URL(input); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Unsupported URL protocol: ${url.protocol}`); + } + + const hostname = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1"); + if (BLOCKED_HOSTS.has(hostname) || hostname.endsWith(".localhost")) { + throw new Error(`Blocked internal hostname: ${hostname}`); + } + + if (isBlockedIp(hostname)) { + throw new Error(`Blocked internal IP address: ${hostname}`); + } + + const addresses = await lookup(hostname, { all: true, verbatim: true }); + for (const address of addresses) { + if (isBlockedIp(address.address)) { + throw new Error(`Blocked internal resolved address: ${address.address}`); + } + } + + return url; +} + +export function canonicalizeUrl(input: string, base?: string) { + const url = new URL(input, base); + url.hash = ""; + + for (const param of [...url.searchParams.keys()]) { + if (/^(utm_|fbclid$|gclid$|mc_cid$|mc_eid$)/i.test(param)) { + url.searchParams.delete(param); + } + } + + if ((url.protocol === "https:" && url.port === "443") || (url.protocol === "http:" && url.port === "80")) { + url.port = ""; + } + + if (url.pathname !== "/" && url.pathname.endsWith("/")) { + url.pathname = url.pathname.slice(0, -1); + } + + return url.toString(); +} + +function isBlockedIp(value: string) { + const kind = isIP(value); + if (kind === 4) { + const parts = value.split(".").map((part) => Number(part)); + const [a, b] = parts; + return ( + a === 0 || + a === 10 || + a === 127 || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 100 && b >= 64 && b <= 127) + ); + } + + if (kind === 6) { + const normalized = value.toLowerCase(); + const mappedDottedIpv4 = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(normalized); + if (mappedDottedIpv4) { + return isBlockedIp(mappedDottedIpv4[1]); + } + const mappedHexIpv4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(normalized); + if (mappedHexIpv4) { + const high = Number.parseInt(mappedHexIpv4[1], 16); + const low = Number.parseInt(mappedHexIpv4[2], 16); + return isBlockedIp(`${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`); + } + return ( + normalized === "::1" || + normalized.startsWith("fc") || + normalized.startsWith("fd") || + normalized.startsWith("fe80") + ); + } + + return false; +} diff --git a/plugins/feed-discovery/src/validate-feed.ts b/plugins/feed-discovery/src/validate-feed.ts new file mode 100644 index 0000000..0076d9a --- /dev/null +++ b/plugins/feed-discovery/src/validate-feed.ts @@ -0,0 +1,27 @@ +import { readFeedDiscoveryConfig } from "./config.js"; +import { parseFeedDocument } from "./feed-parsing.js"; +import { fetchTextWithGuards } from "./http.js"; +import type { FeedDiscoveryConfig, ValidationResult } from "./types.js"; +import { canonicalizeUrl } from "./url-safety.js"; + +export async function validateFeed(feedUrl: string, config: Partial = {}): Promise { + const resolvedConfig = { ...readFeedDiscoveryConfig(), ...config }; + try { + const response = await fetchTextWithGuards(feedUrl, resolvedConfig); + const canonicalFeedUrl = canonicalizeUrl(response.url || feedUrl); + const parsed = parseFeedDocument(canonicalFeedUrl, response.body, response.contentType); + return { + ...parsed, + feedUrl, + canonicalFeedUrl + }; + } catch (error) { + return { + ok: false, + feedUrl, + kind: "unknown", + sampleItems: [], + error: error instanceof Error ? error.message : String(error) + }; + } +} diff --git a/plugins/feed-discovery/tsconfig.json b/plugins/feed-discovery/tsconfig.json new file mode 100644 index 0000000..df59da5 --- /dev/null +++ b/plugins/feed-discovery/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"] +}