mirror of
https://github.com/profullstack/logicsrc.git
synced 2026-08-14 23:07:29 +00:00
Add feed discovery plugin
This commit is contained in:
parent
e47616bf9d
commit
5cfeea6b57
37 changed files with 2432 additions and 5 deletions
|
|
@ -15,6 +15,7 @@
|
|||
"@logicsrc/plugin-core": "file:../../packages/plugin-core",
|
||||
"@logicsrc/plugin-c0mpute": "file:../../plugins/c0mpute",
|
||||
"@logicsrc/plugin-coinpay": "file:../../plugins/coinpay",
|
||||
"@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery",
|
||||
"@logicsrc/plugin-sh1pt": "file:../../plugins/sh1pt",
|
||||
"@logicsrc/plugin-ugig": "file:../../plugins/ugig",
|
||||
"@logicsrc/validators": "file:../../packages/validators"
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ describe("CommandBoard API contracts", () => {
|
|||
};
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute"]);
|
||||
expect(body.plugins.map((plugin) => plugin.id)).toEqual(["coinpay", "ugig", "sh1pt", "c0mpute", "feed-discovery"]);
|
||||
expect(body.plugins.find((plugin) => plugin.id === "sh1pt")).toMatchObject({
|
||||
enabled: true,
|
||||
capabilities: expect.arrayContaining(["projects.sync", "actions.publish", "deployments.status"])
|
||||
|
|
@ -58,6 +58,28 @@ describe("CommandBoard API contracts", () => {
|
|||
});
|
||||
expect(body.capabilities["actions.publish"]).toEqual(["sh1pt"]);
|
||||
expect(body.capabilities["compute.jobs.dispatch"]).toEqual(["c0mpute"]);
|
||||
expect(body.capabilities["feeds.discover"]).toEqual(["feed-discovery"]);
|
||||
});
|
||||
|
||||
it("exposes feed discovery plugin endpoints", async () => {
|
||||
const providersResponse = await fetch(`${baseUrl}/api/feeds/providers`);
|
||||
const providersBody = await providersResponse.json() as { providers: Array<{ id: string; enabledByDefault: boolean }> };
|
||||
const discoverResponse = await fetch(`${baseUrl}/api/feeds/discover?q=microsaas&providers=manual-curated&includeUnvalidated=true`);
|
||||
const discoverBody = await discoverResponse.json() as { query: string; results: Array<{ feedUrl: string; provider: string }> };
|
||||
const rssResponse = await fetch(`${baseUrl}/rss/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`);
|
||||
const rssBody = await rssResponse.text();
|
||||
const opmlResponse = await fetch(`${baseUrl}/opml/discover/microsaas.xml?providers=manual-curated&includeUnvalidated=true`);
|
||||
const opmlBody = await opmlResponse.text();
|
||||
|
||||
expect(providersResponse.status).toBe(200);
|
||||
expect(providersBody.providers.map((provider) => provider.id)).toContain("manual-curated");
|
||||
expect(discoverResponse.status).toBe(200);
|
||||
expect(discoverBody.query).toBe("microsaas");
|
||||
expect(discoverBody.results[0]).toMatchObject({ provider: "manual-curated" });
|
||||
expect(rssResponse.status).toBe(200);
|
||||
expect(rssBody).toContain("<rss");
|
||||
expect(opmlResponse.status).toBe(200);
|
||||
expect(opmlBody).toContain("<opml");
|
||||
});
|
||||
|
||||
it("exposes sh1pt project and action contracts", async () => {
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@ import { pathToFileURL } from "node:url";
|
|||
import { createPluginRegistry } from "@logicsrc/plugin-core";
|
||||
import { c0mputePlugin } from "@logicsrc/plugin-c0mpute";
|
||||
import { coinPayPlugin } from "@logicsrc/plugin-coinpay";
|
||||
import { discoverFeeds, feedDiscoveryPlugin, listFeedProviders, renderAtom, renderJsonFeed, renderOpml, renderRss, type FeedKind } from "@logicsrc/plugin-feed-discovery";
|
||||
import { sh1ptPlugin } from "@logicsrc/plugin-sh1pt";
|
||||
import { uGigPlugin } from "@logicsrc/plugin-ugig";
|
||||
import { schemas, validate } from "@logicsrc/validators";
|
||||
|
||||
const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin]);
|
||||
const registry = createPluginRegistry([coinPayPlugin, uGigPlugin, sh1ptPlugin, c0mputePlugin, feedDiscoveryPlugin]);
|
||||
|
||||
const boards = [
|
||||
{ path: "/general", title: "General", description: "CommandBoard.run general discussion." },
|
||||
|
|
@ -69,7 +70,7 @@ async function route(request: IncomingMessage, response: ServerResponse) {
|
|||
json(response, 200, {
|
||||
ok: true,
|
||||
service: "commandboard-api",
|
||||
endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas"]
|
||||
endpoints: ["/health", "/api/boards", "/api/tasks", "/api/plugins", "/api/schemas", "/api/feeds/discover", "/api/feeds/providers", "/rss/discover/:keyword.xml", "/opml/discover/:keyword.xml", "/atom/discover/:keyword.xml", "/json-feed/discover/:keyword.json"]
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
|
@ -113,6 +114,53 @@ async function route(request: IncomingMessage, response: ServerResponse) {
|
|||
return;
|
||||
}
|
||||
|
||||
if (request.method === "GET" && (url.pathname === "/api/feeds/providers" || url.pathname === "/api/rss/providers")) {
|
||||
json(response, 200, { providers: listFeedProviders() });
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === "GET" && (url.pathname === "/api/feeds/discover" || url.pathname === "/api/rss/discover")) {
|
||||
const query = url.searchParams.get("q");
|
||||
if (!query) {
|
||||
json(response, 422, { error: "Expected q query parameter" });
|
||||
return;
|
||||
}
|
||||
const result = await discoverFeeds({
|
||||
q: query,
|
||||
type: (url.searchParams.get("type") ?? "all") as FeedKind | "all",
|
||||
limit: numberParam(url.searchParams.get("limit")),
|
||||
providers: listParam(url.searchParams.get("providers")),
|
||||
includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true"
|
||||
});
|
||||
json(response, 200, result);
|
||||
return;
|
||||
}
|
||||
|
||||
const formattedDiscover = matchFormattedDiscoverPath(url.pathname);
|
||||
if (request.method === "GET" && formattedDiscover) {
|
||||
const result = await discoverFeeds({
|
||||
q: formattedDiscover.keyword,
|
||||
type: "all",
|
||||
limit: numberParam(url.searchParams.get("limit")),
|
||||
providers: listParam(url.searchParams.get("providers")),
|
||||
includeUnvalidated: url.searchParams.get("includeUnvalidated") === "true"
|
||||
});
|
||||
if (formattedDiscover.format === "rss") {
|
||||
text(response, 200, "application/rss+xml; charset=utf-8", renderRss(result));
|
||||
return;
|
||||
}
|
||||
if (formattedDiscover.format === "opml") {
|
||||
text(response, 200, "text/x-opml; charset=utf-8", renderOpml(result));
|
||||
return;
|
||||
}
|
||||
if (formattedDiscover.format === "atom") {
|
||||
text(response, 200, "application/atom+xml; charset=utf-8", renderAtom(result));
|
||||
return;
|
||||
}
|
||||
text(response, 200, "application/feed+json; charset=utf-8", renderJsonFeed(result));
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === "GET" && url.pathname === "/api/plugins/sh1pt/projects") {
|
||||
json(response, 200, { projects: sh1ptProjects });
|
||||
return;
|
||||
|
|
@ -194,6 +242,11 @@ function json(response: ServerResponse, status: number, data: unknown) {
|
|||
response.end(JSON.stringify(data, null, 2));
|
||||
}
|
||||
|
||||
function text(response: ServerResponse, status: number, contentType: string, body: string) {
|
||||
response.writeHead(status, { "content-type": contentType });
|
||||
response.end(body);
|
||||
}
|
||||
|
||||
async function readJson(request: IncomingMessage) {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) {
|
||||
|
|
@ -207,6 +260,41 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function numberParam(value: string | null) {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function listParam(value: string | null) {
|
||||
return value
|
||||
?.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function matchFormattedDiscoverPath(pathname: string) {
|
||||
const rss = /^\/rss\/discover\/(.+)\.xml$/.exec(pathname);
|
||||
if (rss) {
|
||||
return { format: "rss" as const, keyword: decodeURIComponent(rss[1]) };
|
||||
}
|
||||
const opml = /^\/opml\/discover\/(.+)\.xml$/.exec(pathname) ?? /^\/rss\/discover\/(.+)\.opml$/.exec(pathname);
|
||||
if (opml) {
|
||||
return { format: "opml" as const, keyword: decodeURIComponent(opml[1]) };
|
||||
}
|
||||
const atom = /^\/atom\/discover\/(.+)\.xml$/.exec(pathname);
|
||||
if (atom) {
|
||||
return { format: "atom" as const, keyword: decodeURIComponent(atom[1]) };
|
||||
}
|
||||
const jsonFeed = /^\/json-feed\/discover\/(.+)\.json$/.exec(pathname);
|
||||
if (jsonFeed) {
|
||||
return { format: "json-feed" as const, keyword: decodeURIComponent(jsonFeed[1]) };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function startCommandBoardServer(port = Number(process.env.PORT ?? 4010)) {
|
||||
const server = createCommandBoardServer();
|
||||
server.listen(port, () => {
|
||||
|
|
|
|||
470
package-lock.json
generated
470
package-lock.json
generated
|
|
@ -26,6 +26,7 @@
|
|||
"@logicsrc/plugin-c0mpute": "file:../../plugins/c0mpute",
|
||||
"@logicsrc/plugin-coinpay": "file:../../plugins/coinpay",
|
||||
"@logicsrc/plugin-core": "file:../../packages/plugin-core",
|
||||
"@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery",
|
||||
"@logicsrc/plugin-sh1pt": "file:../../plugins/sh1pt",
|
||||
"@logicsrc/plugin-ugig": "file:../../plugins/ugig",
|
||||
"@logicsrc/validators": "file:../../packages/validators"
|
||||
|
|
@ -1447,6 +1448,10 @@
|
|||
"resolved": "packages/plugin-core",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@logicsrc/plugin-feed-discovery": {
|
||||
"resolved": "plugins/feed-discovery",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@logicsrc/plugin-sh1pt": {
|
||||
"resolved": "plugins/sh1pt",
|
||||
"link": true
|
||||
|
|
@ -1679,6 +1684,18 @@
|
|||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@nodable/entities": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.1.tgz",
|
||||
"integrity": "sha512-Pig3HxDIoMgjdEH8OCf/dkcTmLFjJRjWuq8jSnklu284/TKOPibSRERmOykiwmyXTtv61mP+44f3GMx0tLAyjg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/nodable"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@oxc-project/types": {
|
||||
"version": "0.133.0",
|
||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz",
|
||||
|
|
@ -2551,6 +2568,13 @@
|
|||
"@types/react": "^19.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/rss": {
|
||||
"version": "0.0.32",
|
||||
"resolved": "https://registry.npmjs.org/@types/rss/-/rss-0.0.32.tgz",
|
||||
"integrity": "sha512-2oKNqKyUY4RSdvl5eZR1n2Q9yvw3XTe3mQHsFPn9alaNBxfPnbXBtGP8R0SV8pK1PrVnLul0zx7izbm5/gF5Qw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@vitejs/plugin-react": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.2.0.tgz",
|
||||
|
|
@ -2776,6 +2800,12 @@
|
|||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/browserslist": {
|
||||
"version": "4.28.2",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz",
|
||||
|
|
@ -2877,6 +2907,48 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
|
||||
"integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cheerio-select": "^2.1.0",
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.2.2",
|
||||
"encoding-sniffer": "^0.2.1",
|
||||
"htmlparser2": "^10.1.0",
|
||||
"parse5": "^7.3.0",
|
||||
"parse5-htmlparser2-tree-adapter": "^7.1.0",
|
||||
"parse5-parser-stream": "^7.1.2",
|
||||
"undici": "^7.19.0",
|
||||
"whatwg-mimetype": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/cheerio?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/cheerio-select": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
|
||||
"integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-select": "^5.1.0",
|
||||
"css-what": "^6.1.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.0.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/client-only": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
|
||||
|
|
@ -2969,6 +3041,34 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/css-select": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
|
||||
"integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0",
|
||||
"css-what": "^6.1.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"domutils": "^3.0.1",
|
||||
"nth-check": "^2.0.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/css-what": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
|
||||
"integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.2.3",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz",
|
||||
|
|
@ -3011,6 +3111,61 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/dom-serializer": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
|
||||
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"entities": "^4.2.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/domelementtype": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
|
||||
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/domhandler": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
|
||||
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domhandler?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/domutils": {
|
||||
"version": "3.2.2",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
|
||||
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domutils?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
|
|
@ -3046,6 +3201,43 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding-sniffer": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
|
||||
"integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "^0.6.3",
|
||||
"whatwg-encoding": "^3.1.1"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/encoding-sniffer/node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
|
||||
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/es-define-property": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
|
||||
|
|
@ -3272,6 +3464,44 @@
|
|||
],
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/fast-xml-builder": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz",
|
||||
"integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"path-expression-matcher": "^1.5.0",
|
||||
"xml-naming": "^0.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fast-xml-parser": {
|
||||
"version": "5.8.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.8.0.tgz",
|
||||
"integrity": "sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@nodable/entities": "^2.1.0",
|
||||
"fast-xml-builder": "^1.2.0",
|
||||
"path-expression-matcher": "^1.5.0",
|
||||
"strnum": "^2.3.0",
|
||||
"xml-naming": "^0.1.0"
|
||||
},
|
||||
"bin": {
|
||||
"fxparser": "src/cli/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/fdir": {
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
|
||||
|
|
@ -3442,6 +3672,37 @@
|
|||
"node": ">=16.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/htmlparser2": {
|
||||
"version": "10.1.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
|
||||
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
|
||||
"funding": [
|
||||
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.2.2",
|
||||
"entities": "^7.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/htmlparser2/node_modules/entities": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
|
||||
"integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/http-errors": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
|
||||
|
|
@ -4044,6 +4305,18 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/nth-check": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
|
||||
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
|
|
@ -4100,6 +4373,55 @@
|
|||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
|
||||
"integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"entities": "^6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-htmlparser2-tree-adapter": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
|
||||
"integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"domhandler": "^5.0.3",
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5-parser-stream": {
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
|
||||
"integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"parse5": "^7.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/inikulin/parse5?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parse5/node_modules/entities": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
|
||||
"integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/entities?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/parseurl": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
|
||||
|
|
@ -4109,6 +4431,21 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/path-expression-matcher": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz",
|
||||
"integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/path-key": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
|
||||
|
|
@ -4426,6 +4763,37 @@
|
|||
"node": ">= 18"
|
||||
}
|
||||
},
|
||||
"node_modules/rss": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/rss/-/rss-1.2.2.tgz",
|
||||
"integrity": "sha512-xUhRTgslHeCBeHAqaWSbOYTydN2f0tAzNXvzh3stjz7QDhQMzdgHf3pfgNIngeytQflrFPfy6axHilTETr6gDg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-types": "2.1.13",
|
||||
"xml": "1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/rss/node_modules/mime-db": {
|
||||
"version": "1.25.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.25.0.tgz",
|
||||
"integrity": "sha512-5k547tI4Cy+Lddr/hdjNbBEWBwSl8EBc5aSdKvedav8DReADgWJzcYiktaRIw3GtGC1jjwldXtTzvqJZmtvC7w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/rss/node_modules/mime-types": {
|
||||
"version": "2.1.13",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.13.tgz",
|
||||
"integrity": "sha512-ryBDp1Z/6X90UvjUK3RksH0IBPM137T7cmg4OgD5wQBojlAiUwuok0QeELkim/72EtcYuNlmbkrcGuxj3Kl0YQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mime-db": "~1.25.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
|
|
@ -4656,6 +5024,15 @@
|
|||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/slugify": {
|
||||
"version": "1.6.9",
|
||||
"resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz",
|
||||
"integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
|
|
@ -4688,6 +5065,18 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/strnum": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/strnum/-/strnum-2.3.0.tgz",
|
||||
"integrity": "sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
}
|
||||
],
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/styled-jsx": {
|
||||
"version": "5.1.6",
|
||||
"resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz",
|
||||
|
|
@ -5290,6 +5679,15 @@
|
|||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.27.2",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.27.2.tgz",
|
||||
"integrity": "sha512-uZsKNuzQxDMUY6M3pIMvy5tvlGmtq8XJ2oLAkfRKGNu+1VQAIvLy2xIVG5ATZl5wDXl/tddByAWCizRbOme+TA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.18.2",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
|
||||
|
|
@ -5512,6 +5910,40 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
|
||||
"integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
|
||||
"deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding/node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-mimetype": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
|
||||
"integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
|
@ -5550,6 +5982,27 @@
|
|||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/xml": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz",
|
||||
"integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/xml-naming": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz",
|
||||
"integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/NaturalIntelligence"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
|
|
@ -5595,6 +6048,7 @@
|
|||
"dependencies": {
|
||||
"@logicsrc/plugin-coinpay": "file:../../plugins/coinpay",
|
||||
"@logicsrc/plugin-core": "file:../plugin-core",
|
||||
"@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery",
|
||||
"@logicsrc/plugin-ugig": "file:../../plugins/ugig",
|
||||
"@logicsrc/tui": "file:../tui",
|
||||
"@logicsrc/validators": "file:../validators",
|
||||
|
|
@ -5688,6 +6142,22 @@
|
|||
"vitest": "^4.0.8"
|
||||
}
|
||||
},
|
||||
"plugins/feed-discovery": {
|
||||
"name": "@logicsrc/plugin-feed-discovery",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@logicsrc/plugin-core": "file:../../packages/plugin-core",
|
||||
"cheerio": "^1.0.0",
|
||||
"fast-xml-parser": "^5.2.5",
|
||||
"rss": "^1.2.2",
|
||||
"slugify": "^1.6.6",
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/rss": "^0.0.32",
|
||||
"vitest": "^4.0.8"
|
||||
}
|
||||
},
|
||||
"plugins/sh1pt": {
|
||||
"name": "@logicsrc/plugin-sh1pt",
|
||||
"version": "0.1.0",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
"apps/*"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "npm --workspace @logicsrc/schemas run build && npm --workspace @logicsrc/validators run build && npm --workspace @logicsrc/sdk run build && npm --workspace @logicsrc/plugin-core run build && npm --workspace @logicsrc/plugin-coinpay run build && npm --workspace @logicsrc/plugin-ugig run build && npm --workspace @logicsrc/plugin-sh1pt run build && npm --workspace @logicsrc/plugin-c0mpute run build && npm --workspace @logicsrc/tui run build && npm --workspace @logicsrc/cli run build && npm --workspace @profullstack/logicsrc-mcp run build && npm --workspace @logicsrc/commandboard-api run build && npm --workspace @logicsrc/commandboard-web run build && npm --workspace @logicsrc/web run build",
|
||||
"build": "npm --workspace @logicsrc/schemas run build && npm --workspace @logicsrc/validators run build && npm --workspace @logicsrc/sdk run build && npm --workspace @logicsrc/plugin-core run build && npm --workspace @logicsrc/plugin-coinpay run build && npm --workspace @logicsrc/plugin-ugig run build && npm --workspace @logicsrc/plugin-sh1pt run build && npm --workspace @logicsrc/plugin-c0mpute run build && npm --workspace @logicsrc/plugin-feed-discovery run build && npm --workspace @logicsrc/tui run build && npm --workspace @logicsrc/cli run build && npm --workspace @profullstack/logicsrc-mcp run build && npm --workspace @logicsrc/commandboard-api run build && npm --workspace @logicsrc/commandboard-web run build && npm --workspace @logicsrc/web run build",
|
||||
"start": "npm --workspace @logicsrc/web run start",
|
||||
"test": "npm run test --workspaces --if-present",
|
||||
"check": "npm run build && npm run test",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"dependencies": {
|
||||
"@logicsrc/plugin-core": "file:../plugin-core",
|
||||
"@logicsrc/plugin-coinpay": "file:../../plugins/coinpay",
|
||||
"@logicsrc/plugin-feed-discovery": "file:../../plugins/feed-discovery",
|
||||
"@logicsrc/plugin-ugig": "file:../../plugins/ugig",
|
||||
"@logicsrc/tui": "file:../tui",
|
||||
"@logicsrc/validators": "file:../validators",
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ describe("CLI registry", () => {
|
|||
|
||||
expect(ids).toContain("coinpay");
|
||||
expect(ids).toContain("ugig");
|
||||
expect(ids).toContain("feed-discovery");
|
||||
expect(ids).not.toContain("sh1pt");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/env node
|
||||
import { readFileSync } from "node:fs";
|
||||
import { Command } from "commander";
|
||||
import { discoverFeeds, listFeedProviders, probeSite, renderDiscoveryOutput, validateFeed, type FeedKind, type FeedOutputFormat } from "@logicsrc/plugin-feed-discovery";
|
||||
import { renderArcadeList, renderPluginStatus, renderTui, runArcadeSession, type TaskSnapshot } from "@logicsrc/tui";
|
||||
import { assertSchemaKind, parseDocument, validate } from "@logicsrc/validators";
|
||||
import { getConfigValue, readConfig, setConfigValue, writeConfig } from "./config.js";
|
||||
|
|
@ -295,6 +296,76 @@ openspec
|
|||
outDir: options.out
|
||||
}), "json"));
|
||||
|
||||
const feeds = program.command("feeds").description("Discover, validate, probe, and export feed sources.");
|
||||
|
||||
feeds
|
||||
.command("discover")
|
||||
.argument("<keyword>", "Keyword, phrase, or homepage URL")
|
||||
.option("--type <type>", "Feed kind or all", "all")
|
||||
.option("--format <format>", "json, opml, rss, atom, or json-feed", "json")
|
||||
.option("--limit <limit>", "Maximum results", "25")
|
||||
.option("--freshness-days <days>", "Freshness window for callers that need it")
|
||||
.option("--include-dead-feeds", "Include feeds that fail validation")
|
||||
.option("--include-unvalidated", "Return provider candidates without validation")
|
||||
.option("--providers <providers>", "Comma-separated provider ids")
|
||||
.description("Discover canonical feed URLs by keyword.")
|
||||
.action(async (keyword, options) => {
|
||||
const response = await discoverFeeds({
|
||||
q: keyword,
|
||||
type: options.type as FeedKind | "all",
|
||||
limit: Number(options.limit),
|
||||
freshnessDays: options.freshnessDays ? Number(options.freshnessDays) : undefined,
|
||||
includeDeadFeeds: Boolean(options.includeDeadFeeds),
|
||||
includeUnvalidated: Boolean(options.includeUnvalidated),
|
||||
providers: splitOption(options.providers)
|
||||
});
|
||||
console.log(renderDiscoveryOutput(response, options.format as FeedOutputFormat));
|
||||
});
|
||||
|
||||
feeds
|
||||
.command("validate")
|
||||
.argument("<feed-url>", "Feed URL")
|
||||
.option("--format <format>", "json, table, or markdown", "json")
|
||||
.description("Validate a feed URL with SSRF protections.")
|
||||
.action(async (feedUrl, options) => {
|
||||
const result = await validateFeed(feedUrl);
|
||||
print(result, options.format as OutputFormat);
|
||||
if (!result.ok) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
|
||||
feeds
|
||||
.command("probe")
|
||||
.argument("<homepage-url>", "Homepage URL")
|
||||
.option("--format <format>", "json, table, or markdown", "json")
|
||||
.description("Probe a homepage for alternate feed links and common feed paths.")
|
||||
.action(async (homepageUrl, options) => {
|
||||
const result = await probeSite(homepageUrl);
|
||||
print(result, options.format as OutputFormat);
|
||||
if (result.feeds.length === 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
|
||||
feeds
|
||||
.command("providers")
|
||||
.option("--format <format>", "table, json, or markdown", "table")
|
||||
.description("List feed discovery providers.")
|
||||
.action((options) => {
|
||||
print(listFeedProviders(), options.format as OutputFormat);
|
||||
});
|
||||
|
||||
feeds
|
||||
.command("export-opml")
|
||||
.argument("<keyword>", "Keyword or phrase")
|
||||
.option("--limit <limit>", "Maximum results", "100")
|
||||
.description("Discover feeds and print OPML.")
|
||||
.action(async (keyword, options) => {
|
||||
const response = await discoverFeeds({ q: keyword, limit: Number(options.limit) });
|
||||
console.log(renderDiscoveryOutput(response, "opml"));
|
||||
});
|
||||
|
||||
program.command("plugins").option("--format <format>", "table, json, or markdown", "table").description("Show plugin status.").action((options) => {
|
||||
const snapshot = defaultPluginRegistry().snapshot();
|
||||
print(snapshot.plugins, options.format as OutputFormat);
|
||||
|
|
@ -395,6 +466,13 @@ function resolveArcadeGame(options: { arcade?: string | boolean; waitingArcade?:
|
|||
return undefined;
|
||||
}
|
||||
|
||||
function splitOption(value: string | undefined) {
|
||||
return value
|
||||
?.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function runYoloArcade(game: string, repo?: string) {
|
||||
const task: TaskSnapshot = {
|
||||
id: "agentswarm_yolo",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { createPluginRegistry } from "@logicsrc/plugin-core";
|
||||
import { coinPayPlugin } from "@logicsrc/plugin-coinpay";
|
||||
import { feedDiscoveryPlugin } from "@logicsrc/plugin-feed-discovery";
|
||||
import { uGigPlugin } from "@logicsrc/plugin-ugig";
|
||||
|
||||
export function defaultPluginRegistry() {
|
||||
return createPluginRegistry([coinPayPlugin, uGigPlugin]);
|
||||
return createPluginRegistry([coinPayPlugin, uGigPlugin, feedDiscoveryPlugin]);
|
||||
}
|
||||
|
|
|
|||
101
plugins/feed-discovery/README.md
Normal file
101
plugins/feed-discovery/README.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# LogicSRC Feed Discovery Plugin
|
||||
|
||||
Discover RSS, Atom, JSON Feed, podcast, and feed-like sources by keyword using a provider-based LogicSRC plugin.
|
||||
|
||||
## Install
|
||||
|
||||
This repository includes the plugin as a normal LogicSRC workspace plugin:
|
||||
|
||||
```bash
|
||||
npm --workspace @logicsrc/plugin-feed-discovery run build
|
||||
```
|
||||
|
||||
Consumers can import the runtime API:
|
||||
|
||||
```ts
|
||||
import { discoverFeeds, renderRss } from "@logicsrc/plugin-feed-discovery";
|
||||
|
||||
const result = await discoverFeeds({ q: "microsaas", limit: 25 });
|
||||
console.log(renderRss(result));
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
```bash
|
||||
logicsrc feeds discover "microsaas" --format json
|
||||
logicsrc feeds discover "microsaas" --format opml
|
||||
logicsrc feeds discover "microsaas" --format rss
|
||||
logicsrc feeds discover "ai agents" --type podcast
|
||||
logicsrc feeds validate https://example.com/feed.xml
|
||||
logicsrc feeds probe https://example.com
|
||||
logicsrc feeds providers
|
||||
```
|
||||
|
||||
Validation is enabled by default. Use `--include-unvalidated` to inspect raw provider candidates without network validation.
|
||||
|
||||
## Providers
|
||||
|
||||
MVP providers:
|
||||
|
||||
- `manual-curated`: local high-trust starter feeds.
|
||||
- `opml-directory`: local OPML files configured with `LOGICSRC_FEEDS_OPML_PATHS`.
|
||||
- `web-feed-probe`: probes direct URL queries and configured candidate homepages from `LOGICSRC_FEEDS_CANDIDATE_URLS`.
|
||||
- `itunes-podcast`: public iTunes podcast search.
|
||||
- `podcastindex`: optional PodcastIndex search when `PODCASTINDEX_API_KEY` and `PODCASTINDEX_API_SECRET` are set.
|
||||
|
||||
Provider failures are isolated and returned as `providerErrors`; one failed provider does not fail the whole discovery request.
|
||||
|
||||
## HTTP Reference API
|
||||
|
||||
The CommandBoard reference API exposes:
|
||||
|
||||
```http
|
||||
GET /api/feeds/discover?q=microsaas&type=all&limit=50
|
||||
GET /api/feeds/providers
|
||||
GET /rss/discover/microsaas.xml
|
||||
GET /api/rss/discover?q=microsaas
|
||||
```
|
||||
|
||||
BitTorrented can consume the plugin package directly and map its public routes to the same runtime calls:
|
||||
|
||||
- `/api/rss/discover?q=:keyword` -> `discoverFeeds()`
|
||||
- `/rss/discover/:keyword.xml` -> `discoverFeeds()` plus `renderRss()`
|
||||
- `/rss/discover/:keyword.opml` -> `discoverFeeds()` plus `renderOpml()`
|
||||
|
||||
## Configuration
|
||||
|
||||
```bash
|
||||
LOGICSRC_FEEDS_CACHE_TTL_SECONDS=86400
|
||||
LOGICSRC_FEEDS_MAX_PROVIDERS=10
|
||||
LOGICSRC_FEEDS_MAX_PROBES=50
|
||||
LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS=8000
|
||||
LOGICSRC_FEEDS_MAX_BODY_BYTES=1000000
|
||||
LOGICSRC_FEEDS_USER_AGENT="LogicSrcFeedDiscovery/0.1"
|
||||
LOGICSRC_FEEDS_OPML_PATHS="./data/feeds.opml,./data/podcasts.opml"
|
||||
LOGICSRC_FEEDS_CANDIDATE_URLS="https://example.com|microsaas,https://another.example|ai agents"
|
||||
|
||||
PODCASTINDEX_API_KEY=
|
||||
PODCASTINDEX_API_SECRET=
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
`validateFeed()` and `probeSite()` use guarded fetches:
|
||||
|
||||
- Only `http` and `https` URLs are allowed.
|
||||
- `localhost`, loopback, private, link-local, carrier-grade NAT, and common metadata targets are blocked.
|
||||
- DNS-resolved private/internal addresses are blocked.
|
||||
- Redirects are limited and checked through the same guard.
|
||||
- Request timeouts and response body size limits are enforced.
|
||||
|
||||
## Database
|
||||
|
||||
Supabase-compatible schema SQL is included at `src/db/schema.sql`. The current LogicSRC repo has no shared Supabase migration convention, so the plugin ships the schema for host applications to apply.
|
||||
|
||||
## Deferred v0.2 Work
|
||||
|
||||
- YouTube, Reddit, GitHub, and RSSHub adapters.
|
||||
- Persistent cache and refresh jobs.
|
||||
- Atom and JSON Feed public endpoints in host apps.
|
||||
- Provider health dashboard and admin tooling.
|
||||
- Paid provider adapters such as RSS.app, Feedly, Inoreader, Twingly, and Listen Notes.
|
||||
24
plugins/feed-discovery/package.json
Normal file
24
plugins/feed-discovery/package.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "@logicsrc/plugin-feed-discovery",
|
||||
"version": "0.1.0",
|
||||
"description": "LogicSRC feed discovery plugin for RSS, Atom, JSON Feed, podcasts, OPML directories, and probed sites.",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"test": "vitest run src --passWithNoTests"
|
||||
},
|
||||
"dependencies": {
|
||||
"@logicsrc/plugin-core": "file:../../packages/plugin-core",
|
||||
"cheerio": "^1.0.0",
|
||||
"fast-xml-parser": "^5.2.5",
|
||||
"rss": "^1.2.2",
|
||||
"slugify": "^1.6.6",
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/rss": "^0.0.32",
|
||||
"vitest": "^4.0.8"
|
||||
}
|
||||
}
|
||||
34
plugins/feed-discovery/src/config.ts
Normal file
34
plugins/feed-discovery/src/config.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import type { FeedDiscoveryConfig } from "./types.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 8_000;
|
||||
const DEFAULT_MAX_BODY_BYTES = 1_000_000;
|
||||
|
||||
export function readFeedDiscoveryConfig(env: NodeJS.ProcessEnv = process.env): FeedDiscoveryConfig {
|
||||
return {
|
||||
cacheTtlSeconds: readNumber(env.LOGICSRC_FEEDS_CACHE_TTL_SECONDS, 86_400),
|
||||
maxProviders: readNumber(env.LOGICSRC_FEEDS_MAX_PROVIDERS, 10),
|
||||
maxProbes: readNumber(env.LOGICSRC_FEEDS_MAX_PROBES, 50),
|
||||
requestTimeoutMs: readNumber(env.LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS, DEFAULT_TIMEOUT_MS),
|
||||
maxBodyBytes: readNumber(env.LOGICSRC_FEEDS_MAX_BODY_BYTES, DEFAULT_MAX_BODY_BYTES),
|
||||
userAgent: env.LOGICSRC_FEEDS_USER_AGENT || "LogicSrcFeedDiscovery/0.1",
|
||||
opmlPaths: splitList(env.LOGICSRC_FEEDS_OPML_PATHS),
|
||||
candidateUrls: splitList(env.LOGICSRC_FEEDS_CANDIDATE_URLS),
|
||||
podcastIndexApiKey: env.PODCASTINDEX_API_KEY,
|
||||
podcastIndexApiSecret: env.PODCASTINDEX_API_SECRET
|
||||
};
|
||||
}
|
||||
|
||||
function readNumber(value: string | undefined, fallback: number) {
|
||||
if (!value) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function splitList(value: string | undefined) {
|
||||
return (value ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
62
plugins/feed-discovery/src/db/schema.sql
Normal file
62
plugins/feed-discovery/src/db/schema.sql
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
create table if not exists feed_sources (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
title text not null,
|
||||
description text,
|
||||
homepage_url text,
|
||||
feed_url text not null unique,
|
||||
canonical_feed_url text,
|
||||
kind text not null default 'unknown',
|
||||
provider text not null,
|
||||
language text,
|
||||
image_url text,
|
||||
score numeric not null default 0,
|
||||
confidence numeric not null default 0,
|
||||
freshness_score numeric not null default 0,
|
||||
keyword_score numeric not null default 0,
|
||||
provider_score numeric not null default 0,
|
||||
last_published_at timestamptz,
|
||||
last_checked_at timestamptz,
|
||||
is_valid boolean default true,
|
||||
created_at timestamptz default now(),
|
||||
updated_at timestamptz default now()
|
||||
);
|
||||
|
||||
create table if not exists feed_discovery_queries (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
query text not null,
|
||||
normalized_query text not null,
|
||||
type text default 'all',
|
||||
result_count integer not null default 0,
|
||||
created_at timestamptz default now()
|
||||
);
|
||||
|
||||
create table if not exists feed_discovery_results (
|
||||
query_id uuid references feed_discovery_queries(id) on delete cascade,
|
||||
source_id uuid references feed_sources(id) on delete cascade,
|
||||
rank integer not null,
|
||||
score numeric not null,
|
||||
created_at timestamptz default now(),
|
||||
primary key (query_id, source_id)
|
||||
);
|
||||
|
||||
create table if not exists feed_items (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
source_id uuid references feed_sources(id) on delete cascade,
|
||||
title text not null,
|
||||
url text not null unique,
|
||||
description text,
|
||||
published_at timestamptz,
|
||||
guid text,
|
||||
created_at timestamptz default now()
|
||||
);
|
||||
|
||||
create table if not exists feed_provider_logs (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
provider text not null,
|
||||
query text not null,
|
||||
status text not null,
|
||||
result_count integer default 0,
|
||||
error_message text,
|
||||
duration_ms integer,
|
||||
created_at timestamptz default now()
|
||||
);
|
||||
38
plugins/feed-discovery/src/dedupe.ts
Normal file
38
plugins/feed-discovery/src/dedupe.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import type { DiscoveredFeed } from "./types.js";
|
||||
import { canonicalizeUrl } from "./url-safety.js";
|
||||
|
||||
export function dedupeFeeds(feeds: DiscoveredFeed[]) {
|
||||
const byKey = new Map<string, DiscoveredFeed>();
|
||||
|
||||
for (const feed of feeds) {
|
||||
const key = dedupeKey(feed);
|
||||
const existing = byKey.get(key);
|
||||
if (!existing || feed.score > existing.score || feed.confidence > existing.confidence) {
|
||||
byKey.set(key, mergeFeed(existing, feed));
|
||||
}
|
||||
}
|
||||
|
||||
return [...byKey.values()].sort((a, b) => b.score - a.score || b.confidence - a.confidence);
|
||||
}
|
||||
|
||||
function dedupeKey(feed: DiscoveredFeed) {
|
||||
const feedUrl = feed.canonicalFeedUrl || feed.feedUrl;
|
||||
try {
|
||||
return `feed:${canonicalizeUrl(feedUrl)}`;
|
||||
} catch {
|
||||
return `feed:${feedUrl.toLowerCase()}`;
|
||||
}
|
||||
}
|
||||
|
||||
function mergeFeed(existing: DiscoveredFeed | undefined, next: DiscoveredFeed): DiscoveredFeed {
|
||||
if (!existing) {
|
||||
return next;
|
||||
}
|
||||
|
||||
return {
|
||||
...existing,
|
||||
...next,
|
||||
tags: [...new Set([...existing.tags, ...next.tags])],
|
||||
sampleItems: next.sampleItems?.length ? next.sampleItems : existing.sampleItems
|
||||
};
|
||||
}
|
||||
120
plugins/feed-discovery/src/discovery.ts
Normal file
120
plugins/feed-discovery/src/discovery.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import slugify from "slugify";
|
||||
import { readFeedDiscoveryConfig } from "./config.js";
|
||||
import { dedupeFeeds } from "./dedupe.js";
|
||||
import { createDefaultFeedProviders } from "./providers/index.js";
|
||||
import { scoreFeed } from "./scoring.js";
|
||||
import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery, FeedDiscoveryResponse } from "./types.js";
|
||||
import { canonicalizeUrl } from "./url-safety.js";
|
||||
import { validateFeed } from "./validate-feed.js";
|
||||
|
||||
export async function discoverFeeds(query: FeedDiscoveryQuery, options: { providers?: FeedDiscoveryProvider[]; config?: Partial<FeedDiscoveryConfig> } = {}): Promise<FeedDiscoveryResponse> {
|
||||
const config = { ...readFeedDiscoveryConfig(), ...options.config };
|
||||
const normalizedQuery = normalizeQuery(query.q);
|
||||
const resolvedQuery: FeedDiscoveryQuery = {
|
||||
...query,
|
||||
q: query.q.trim(),
|
||||
type: query.type ?? "all",
|
||||
limit: clampLimit(query.limit),
|
||||
includeUnvalidated: query.includeUnvalidated ?? false,
|
||||
includeDeadFeeds: query.includeDeadFeeds ?? false
|
||||
};
|
||||
|
||||
const providers = (options.providers ?? createDefaultFeedProviders(config))
|
||||
.filter((provider) => provider.enabledByDefault || provider.id === "podcastindex")
|
||||
.filter((provider) => !resolvedQuery.providers?.length || resolvedQuery.providers.includes(provider.id))
|
||||
.slice(0, config.maxProviders);
|
||||
|
||||
const settled = await Promise.all(providers.map(async (provider) => {
|
||||
try {
|
||||
return {
|
||||
provider: provider.id,
|
||||
results: await provider.search(resolvedQuery),
|
||||
error: undefined
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
provider: provider.id,
|
||||
results: [],
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}));
|
||||
|
||||
const providerErrors: FeedDiscoveryResponse["providerErrors"] = [];
|
||||
const candidates: DiscoveredFeed[] = [];
|
||||
|
||||
for (const result of settled) {
|
||||
if (result.error) {
|
||||
providerErrors.push({ provider: result.provider, error: result.error });
|
||||
continue;
|
||||
}
|
||||
candidates.push(...result.results.map((feed) => ({ ...feed, provider: feed.provider || result.provider })));
|
||||
}
|
||||
|
||||
const validated = await validateCandidates(candidates.slice(0, Math.max((resolvedQuery.limit ?? 25) * 2, 25)), resolvedQuery, config);
|
||||
const scored = validated
|
||||
.filter((feed) => !resolvedQuery.type || resolvedQuery.type === "all" || feed.kind === resolvedQuery.type)
|
||||
.map((feed) => scoreFeed(feed, resolvedQuery));
|
||||
const results = dedupeFeeds(scored).slice(0, resolvedQuery.limit ?? 25);
|
||||
|
||||
return {
|
||||
query: query.q,
|
||||
normalizedQuery,
|
||||
count: results.length,
|
||||
providerErrors,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
async function validateCandidates(candidates: DiscoveredFeed[], query: FeedDiscoveryQuery, config: FeedDiscoveryConfig) {
|
||||
if (query.includeUnvalidated) {
|
||||
return candidates.map((feed) => ({
|
||||
...feed,
|
||||
canonicalFeedUrl: safeCanonical(feed.feedUrl),
|
||||
isValid: feed.isValid ?? false,
|
||||
validationScore: feed.validationScore ?? 0.4
|
||||
}));
|
||||
}
|
||||
|
||||
const feeds: DiscoveredFeed[] = [];
|
||||
for (const candidate of candidates) {
|
||||
const validation = await validateFeed(candidate.feedUrl, config);
|
||||
if (!validation.ok && !query.includeDeadFeeds) {
|
||||
continue;
|
||||
}
|
||||
feeds.push({
|
||||
...candidate,
|
||||
title: validation.title ?? candidate.title,
|
||||
description: validation.description ?? candidate.description,
|
||||
homepageUrl: validation.homepageUrl ?? candidate.homepageUrl,
|
||||
canonicalFeedUrl: validation.canonicalFeedUrl ?? safeCanonical(candidate.feedUrl),
|
||||
kind: validation.ok && validation.kind !== "unknown" ? validation.kind : candidate.kind,
|
||||
language: validation.language ?? candidate.language,
|
||||
imageUrl: validation.imageUrl ?? candidate.imageUrl,
|
||||
lastPublishedAt: validation.lastPublishedAt ?? candidate.lastPublishedAt,
|
||||
sampleItems: validation.sampleItems.length ? validation.sampleItems : candidate.sampleItems,
|
||||
isValid: validation.ok,
|
||||
validationScore: validation.ok ? 1 : 0.2
|
||||
});
|
||||
}
|
||||
return feeds;
|
||||
}
|
||||
|
||||
function normalizeQuery(query: string) {
|
||||
return slugify(query.trim().toLowerCase(), { lower: true, strict: true }) || "feeds";
|
||||
}
|
||||
|
||||
function clampLimit(limit: number | undefined) {
|
||||
if (!limit) {
|
||||
return 25;
|
||||
}
|
||||
return Math.min(Math.max(Math.trunc(limit), 1), 100);
|
||||
}
|
||||
|
||||
function safeCanonical(url: string) {
|
||||
try {
|
||||
return canonicalizeUrl(url);
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
102
plugins/feed-discovery/src/feed-discovery.test.ts
Normal file
102
plugins/feed-discovery/src/feed-discovery.test.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { dedupeFeeds } from "./dedupe.js";
|
||||
import { discoverFeeds } from "./discovery.js";
|
||||
import { parseFeedDocument } from "./feed-parsing.js";
|
||||
import { renderDiscoveryOutput } from "./output/index.js";
|
||||
import { extractAlternateFeedLinks } from "./probe-site.js";
|
||||
import { scoreFeed } from "./scoring.js";
|
||||
import type { DiscoveredFeed, FeedDiscoveryProvider } from "./types.js";
|
||||
import { assertSafeHttpUrl } from "./url-safety.js";
|
||||
|
||||
const baseFeed: DiscoveredFeed = {
|
||||
title: "MicroSaaS Ideas",
|
||||
description: "Ideas for indie SaaS founders",
|
||||
homepageUrl: "https://example.com",
|
||||
feedUrl: "https://example.com/feed.xml",
|
||||
kind: "blog",
|
||||
provider: "manual-curated",
|
||||
score: 0,
|
||||
confidence: 0.5,
|
||||
tags: ["microsaas", "saas"]
|
||||
};
|
||||
|
||||
describe("feed parsing", () => {
|
||||
it("parses RSS feeds with sample items", () => {
|
||||
const result = parseFeedDocument("https://example.com/rss.xml", `<?xml version="1.0"?><rss version="2.0"><channel><title>MicroSaaS Ideas</title><link>https://example.com</link><item><title>Launch tiny products</title><link>https://example.com/post</link><pubDate>Tue, 09 Jun 2026 00:00:00 GMT</pubDate></item></channel></rss>`);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.title).toBe("MicroSaaS Ideas");
|
||||
expect(result.sampleItems[0]).toMatchObject({ title: "Launch tiny products", url: "https://example.com/post" });
|
||||
});
|
||||
|
||||
it("parses Atom and JSON Feed documents", () => {
|
||||
const atom = parseFeedDocument("https://example.com/atom.xml", `<feed><title>Atom Feed</title><entry><title>Entry</title><updated>2026-06-09T00:00:00Z</updated></entry></feed>`);
|
||||
const json = parseFeedDocument("https://example.com/feed.json", JSON.stringify({ version: "https://jsonfeed.org/version/1.1", title: "JSON Feed", items: [{ title: "Item", url: "https://example.com/item" }] }), "application/feed+json");
|
||||
|
||||
expect(atom.ok).toBe(true);
|
||||
expect(json.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("site probing helpers", () => {
|
||||
it("extracts alternate feed links", () => {
|
||||
const links = extractAlternateFeedLinks(
|
||||
`<html><head><link rel="alternate" type="application/rss+xml" href="/rss.xml"><link rel="alternate" type="text/html" href="/plain"></head></html>`,
|
||||
"https://example.com/blog"
|
||||
);
|
||||
|
||||
expect(links).toEqual(["https://example.com/rss.xml"]);
|
||||
});
|
||||
|
||||
it("blocks private SSRF targets", async () => {
|
||||
await expect(assertSafeHttpUrl("http://127.0.0.1/feed")).rejects.toThrow(/Blocked internal/);
|
||||
await expect(assertSafeHttpUrl("http://[::ffff:192.168.1.10]/feed")).rejects.toThrow(/Blocked internal/);
|
||||
await expect(assertSafeHttpUrl("file:///etc/passwd")).rejects.toThrow(/Unsupported URL protocol/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scoring, dedupe, and output", () => {
|
||||
it("scores relevant fresh feeds and dedupes canonical URLs", () => {
|
||||
const scored = scoreFeed({ ...baseFeed, lastPublishedAt: new Date().toISOString() }, { q: "microsaas" });
|
||||
const duplicate = { ...scored, feedUrl: "https://example.com/feed.xml?utm_source=test", score: scored.score - 0.1 };
|
||||
|
||||
expect(scored.score).toBeGreaterThan(0.5);
|
||||
expect(dedupeFeeds([duplicate, scored])).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("renders JSON, OPML, and RSS outputs", () => {
|
||||
const response = { query: "microsaas", normalizedQuery: "microsaas", count: 1, providerErrors: [], results: [scoreFeed(baseFeed, { q: "microsaas" })] };
|
||||
|
||||
expect(JSON.parse(renderDiscoveryOutput(response, "json"))).toMatchObject({ count: 1 });
|
||||
expect(renderDiscoveryOutput(response, "opml")).toContain("<opml");
|
||||
expect(renderDiscoveryOutput(response, "rss")).toContain("<rss");
|
||||
});
|
||||
});
|
||||
|
||||
describe("discovery orchestration", () => {
|
||||
it("isolates provider failures", async () => {
|
||||
const okProvider: FeedDiscoveryProvider = {
|
||||
id: "ok-provider",
|
||||
name: "OK",
|
||||
enabledByDefault: true,
|
||||
requiresApiKey: false,
|
||||
async search() {
|
||||
return [baseFeed];
|
||||
}
|
||||
};
|
||||
const failingProvider: FeedDiscoveryProvider = {
|
||||
id: "broken-provider",
|
||||
name: "Broken",
|
||||
enabledByDefault: true,
|
||||
requiresApiKey: false,
|
||||
async search() {
|
||||
throw new Error("provider unavailable");
|
||||
}
|
||||
};
|
||||
|
||||
const response = await discoverFeeds({ q: "microsaas", includeUnvalidated: true }, { providers: [failingProvider, okProvider] });
|
||||
|
||||
expect(response.providerErrors).toEqual([{ provider: "broken-provider", error: "provider unavailable" }]);
|
||||
expect(response.results).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
204
plugins/feed-discovery/src/feed-parsing.ts
Normal file
204
plugins/feed-discovery/src/feed-parsing.ts
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
import { XMLParser } from "fast-xml-parser";
|
||||
import type { FeedKind, FeedSampleItem, ValidationResult } from "./types.js";
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: "",
|
||||
textNodeName: "text",
|
||||
cdataPropName: "text",
|
||||
removeNSPrefix: true
|
||||
});
|
||||
|
||||
export function parseFeedDocument(feedUrl: string, body: string, contentType = ""): ValidationResult {
|
||||
const trimmed = body.trim();
|
||||
if (!trimmed) {
|
||||
return invalid(feedUrl, "Empty feed body");
|
||||
}
|
||||
|
||||
if (contentType.includes("json") || trimmed.startsWith("{")) {
|
||||
return parseJsonFeed(feedUrl, trimmed);
|
||||
}
|
||||
|
||||
try {
|
||||
const document = parser.parse(trimmed) as unknown;
|
||||
if (!isRecord(document)) {
|
||||
return invalid(feedUrl, "Feed did not parse as an object");
|
||||
}
|
||||
|
||||
if (isRecord(document.rss)) {
|
||||
return parseRss(feedUrl, document.rss);
|
||||
}
|
||||
if (isRecord(document.feed)) {
|
||||
return parseAtom(feedUrl, document.feed);
|
||||
}
|
||||
return invalid(feedUrl, "Document is not RSS, Atom, or JSON Feed");
|
||||
} catch (error) {
|
||||
return invalid(feedUrl, error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
|
||||
function parseJsonFeed(feedUrl: string, body: string): ValidationResult {
|
||||
try {
|
||||
const document = JSON.parse(body) as unknown;
|
||||
if (!isRecord(document) || typeof document.version !== "string" || !document.version.includes("jsonfeed")) {
|
||||
return invalid(feedUrl, "JSON document is not JSON Feed");
|
||||
}
|
||||
|
||||
const items = asArray(document.items).filter(isRecord);
|
||||
const sampleItems = items.slice(0, 5).map((item) => ({
|
||||
title: stringValue(item.title) || stringValue(item.url) || "Untitled item",
|
||||
url: stringValue(item.url),
|
||||
publishedAt: stringValue(item.date_published),
|
||||
description: stringValue(item.summary) || stringValue(item.content_text)
|
||||
}));
|
||||
|
||||
const title = stringValue(document.title);
|
||||
if (!title && sampleItems.length === 0) {
|
||||
return invalid(feedUrl, "Feed has no title or items");
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
feedUrl,
|
||||
canonicalFeedUrl: feedUrl,
|
||||
title: title || "Untitled JSON Feed",
|
||||
description: stringValue(document.description),
|
||||
homepageUrl: stringValue(document.home_page_url),
|
||||
kind: "blog",
|
||||
imageUrl: stringValue(document.icon) || stringValue(document.favicon),
|
||||
lastPublishedAt: newestDate(sampleItems.map((item) => item.publishedAt)),
|
||||
sampleItems
|
||||
};
|
||||
} catch (error) {
|
||||
return invalid(feedUrl, error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
|
||||
function parseRss(feedUrl: string, rss: Record<string, unknown>): ValidationResult {
|
||||
const channel = isRecord(rss.channel) ? rss.channel : rss;
|
||||
const items = asArray(channel.item).filter(isRecord);
|
||||
const sampleItems = items.slice(0, 5).map((item) => ({
|
||||
title: stringValue(item.title) || stringValue(item.guid) || "Untitled item",
|
||||
url: linkValue(item.link) || stringValue(item.guid),
|
||||
publishedAt: stringValue(item.pubDate) || stringValue(item.published) || stringValue(item.updated),
|
||||
description: stringValue(item.description)
|
||||
}));
|
||||
|
||||
const title = stringValue(channel.title);
|
||||
if (!title && sampleItems.length === 0) {
|
||||
return invalid(feedUrl, "RSS feed has no title or items");
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
feedUrl,
|
||||
canonicalFeedUrl: feedUrl,
|
||||
title: title || "Untitled RSS Feed",
|
||||
description: stringValue(channel.description),
|
||||
homepageUrl: linkValue(channel.link),
|
||||
kind: detectRssKind(channel),
|
||||
language: stringValue(channel.language),
|
||||
imageUrl: imageValue(channel.image),
|
||||
lastPublishedAt: newestDate([stringValue(channel.lastBuildDate), stringValue(channel.pubDate), ...sampleItems.map((item) => item.publishedAt)]),
|
||||
sampleItems
|
||||
};
|
||||
}
|
||||
|
||||
function parseAtom(feedUrl: string, feed: Record<string, unknown>): ValidationResult {
|
||||
const entries = asArray(feed.entry).filter(isRecord);
|
||||
const sampleItems = entries.slice(0, 5).map((entry) => ({
|
||||
title: stringValue(entry.title) || "Untitled item",
|
||||
url: atomLink(entry.link),
|
||||
publishedAt: stringValue(entry.published) || stringValue(entry.updated),
|
||||
description: stringValue(entry.summary) || stringValue(entry.content)
|
||||
}));
|
||||
|
||||
const title = stringValue(feed.title);
|
||||
if (!title && sampleItems.length === 0) {
|
||||
return invalid(feedUrl, "Atom feed has no title or entries");
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
feedUrl,
|
||||
canonicalFeedUrl: feedUrl,
|
||||
title: title || "Untitled Atom Feed",
|
||||
description: stringValue(feed.subtitle),
|
||||
homepageUrl: atomLink(feed.link),
|
||||
kind: "blog",
|
||||
language: stringValue(feed.lang),
|
||||
lastPublishedAt: newestDate([stringValue(feed.updated), ...sampleItems.map((item) => item.publishedAt)]),
|
||||
sampleItems
|
||||
};
|
||||
}
|
||||
|
||||
function detectRssKind(channel: Record<string, unknown>): FeedKind {
|
||||
if (channel.itunes || channel["itunes:author"] || channel.enclosure) {
|
||||
return "podcast";
|
||||
}
|
||||
return "blog";
|
||||
}
|
||||
|
||||
function imageValue(value: unknown) {
|
||||
if (isRecord(value)) {
|
||||
return stringValue(value.url);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function atomLink(value: unknown) {
|
||||
const links = asArray(value).filter(isRecord);
|
||||
const alternate = links.find((link) => stringValue(link.rel) === "alternate") ?? links[0];
|
||||
return alternate ? stringValue(alternate.href) : stringValue(value);
|
||||
}
|
||||
|
||||
function linkValue(value: unknown) {
|
||||
if (isRecord(value)) {
|
||||
return stringValue(value.href) || stringValue(value.text);
|
||||
}
|
||||
return stringValue(value);
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
if (typeof value === "string") {
|
||||
return value.trim() || undefined;
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
return String(value);
|
||||
}
|
||||
if (isRecord(value) && typeof value.text === "string") {
|
||||
return value.text.trim() || undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function newestDate(values: Array<string | undefined>) {
|
||||
const timestamps = values
|
||||
.map((value) => (value ? Date.parse(value) : NaN))
|
||||
.filter((value) => Number.isFinite(value));
|
||||
if (timestamps.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return new Date(Math.max(...timestamps)).toISOString();
|
||||
}
|
||||
|
||||
function invalid(feedUrl: string, error: string): ValidationResult {
|
||||
return {
|
||||
ok: false,
|
||||
feedUrl,
|
||||
kind: "unknown",
|
||||
sampleItems: [],
|
||||
error
|
||||
};
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] {
|
||||
if (Array.isArray(value)) {
|
||||
return value;
|
||||
}
|
||||
return value === undefined || value === null ? [] : [value];
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
80
plugins/feed-discovery/src/http.ts
Normal file
80
plugins/feed-discovery/src/http.ts
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import type { FeedDiscoveryConfig } from "./types.js";
|
||||
import { assertSafeHttpUrl } from "./url-safety.js";
|
||||
|
||||
export interface BoundedFetchResult {
|
||||
url: string;
|
||||
contentType: string;
|
||||
body: string;
|
||||
status: number;
|
||||
}
|
||||
|
||||
export async function fetchTextWithGuards(input: string, config: Pick<FeedDiscoveryConfig, "requestTimeoutMs" | "maxBodyBytes" | "userAgent">, redirects = 3): Promise<BoundedFetchResult> {
|
||||
const url = await assertSafeHttpUrl(input);
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), config.requestTimeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
redirect: "manual",
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"user-agent": config.userAgent,
|
||||
accept: "application/rss+xml, application/atom+xml, application/feed+json, application/json, text/xml, application/xml, text/html;q=0.8, */*;q=0.5"
|
||||
}
|
||||
});
|
||||
|
||||
if (isRedirect(response.status)) {
|
||||
if (redirects <= 0) {
|
||||
throw new Error("Too many redirects");
|
||||
}
|
||||
const location = response.headers.get("location");
|
||||
if (!location) {
|
||||
throw new Error("Redirect without location");
|
||||
}
|
||||
return fetchTextWithGuards(new URL(location, url).toString(), config, redirects - 1);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const body = await readLimitedText(response, config.maxBodyBytes);
|
||||
return {
|
||||
url: response.url || url.toString(),
|
||||
contentType: response.headers.get("content-type") ?? "",
|
||||
body,
|
||||
status: response.status
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function readLimitedText(response: Response, maxBodyBytes: number) {
|
||||
if (!response.body) {
|
||||
return response.text();
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let received = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
received += value.byteLength;
|
||||
if (received > maxBodyBytes) {
|
||||
await reader.cancel();
|
||||
throw new Error(`Response exceeded ${maxBodyBytes} bytes`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
|
||||
return new TextDecoder().decode(Buffer.concat(chunks));
|
||||
}
|
||||
|
||||
function isRedirect(status: number) {
|
||||
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
||||
}
|
||||
47
plugins/feed-discovery/src/index.ts
Normal file
47
plugins/feed-discovery/src/index.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import type { PluginDefinition } from "@logicsrc/plugin-core";
|
||||
import { readFeedDiscoveryConfig } from "./config.js";
|
||||
import { discoverFeeds } from "./discovery.js";
|
||||
import { feedDiscoveryManifest } from "./manifest.js";
|
||||
import { probeSite } from "./probe-site.js";
|
||||
import { providerManifests } from "./providers/index.js";
|
||||
import { validateFeed } from "./validate-feed.js";
|
||||
|
||||
export const feedDiscoveryPlugin: PluginDefinition = {
|
||||
manifest: feedDiscoveryManifest,
|
||||
configDefaults: {
|
||||
enabled: true,
|
||||
cache_ttl_seconds: "${LOGICSRC_FEEDS_CACHE_TTL_SECONDS}",
|
||||
max_providers: "${LOGICSRC_FEEDS_MAX_PROVIDERS}",
|
||||
max_probes: "${LOGICSRC_FEEDS_MAX_PROBES}",
|
||||
request_timeout_ms: "${LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS}",
|
||||
user_agent: "${LOGICSRC_FEEDS_USER_AGENT}"
|
||||
},
|
||||
routes: [
|
||||
{ method: "GET", path: "/api/feeds/discover", capability: "feeds.discover" },
|
||||
{ method: "GET", path: "/api/feeds/providers", capability: "feeds.providers.list" },
|
||||
{ method: "GET", path: "/rss/discover/:keyword.xml", capability: "feeds.export.rss" }
|
||||
],
|
||||
permissions: ["feeds:discover", "feeds:validate", "feeds:probe", "feeds:export"],
|
||||
tuiPanels: [{ id: "feed-discovery-status", title: "Feed Discovery" }]
|
||||
};
|
||||
|
||||
export function listFeedProviders() {
|
||||
return providerManifests(readFeedDiscoveryConfig());
|
||||
}
|
||||
|
||||
export { discoverFeeds, feedDiscoveryManifest, probeSite, readFeedDiscoveryConfig, validateFeed };
|
||||
export type {
|
||||
DiscoveredFeed,
|
||||
FeedDiscoveryConfig,
|
||||
FeedDiscoveryProvider,
|
||||
FeedDiscoveryQuery,
|
||||
FeedDiscoveryResponse,
|
||||
FeedKind,
|
||||
FeedOutputFormat,
|
||||
FeedProviderManifest,
|
||||
FeedSampleItem,
|
||||
ProbeResult,
|
||||
ValidationResult
|
||||
} from "./types.js";
|
||||
export { renderAtom, renderDiscoveryOutput, renderJsonFeed, renderOpml, renderRss } from "./output/index.js";
|
||||
export { createDefaultFeedProviders, providerManifests } from "./providers/index.js";
|
||||
27
plugins/feed-discovery/src/manifest.ts
Normal file
27
plugins/feed-discovery/src/manifest.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import type { PluginManifest } from "@logicsrc/plugin-core";
|
||||
|
||||
export const feedDiscoveryManifest: PluginManifest = {
|
||||
id: "feed-discovery",
|
||||
name: "Feed Discovery",
|
||||
version: "0.1.0",
|
||||
type: ["feeds", "rss", "discovery", "content"],
|
||||
default: true,
|
||||
capabilities: [
|
||||
"feeds.discover",
|
||||
"feeds.validate",
|
||||
"feeds.probe",
|
||||
"feeds.export.opml",
|
||||
"feeds.export.rss",
|
||||
"feeds.providers.list"
|
||||
],
|
||||
commands: ["feeds"],
|
||||
env: [
|
||||
"LOGICSRC_FEEDS_CACHE_TTL_SECONDS",
|
||||
"LOGICSRC_FEEDS_MAX_PROVIDERS",
|
||||
"LOGICSRC_FEEDS_MAX_PROBES",
|
||||
"LOGICSRC_FEEDS_REQUEST_TIMEOUT_MS",
|
||||
"LOGICSRC_FEEDS_USER_AGENT",
|
||||
"PODCASTINDEX_API_KEY",
|
||||
"PODCASTINDEX_API_SECRET"
|
||||
]
|
||||
};
|
||||
26
plugins/feed-discovery/src/output/atom.ts
Normal file
26
plugins/feed-discovery/src/output/atom.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import type { FeedDiscoveryResponse } from "../types.js";
|
||||
import { escapeXml } from "./opml.js";
|
||||
|
||||
export function renderAtom(response: FeedDiscoveryResponse) {
|
||||
const updated = new Date().toISOString();
|
||||
const entries = response.results
|
||||
.map((result) => {
|
||||
const id = result.canonicalFeedUrl ?? result.feedUrl;
|
||||
return ` <entry>
|
||||
<id>${escapeXml(id)}</id>
|
||||
<title>${escapeXml(result.title)}</title>
|
||||
<link href="${escapeXml(result.homepageUrl ?? result.feedUrl)}" />
|
||||
<updated>${escapeXml(result.lastPublishedAt ?? updated)}</updated>
|
||||
<summary>${escapeXml(result.description ?? `Discovered ${result.kind} feed from ${result.provider}`)}</summary>
|
||||
</entry>`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<id>logicsrc:feed-discovery:${escapeXml(response.normalizedQuery)}</id>
|
||||
<title>LogicSRC feed discovery: ${escapeXml(response.query)}</title>
|
||||
<updated>${updated}</updated>
|
||||
${entries}
|
||||
</feed>`;
|
||||
}
|
||||
23
plugins/feed-discovery/src/output/index.ts
Normal file
23
plugins/feed-discovery/src/output/index.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import type { FeedDiscoveryResponse, FeedOutputFormat } from "../types.js";
|
||||
import { renderAtom } from "./atom.js";
|
||||
import { renderJsonFeed } from "./json-feed.js";
|
||||
import { renderOpml } from "./opml.js";
|
||||
import { renderRss } from "./rss.js";
|
||||
|
||||
export function renderDiscoveryOutput(response: FeedDiscoveryResponse, format: FeedOutputFormat) {
|
||||
if (format === "opml") {
|
||||
return renderOpml(response);
|
||||
}
|
||||
if (format === "rss") {
|
||||
return renderRss(response);
|
||||
}
|
||||
if (format === "atom") {
|
||||
return renderAtom(response);
|
||||
}
|
||||
if (format === "json-feed") {
|
||||
return renderJsonFeed(response);
|
||||
}
|
||||
return JSON.stringify(response, null, 2);
|
||||
}
|
||||
|
||||
export { renderAtom, renderJsonFeed, renderOpml, renderRss };
|
||||
22
plugins/feed-discovery/src/output/json-feed.ts
Normal file
22
plugins/feed-discovery/src/output/json-feed.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import type { FeedDiscoveryResponse } from "../types.js";
|
||||
|
||||
export function renderJsonFeed(response: FeedDiscoveryResponse) {
|
||||
return JSON.stringify(
|
||||
{
|
||||
version: "https://jsonfeed.org/version/1.1",
|
||||
title: `LogicSRC feed discovery: ${response.query}`,
|
||||
home_page_url: "https://bittorrented.com/rss",
|
||||
feed_url: `https://bittorrented.com/json-feed/discover/${encodeURIComponent(response.normalizedQuery)}.json`,
|
||||
items: response.results.map((result) => ({
|
||||
id: result.canonicalFeedUrl ?? result.feedUrl,
|
||||
url: result.homepageUrl ?? result.feedUrl,
|
||||
title: result.title,
|
||||
summary: result.description ?? `Discovered ${result.kind} feed from ${result.provider}`,
|
||||
date_published: result.lastPublishedAt,
|
||||
tags: [result.kind, result.provider, ...result.tags]
|
||||
}))
|
||||
},
|
||||
null,
|
||||
2
|
||||
);
|
||||
}
|
||||
27
plugins/feed-discovery/src/output/opml.ts
Normal file
27
plugins/feed-discovery/src/output/opml.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import type { DiscoveredFeed, FeedDiscoveryResponse } from "../types.js";
|
||||
|
||||
export function renderOpml(response: FeedDiscoveryResponse) {
|
||||
const outlines = response.results.map(renderOutline).join("\n");
|
||||
return `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<opml version="2.0">
|
||||
<head>
|
||||
<title>LogicSRC feed discovery: ${escapeXml(response.query)}</title>
|
||||
</head>
|
||||
<body>
|
||||
${outlines}
|
||||
</body>
|
||||
</opml>`;
|
||||
}
|
||||
|
||||
function renderOutline(feed: DiscoveredFeed) {
|
||||
return ` <outline text="${escapeXml(feed.title)}" title="${escapeXml(feed.title)}" type="rss" xmlUrl="${escapeXml(feed.canonicalFeedUrl ?? feed.feedUrl)}"${feed.homepageUrl ? ` htmlUrl="${escapeXml(feed.homepageUrl)}"` : ""} category="${escapeXml(feed.kind)}" />`;
|
||||
}
|
||||
|
||||
export function escapeXml(value: string) {
|
||||
return value
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
25
plugins/feed-discovery/src/output/rss.ts
Normal file
25
plugins/feed-discovery/src/output/rss.ts
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
import RSS from "rss";
|
||||
import type { FeedDiscoveryResponse } from "../types.js";
|
||||
|
||||
export function renderRss(response: FeedDiscoveryResponse) {
|
||||
const feed = new RSS({
|
||||
title: `LogicSRC feed discovery: ${response.query}`,
|
||||
description: `Discovered feed sources for ${response.query}`,
|
||||
feed_url: `https://bittorrented.com/rss/discover/${encodeURIComponent(response.normalizedQuery)}.xml`,
|
||||
site_url: "https://bittorrented.com/rss",
|
||||
language: "en"
|
||||
});
|
||||
|
||||
for (const result of response.results) {
|
||||
feed.item({
|
||||
title: result.title,
|
||||
description: result.description ?? `Discovered ${result.kind} feed from ${result.provider}`,
|
||||
url: result.homepageUrl ?? result.feedUrl,
|
||||
guid: result.canonicalFeedUrl ?? result.feedUrl,
|
||||
categories: [result.kind, result.provider, ...result.tags],
|
||||
date: result.lastPublishedAt ? new Date(result.lastPublishedAt) : new Date()
|
||||
});
|
||||
}
|
||||
|
||||
return feed.xml({ indent: true });
|
||||
}
|
||||
93
plugins/feed-discovery/src/probe-site.ts
Normal file
93
plugins/feed-discovery/src/probe-site.ts
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import * as cheerio from "cheerio";
|
||||
import { readFeedDiscoveryConfig } from "./config.js";
|
||||
import { fetchTextWithGuards } from "./http.js";
|
||||
import type { DiscoveredFeed, FeedDiscoveryConfig, ProbeResult } from "./types.js";
|
||||
import { canonicalizeUrl } from "./url-safety.js";
|
||||
import { validateFeed } from "./validate-feed.js";
|
||||
|
||||
const COMMON_FEED_PATHS = [
|
||||
"/feed",
|
||||
"/rss",
|
||||
"/rss.xml",
|
||||
"/atom.xml",
|
||||
"/index.xml",
|
||||
"/feed.xml",
|
||||
"/blog/feed",
|
||||
"/blog/rss.xml",
|
||||
"/news/feed",
|
||||
"/posts/feed"
|
||||
];
|
||||
|
||||
const FEED_MIME_PATTERN = /(rss|atom|feed\+json|xml)/i;
|
||||
|
||||
export async function probeSite(homepageUrl: string, config: Partial<FeedDiscoveryConfig> = {}): Promise<ProbeResult> {
|
||||
const resolvedConfig = { ...readFeedDiscoveryConfig(), ...config };
|
||||
const errors: string[] = [];
|
||||
const candidates = new Set<string>();
|
||||
const canonicalHomepage = canonicalizeUrl(homepageUrl);
|
||||
|
||||
try {
|
||||
const response = await fetchTextWithGuards(canonicalHomepage, resolvedConfig);
|
||||
for (const href of extractAlternateFeedLinks(response.body, response.url || canonicalHomepage)) {
|
||||
candidates.add(href);
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
const homepage = new URL(canonicalHomepage);
|
||||
for (const path of COMMON_FEED_PATHS) {
|
||||
candidates.add(new URL(path, homepage.origin).toString());
|
||||
}
|
||||
|
||||
const feeds: DiscoveredFeed[] = [];
|
||||
for (const candidate of [...candidates].slice(0, resolvedConfig.maxProbes)) {
|
||||
const validation = await validateFeed(candidate, resolvedConfig);
|
||||
if (!validation.ok) {
|
||||
if (validation.error) {
|
||||
errors.push(`${candidate}: ${validation.error}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
feeds.push({
|
||||
title: validation.title ?? candidate,
|
||||
description: validation.description,
|
||||
homepageUrl: validation.homepageUrl ?? canonicalHomepage,
|
||||
feedUrl: candidate,
|
||||
canonicalFeedUrl: validation.canonicalFeedUrl,
|
||||
kind: validation.kind,
|
||||
provider: "web-feed-probe",
|
||||
language: validation.language,
|
||||
imageUrl: validation.imageUrl,
|
||||
lastPublishedAt: validation.lastPublishedAt,
|
||||
score: 0,
|
||||
confidence: 0.8,
|
||||
tags: [],
|
||||
sampleItems: validation.sampleItems,
|
||||
isValid: true,
|
||||
validationScore: 1
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
homepageUrl: canonicalHomepage,
|
||||
feeds,
|
||||
errors
|
||||
};
|
||||
}
|
||||
|
||||
export function extractAlternateFeedLinks(html: string, baseUrl: string) {
|
||||
const $ = cheerio.load(html);
|
||||
const links: string[] = [];
|
||||
|
||||
$("link[rel~='alternate']").each((_, element) => {
|
||||
const type = String($(element).attr("type") ?? "");
|
||||
const href = $(element).attr("href");
|
||||
if (href && FEED_MIME_PATTERN.test(type)) {
|
||||
links.push(canonicalizeUrl(href, baseUrl));
|
||||
}
|
||||
});
|
||||
|
||||
return [...new Set(links)];
|
||||
}
|
||||
45
plugins/feed-discovery/src/providers/index.ts
Normal file
45
plugins/feed-discovery/src/providers/index.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import type { FeedDiscoveryConfig, FeedDiscoveryProvider, FeedProviderManifest } from "../types.js";
|
||||
import { ITunesPodcastProvider } from "./itunes-podcast.js";
|
||||
import { ManualCuratedProvider } from "./manual-curated.js";
|
||||
import { OpmlDirectoryProvider } from "./opml-directory.js";
|
||||
import { PodcastIndexProvider } from "./podcastindex.js";
|
||||
import { WebCandidateFeedProbeProvider } from "./web-feed-probe.js";
|
||||
|
||||
export function createDefaultFeedProviders(config: FeedDiscoveryConfig): FeedDiscoveryProvider[] {
|
||||
return [
|
||||
new ManualCuratedProvider(),
|
||||
new OpmlDirectoryProvider(config.opmlPaths),
|
||||
new WebCandidateFeedProbeProvider(config),
|
||||
new ITunesPodcastProvider(config),
|
||||
new PodcastIndexProvider(config)
|
||||
];
|
||||
}
|
||||
|
||||
export function providerManifests(config: FeedDiscoveryConfig): FeedProviderManifest[] {
|
||||
return createDefaultFeedProviders(config).map((provider) => ({
|
||||
id: provider.id,
|
||||
name: provider.name,
|
||||
type: provider.id.includes("podcast") || provider.id.includes("itunes") ? "podcast" : "all",
|
||||
requiresApiKey: provider.requiresApiKey,
|
||||
enabledByDefault: provider.enabledByDefault && (!provider.requiresApiKey || hasProviderKey(provider.id, config)),
|
||||
description: describeProvider(provider.id)
|
||||
}));
|
||||
}
|
||||
|
||||
function hasProviderKey(id: string, config: FeedDiscoveryConfig) {
|
||||
if (id === "podcastindex") {
|
||||
return Boolean(config.podcastIndexApiKey && config.podcastIndexApiSecret);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function describeProvider(id: string) {
|
||||
const descriptions: Record<string, string> = {
|
||||
"manual-curated": "Searches locally curated high-trust feed sources.",
|
||||
"opml-directory": "Searches configured local OPML files.",
|
||||
"web-feed-probe": "Probes configured candidate homepages and direct URL queries for feeds.",
|
||||
"itunes-podcast": "Searches the public iTunes podcast directory.",
|
||||
podcastindex: "Searches PodcastIndex when API credentials are configured."
|
||||
};
|
||||
return descriptions[id] ?? "Feed discovery provider.";
|
||||
}
|
||||
64
plugins/feed-discovery/src/providers/itunes-podcast.ts
Normal file
64
plugins/feed-discovery/src/providers/itunes-podcast.ts
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
|
||||
|
||||
interface ITunesPodcast {
|
||||
collectionName?: string;
|
||||
artistName?: string;
|
||||
feedUrl?: string;
|
||||
collectionViewUrl?: string;
|
||||
artworkUrl600?: string;
|
||||
primaryGenreName?: string;
|
||||
releaseDate?: string;
|
||||
}
|
||||
|
||||
export class ITunesPodcastProvider implements FeedDiscoveryProvider {
|
||||
id = "itunes-podcast";
|
||||
name = "iTunes Podcast Search";
|
||||
enabledByDefault = true;
|
||||
requiresApiKey = false;
|
||||
|
||||
constructor(private readonly config: Pick<FeedDiscoveryConfig, "requestTimeoutMs" | "userAgent">) {}
|
||||
|
||||
async search(query: FeedDiscoveryQuery) {
|
||||
if (query.type && query.type !== "all" && query.type !== "podcast") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const url = new URL("https://itunes.apple.com/search");
|
||||
url.searchParams.set("term", query.q);
|
||||
url.searchParams.set("entity", "podcast");
|
||||
url.searchParams.set("limit", String(Math.min(query.limit ?? 25, 50)));
|
||||
if (query.locale) {
|
||||
url.searchParams.set("country", query.locale.toUpperCase());
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: { "user-agent": this.config.userAgent }
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`iTunes HTTP ${response.status}`);
|
||||
}
|
||||
const body = await response.json() as { results?: ITunesPodcast[] };
|
||||
return (body.results ?? [])
|
||||
.filter((item) => item.feedUrl)
|
||||
.map((item): DiscoveredFeed => ({
|
||||
title: item.collectionName ?? item.feedUrl ?? "Untitled podcast",
|
||||
description: item.artistName,
|
||||
homepageUrl: item.collectionViewUrl,
|
||||
feedUrl: item.feedUrl as string,
|
||||
kind: "podcast",
|
||||
provider: this.id,
|
||||
imageUrl: item.artworkUrl600,
|
||||
lastPublishedAt: item.releaseDate,
|
||||
score: 0,
|
||||
confidence: 0.85,
|
||||
tags: [item.primaryGenreName, "podcast"].filter((tag): tag is string => Boolean(tag))
|
||||
}));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
59
plugins/feed-discovery/src/providers/manual-curated.ts
Normal file
59
plugins/feed-discovery/src/providers/manual-curated.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
|
||||
|
||||
const CURATED_FEEDS: DiscoveredFeed[] = [
|
||||
{
|
||||
title: "Indie Hackers",
|
||||
description: "Stories and discussions from founders building profitable internet businesses.",
|
||||
homepageUrl: "https://www.indiehackers.com",
|
||||
feedUrl: "https://www.indiehackers.com/feed",
|
||||
kind: "blog",
|
||||
provider: "manual-curated",
|
||||
score: 0,
|
||||
confidence: 0.8,
|
||||
tags: ["indie", "startup", "microsaas", "saas"]
|
||||
},
|
||||
{
|
||||
title: "Hacker News",
|
||||
description: "Technology, startup, software, and open-source discussion.",
|
||||
homepageUrl: "https://news.ycombinator.com",
|
||||
feedUrl: "https://news.ycombinator.com/rss",
|
||||
kind: "news",
|
||||
provider: "manual-curated",
|
||||
score: 0,
|
||||
confidence: 0.75,
|
||||
tags: ["startups", "software", "open-source", "technology"]
|
||||
},
|
||||
{
|
||||
title: "GitHub Blog",
|
||||
description: "GitHub product, open-source, and developer ecosystem news.",
|
||||
homepageUrl: "https://github.blog",
|
||||
feedUrl: "https://github.blog/feed/",
|
||||
kind: "github",
|
||||
provider: "manual-curated",
|
||||
score: 0,
|
||||
confidence: 0.7,
|
||||
tags: ["github", "open-source", "software", "developers"]
|
||||
}
|
||||
];
|
||||
|
||||
export class ManualCuratedProvider implements FeedDiscoveryProvider {
|
||||
id = "manual-curated";
|
||||
name = "Manual Curated";
|
||||
enabledByDefault = true;
|
||||
requiresApiKey = false;
|
||||
|
||||
async search(query: FeedDiscoveryQuery) {
|
||||
const terms = normalize(query.q);
|
||||
return CURATED_FEEDS.filter((feed) => {
|
||||
if (query.type && query.type !== "all" && feed.kind !== query.type) {
|
||||
return false;
|
||||
}
|
||||
const haystack = normalize([feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" "));
|
||||
return terms.length === 0 || terms.some((term) => haystack.includes(term));
|
||||
}).map((feed) => ({ ...feed, provider: this.id }));
|
||||
}
|
||||
}
|
||||
|
||||
function normalize(value: string) {
|
||||
return value.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
||||
}
|
||||
92
plugins/feed-discovery/src/providers/opml-directory.ts
Normal file
92
plugins/feed-discovery/src/providers/opml-directory.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
import { XMLParser } from "fast-xml-parser";
|
||||
import type { DiscoveredFeed, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: "",
|
||||
textNodeName: "text"
|
||||
});
|
||||
|
||||
export class OpmlDirectoryProvider implements FeedDiscoveryProvider {
|
||||
id = "opml-directory";
|
||||
name = "OPML Directory";
|
||||
enabledByDefault = true;
|
||||
requiresApiKey = false;
|
||||
|
||||
constructor(private readonly paths: string[] = []) {}
|
||||
|
||||
async search(query: FeedDiscoveryQuery) {
|
||||
const feeds: DiscoveredFeed[] = [];
|
||||
for (const path of this.paths) {
|
||||
const content = await readFile(path, "utf8");
|
||||
const parsed = parser.parse(content) as unknown;
|
||||
for (const outline of collectOutlines(parsed)) {
|
||||
const feedUrl = stringValue(outline.xmlUrl);
|
||||
if (!feedUrl) {
|
||||
continue;
|
||||
}
|
||||
const title = stringValue(outline.title) || stringValue(outline.text) || feedUrl;
|
||||
const tags = [stringValue(outline.category)].filter((tag): tag is string => Boolean(tag));
|
||||
const feed: DiscoveredFeed = {
|
||||
title,
|
||||
description: stringValue(outline.description),
|
||||
homepageUrl: stringValue(outline.htmlUrl),
|
||||
feedUrl,
|
||||
kind: inferKind(tags.join(" "), query.type),
|
||||
provider: this.id,
|
||||
score: 0,
|
||||
confidence: 0.7,
|
||||
tags
|
||||
};
|
||||
if (matches(feed, query)) {
|
||||
feeds.push(feed);
|
||||
}
|
||||
}
|
||||
}
|
||||
return feeds;
|
||||
}
|
||||
}
|
||||
|
||||
function collectOutlines(value: unknown): Array<Record<string, unknown>> {
|
||||
if (!isRecord(value)) {
|
||||
return [];
|
||||
}
|
||||
const current = "xmlUrl" in value ? [value] : [];
|
||||
return [
|
||||
...current,
|
||||
...Object.values(value).flatMap((entry) => {
|
||||
if (Array.isArray(entry)) {
|
||||
return entry.flatMap(collectOutlines);
|
||||
}
|
||||
return collectOutlines(entry);
|
||||
})
|
||||
];
|
||||
}
|
||||
|
||||
function matches(feed: DiscoveredFeed, query: FeedDiscoveryQuery) {
|
||||
if (query.type && query.type !== "all" && feed.kind !== query.type) {
|
||||
return false;
|
||||
}
|
||||
const terms = query.q.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
|
||||
const haystack = [feed.title, feed.description, feed.homepageUrl, feed.feedUrl, feed.tags.join(" ")].join(" ").toLowerCase();
|
||||
return terms.length === 0 || terms.some((term) => haystack.includes(term));
|
||||
}
|
||||
|
||||
function inferKind(tags: string, requested?: FeedDiscoveryQuery["type"]) {
|
||||
if (requested && requested !== "all") {
|
||||
return requested;
|
||||
}
|
||||
if (tags.includes("podcast")) {
|
||||
return "podcast";
|
||||
}
|
||||
return "opml";
|
||||
}
|
||||
|
||||
function stringValue(value: unknown) {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
76
plugins/feed-discovery/src/providers/podcastindex.ts
Normal file
76
plugins/feed-discovery/src/providers/podcastindex.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
|
||||
|
||||
interface PodcastIndexFeed {
|
||||
title?: string;
|
||||
description?: string;
|
||||
url?: string;
|
||||
link?: string;
|
||||
image?: string;
|
||||
language?: string;
|
||||
newestItemPublishTime?: number;
|
||||
categories?: Record<string, string>;
|
||||
}
|
||||
|
||||
export class PodcastIndexProvider implements FeedDiscoveryProvider {
|
||||
id = "podcastindex";
|
||||
name = "PodcastIndex";
|
||||
enabledByDefault = false;
|
||||
requiresApiKey = true;
|
||||
|
||||
constructor(private readonly config: FeedDiscoveryConfig) {}
|
||||
|
||||
async search(query: FeedDiscoveryQuery) {
|
||||
if (!this.config.podcastIndexApiKey || !this.config.podcastIndexApiSecret) {
|
||||
return [];
|
||||
}
|
||||
if (query.type && query.type !== "all" && query.type !== "podcast") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const authDate = Math.floor(Date.now() / 1000).toString();
|
||||
const auth = createHash("sha1")
|
||||
.update(this.config.podcastIndexApiKey + this.config.podcastIndexApiSecret + authDate)
|
||||
.digest("hex");
|
||||
|
||||
const url = new URL("https://api.podcastindex.org/api/1.0/search/byterm");
|
||||
url.searchParams.set("q", query.q);
|
||||
url.searchParams.set("max", String(Math.min(query.limit ?? 25, 100)));
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), this.config.requestTimeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"user-agent": this.config.userAgent,
|
||||
"X-Auth-Date": authDate,
|
||||
"X-Auth-Key": this.config.podcastIndexApiKey,
|
||||
Authorization: auth
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`PodcastIndex HTTP ${response.status}`);
|
||||
}
|
||||
const body = await response.json() as { feeds?: PodcastIndexFeed[] };
|
||||
return (body.feeds ?? [])
|
||||
.filter((feed) => feed.url)
|
||||
.map((feed): DiscoveredFeed => ({
|
||||
title: feed.title ?? feed.url ?? "Untitled podcast",
|
||||
description: feed.description,
|
||||
homepageUrl: feed.link,
|
||||
feedUrl: feed.url as string,
|
||||
kind: "podcast",
|
||||
provider: this.id,
|
||||
language: feed.language,
|
||||
imageUrl: feed.image,
|
||||
lastPublishedAt: feed.newestItemPublishTime ? new Date(feed.newestItemPublishTime * 1000).toISOString() : undefined,
|
||||
score: 0,
|
||||
confidence: 0.9,
|
||||
tags: [...Object.values(feed.categories ?? {}), "podcast"]
|
||||
}));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
42
plugins/feed-discovery/src/providers/web-feed-probe.ts
Normal file
42
plugins/feed-discovery/src/providers/web-feed-probe.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import type { DiscoveredFeed, FeedDiscoveryConfig, FeedDiscoveryProvider, FeedDiscoveryQuery } from "../types.js";
|
||||
import { probeSite } from "../probe-site.js";
|
||||
|
||||
export class WebCandidateFeedProbeProvider implements FeedDiscoveryProvider {
|
||||
id = "web-feed-probe";
|
||||
name = "Web Candidate Feed Probe";
|
||||
enabledByDefault = true;
|
||||
requiresApiKey = false;
|
||||
|
||||
constructor(private readonly config: FeedDiscoveryConfig) {}
|
||||
|
||||
async search(query: FeedDiscoveryQuery) {
|
||||
const candidates = candidateUrls(query, this.config.candidateUrls).slice(0, this.config.maxProbes);
|
||||
const feeds: DiscoveredFeed[] = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const result = await probeSite(candidate, this.config);
|
||||
feeds.push(...result.feeds.map((feed) => ({ ...feed, provider: this.id })));
|
||||
}
|
||||
|
||||
return feeds.filter((feed) => !query.type || query.type === "all" || feed.kind === query.type);
|
||||
}
|
||||
}
|
||||
|
||||
function candidateUrls(query: FeedDiscoveryQuery, configured: string[]) {
|
||||
const candidates = new Set<string>();
|
||||
if (/^https?:\/\//i.test(query.q)) {
|
||||
candidates.add(query.q);
|
||||
}
|
||||
|
||||
for (const entry of configured) {
|
||||
const [url, ...keywords] = entry.split("|").map((part) => part.trim());
|
||||
if (!url) {
|
||||
continue;
|
||||
}
|
||||
if (keywords.length === 0 || keywords.some((keyword) => query.q.toLowerCase().includes(keyword.toLowerCase()))) {
|
||||
candidates.add(url);
|
||||
}
|
||||
}
|
||||
|
||||
return [...candidates];
|
||||
}
|
||||
95
plugins/feed-discovery/src/scoring.ts
Normal file
95
plugins/feed-discovery/src/scoring.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import type { DiscoveredFeed, FeedDiscoveryQuery } from "./types.js";
|
||||
|
||||
const PROVIDER_SCORES: Record<string, number> = {
|
||||
"manual-curated": 1,
|
||||
podcastindex: 0.9,
|
||||
"itunes-podcast": 0.85,
|
||||
"opml-directory": 0.75,
|
||||
"web-feed-probe": 0.7,
|
||||
rsshub: 0.7,
|
||||
reddit: 0.65,
|
||||
github: 0.65,
|
||||
youtube: 0.65,
|
||||
unknown: 0.3
|
||||
};
|
||||
|
||||
export function scoreFeed(feed: DiscoveredFeed, query: FeedDiscoveryQuery): DiscoveredFeed {
|
||||
const keywordScore = feed.keywordScore ?? calculateKeywordScore(feed, query.q);
|
||||
const freshnessScore = feed.freshnessScore ?? calculateFreshnessScore(feed.lastPublishedAt);
|
||||
const providerScore = feed.providerScore ?? PROVIDER_SCORES[feed.provider] ?? PROVIDER_SCORES.unknown;
|
||||
const validationScore = feed.validationScore ?? (feed.isValid === false ? 0.2 : 1);
|
||||
const score = keywordScore * 0.4 + freshnessScore * 0.25 + providerScore * 0.2 + validationScore * 0.15;
|
||||
|
||||
return {
|
||||
...feed,
|
||||
score: round(score),
|
||||
confidence: round(Math.max(feed.confidence, score * validationScore)),
|
||||
freshnessScore: round(freshnessScore),
|
||||
keywordScore: round(keywordScore),
|
||||
providerScore: round(providerScore),
|
||||
validationScore: round(validationScore)
|
||||
};
|
||||
}
|
||||
|
||||
export function calculateKeywordScore(feed: DiscoveredFeed, query: string) {
|
||||
const terms = normalizeTerms(query);
|
||||
if (terms.length === 0) {
|
||||
return 0.1;
|
||||
}
|
||||
|
||||
const weightedText = [
|
||||
[feed.title, 0.35],
|
||||
[feed.description, 0.2],
|
||||
[feed.homepageUrl, 0.1],
|
||||
[feed.feedUrl, 0.05],
|
||||
[feed.tags.join(" "), 0.15],
|
||||
[feed.sampleItems?.map((item) => item.title).join(" "), 0.15]
|
||||
] as const;
|
||||
|
||||
let score = 0;
|
||||
for (const [value, weight] of weightedText) {
|
||||
const text = String(value ?? "").toLowerCase();
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
const matches = terms.filter((term) => text.includes(term)).length;
|
||||
score += weight * (matches / terms.length);
|
||||
}
|
||||
|
||||
return Math.min(1, Math.max(0.05, score));
|
||||
}
|
||||
|
||||
export function calculateFreshnessScore(lastPublishedAt?: string) {
|
||||
if (!lastPublishedAt) {
|
||||
return 0.1;
|
||||
}
|
||||
const ageDays = (Date.now() - Date.parse(lastPublishedAt)) / 86_400_000;
|
||||
if (!Number.isFinite(ageDays) || ageDays < 0) {
|
||||
return 0.1;
|
||||
}
|
||||
if (ageDays <= 7) {
|
||||
return 1;
|
||||
}
|
||||
if (ageDays <= 30) {
|
||||
return 0.8;
|
||||
}
|
||||
if (ageDays <= 90) {
|
||||
return 0.6;
|
||||
}
|
||||
if (ageDays <= 365) {
|
||||
return 0.3;
|
||||
}
|
||||
return 0.1;
|
||||
}
|
||||
|
||||
function normalizeTerms(query: string) {
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function round(value: number) {
|
||||
return Math.round(value * 1000) / 1000;
|
||||
}
|
||||
113
plugins/feed-discovery/src/types.ts
Normal file
113
plugins/feed-discovery/src/types.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
export type FeedKind =
|
||||
| "blog"
|
||||
| "news"
|
||||
| "podcast"
|
||||
| "youtube"
|
||||
| "reddit"
|
||||
| "github"
|
||||
| "torrent"
|
||||
| "opml"
|
||||
| "rsshub"
|
||||
| "unknown";
|
||||
|
||||
export type FeedOutputFormat = "json" | "opml" | "rss" | "atom" | "json-feed";
|
||||
|
||||
export interface FeedDiscoveryQuery {
|
||||
q: string;
|
||||
type?: FeedKind | "all";
|
||||
limit?: number;
|
||||
locale?: string;
|
||||
freshnessDays?: number;
|
||||
includeDeadFeeds?: boolean;
|
||||
includeUnvalidated?: boolean;
|
||||
providers?: string[];
|
||||
}
|
||||
|
||||
export interface DiscoveredFeed {
|
||||
id?: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
homepageUrl?: string;
|
||||
feedUrl: string;
|
||||
canonicalFeedUrl?: string;
|
||||
kind: FeedKind;
|
||||
provider: string;
|
||||
language?: string;
|
||||
imageUrl?: string;
|
||||
lastPublishedAt?: string;
|
||||
score: number;
|
||||
confidence: number;
|
||||
freshnessScore?: number;
|
||||
keywordScore?: number;
|
||||
providerScore?: number;
|
||||
validationScore?: number;
|
||||
tags: string[];
|
||||
sampleItems?: FeedSampleItem[];
|
||||
isValid?: boolean;
|
||||
}
|
||||
|
||||
export interface FeedSampleItem {
|
||||
title: string;
|
||||
url?: string;
|
||||
publishedAt?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface FeedDiscoveryProvider {
|
||||
id: string;
|
||||
name: string;
|
||||
enabledByDefault: boolean;
|
||||
requiresApiKey: boolean;
|
||||
search(query: FeedDiscoveryQuery): Promise<DiscoveredFeed[]>;
|
||||
}
|
||||
|
||||
export interface FeedProviderManifest {
|
||||
id: string;
|
||||
name: string;
|
||||
type: FeedKind | "all";
|
||||
requiresApiKey: boolean;
|
||||
enabledByDefault: boolean;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface FeedDiscoveryResponse {
|
||||
query: string;
|
||||
normalizedQuery: string;
|
||||
count: number;
|
||||
providerErrors: Array<{ provider: string; error: string }>;
|
||||
results: DiscoveredFeed[];
|
||||
}
|
||||
|
||||
export interface FeedDiscoveryConfig {
|
||||
cacheTtlSeconds: number;
|
||||
maxProviders: number;
|
||||
maxProbes: number;
|
||||
requestTimeoutMs: number;
|
||||
maxBodyBytes: number;
|
||||
userAgent: string;
|
||||
opmlPaths: string[];
|
||||
candidateUrls: string[];
|
||||
podcastIndexApiKey?: string;
|
||||
podcastIndexApiSecret?: string;
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
ok: boolean;
|
||||
feedUrl: string;
|
||||
canonicalFeedUrl?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
homepageUrl?: string;
|
||||
kind: FeedKind;
|
||||
language?: string;
|
||||
imageUrl?: string;
|
||||
lastPublishedAt?: string;
|
||||
sampleItems: FeedSampleItem[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ProbeResult {
|
||||
homepageUrl: string;
|
||||
feeds: DiscoveredFeed[];
|
||||
errors: string[];
|
||||
}
|
||||
89
plugins/feed-discovery/src/url-safety.ts
Normal file
89
plugins/feed-discovery/src/url-safety.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
import { lookup } from "node:dns/promises";
|
||||
import { isIP } from "node:net";
|
||||
|
||||
const BLOCKED_HOSTS = new Set(["localhost", "metadata.google.internal"]);
|
||||
|
||||
export async function assertSafeHttpUrl(input: string) {
|
||||
const url = new URL(input);
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error(`Unsupported URL protocol: ${url.protocol}`);
|
||||
}
|
||||
|
||||
const hostname = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1");
|
||||
if (BLOCKED_HOSTS.has(hostname) || hostname.endsWith(".localhost")) {
|
||||
throw new Error(`Blocked internal hostname: ${hostname}`);
|
||||
}
|
||||
|
||||
if (isBlockedIp(hostname)) {
|
||||
throw new Error(`Blocked internal IP address: ${hostname}`);
|
||||
}
|
||||
|
||||
const addresses = await lookup(hostname, { all: true, verbatim: true });
|
||||
for (const address of addresses) {
|
||||
if (isBlockedIp(address.address)) {
|
||||
throw new Error(`Blocked internal resolved address: ${address.address}`);
|
||||
}
|
||||
}
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
export function canonicalizeUrl(input: string, base?: string) {
|
||||
const url = new URL(input, base);
|
||||
url.hash = "";
|
||||
|
||||
for (const param of [...url.searchParams.keys()]) {
|
||||
if (/^(utm_|fbclid$|gclid$|mc_cid$|mc_eid$)/i.test(param)) {
|
||||
url.searchParams.delete(param);
|
||||
}
|
||||
}
|
||||
|
||||
if ((url.protocol === "https:" && url.port === "443") || (url.protocol === "http:" && url.port === "80")) {
|
||||
url.port = "";
|
||||
}
|
||||
|
||||
if (url.pathname !== "/" && url.pathname.endsWith("/")) {
|
||||
url.pathname = url.pathname.slice(0, -1);
|
||||
}
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function isBlockedIp(value: string) {
|
||||
const kind = isIP(value);
|
||||
if (kind === 4) {
|
||||
const parts = value.split(".").map((part) => Number(part));
|
||||
const [a, b] = parts;
|
||||
return (
|
||||
a === 0 ||
|
||||
a === 10 ||
|
||||
a === 127 ||
|
||||
(a === 169 && b === 254) ||
|
||||
(a === 172 && b >= 16 && b <= 31) ||
|
||||
(a === 192 && b === 168) ||
|
||||
(a === 100 && b >= 64 && b <= 127)
|
||||
);
|
||||
}
|
||||
|
||||
if (kind === 6) {
|
||||
const normalized = value.toLowerCase();
|
||||
const mappedDottedIpv4 = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(normalized);
|
||||
if (mappedDottedIpv4) {
|
||||
return isBlockedIp(mappedDottedIpv4[1]);
|
||||
}
|
||||
const mappedHexIpv4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(normalized);
|
||||
if (mappedHexIpv4) {
|
||||
const high = Number.parseInt(mappedHexIpv4[1], 16);
|
||||
const low = Number.parseInt(mappedHexIpv4[2], 16);
|
||||
return isBlockedIp(`${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`);
|
||||
}
|
||||
return (
|
||||
normalized === "::1" ||
|
||||
normalized.startsWith("fc") ||
|
||||
normalized.startsWith("fd") ||
|
||||
normalized.startsWith("fe80")
|
||||
);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
27
plugins/feed-discovery/src/validate-feed.ts
Normal file
27
plugins/feed-discovery/src/validate-feed.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { readFeedDiscoveryConfig } from "./config.js";
|
||||
import { parseFeedDocument } from "./feed-parsing.js";
|
||||
import { fetchTextWithGuards } from "./http.js";
|
||||
import type { FeedDiscoveryConfig, ValidationResult } from "./types.js";
|
||||
import { canonicalizeUrl } from "./url-safety.js";
|
||||
|
||||
export async function validateFeed(feedUrl: string, config: Partial<FeedDiscoveryConfig> = {}): Promise<ValidationResult> {
|
||||
const resolvedConfig = { ...readFeedDiscoveryConfig(), ...config };
|
||||
try {
|
||||
const response = await fetchTextWithGuards(feedUrl, resolvedConfig);
|
||||
const canonicalFeedUrl = canonicalizeUrl(response.url || feedUrl);
|
||||
const parsed = parseFeedDocument(canonicalFeedUrl, response.body, response.contentType);
|
||||
return {
|
||||
...parsed,
|
||||
feedUrl,
|
||||
canonicalFeedUrl
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
feedUrl,
|
||||
kind: "unknown",
|
||||
sampleItems: [],
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
8
plugins/feed-discovery/tsconfig.json
Normal file
8
plugins/feed-discovery/tsconfig.json
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue