From e1d7ec174243334facd24cfcb5207acf0f859676 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 04:42:01 -0600 Subject: [PATCH 1/7] docs: rework llms.txt and add .md responses --- docs/site/app/llms-full.txt/route.ts | 45 ++++++++++++++++++++++ docs/site/app/llms.md/[[...slug]]/route.ts | 24 ++++++++++++ docs/site/app/llms.txt/route.ts | 36 ++++++++--------- docs/site/next.config.ts | 11 +++++- 4 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 docs/site/app/llms-full.txt/route.ts create mode 100644 docs/site/app/llms.md/[[...slug]]/route.ts diff --git a/docs/site/app/llms-full.txt/route.ts b/docs/site/app/llms-full.txt/route.ts new file mode 100644 index 0000000000000..97b80f0fa0924 --- /dev/null +++ b/docs/site/app/llms-full.txt/route.ts @@ -0,0 +1,45 @@ +// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. + +import * as fs from "node:fs/promises"; +import fg from "fast-glob"; +import matter from "gray-matter"; +import { remark } from "remark"; +import remarkStringify from "remark-stringify"; +import remarkMdx from "remark-mdx"; + +export const revalidate = false; + +export async function GET(): Promise { + // all scanned content + const files = await fg([ + "./content/docs/**/*.mdx", + "!./content/docs/acknowledgments.mdx", + "!./content/docs/community.mdx", + "!./content/docs/telemetry.mdx", + ]); + + const scan = files.map(async (file) => { + const fileContent = await fs.readFile(file); + const { content, data } = matter(fileContent.toString()); + + const processed = await processContent(content); + return `- [${data.title}](${file + .replace("./content/docs", "") + .replace(/\.mdx$/, ".md")}): ${data.description} + + ${processed}`; + }); + + const scanned = await Promise.all(scan); + + return new Response(scanned.join("\n\n")); +} + +async function processContent(content: string): Promise { + const file = await remark() + .use(remarkMdx) + .use(remarkStringify) + .process(content); + + return String(file); +} diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts new file mode 100644 index 0000000000000..c051453ad497b --- /dev/null +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -0,0 +1,24 @@ +// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. + +import * as fs from "node:fs/promises"; +import { notFound } from "next/navigation"; +import { type NextRequest, NextResponse } from "next/server"; +import { repoDocsPages } from "../../source"; + +export const revalidate = false; + +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ slug?: Array }> } +) { + const { slug } = await params; + const page = repoDocsPages.getPage(slug); + if (!page) notFound(); + + const fileContent = await fs.readFile(page.data._file.absolutePath); + return new NextResponse(fileContent); +} + +export function generateStaticParams() { + return repoDocsPages.generateParams(); +} diff --git a/docs/site/app/llms.txt/route.ts b/docs/site/app/llms.txt/route.ts index 51b756ac47fc2..185740ac4c5d7 100644 --- a/docs/site/app/llms.txt/route.ts +++ b/docs/site/app/llms.txt/route.ts @@ -3,9 +3,7 @@ import * as fs from "node:fs/promises"; import fg from "fast-glob"; import matter from "gray-matter"; -import { remark } from "remark"; -import remarkStringify from "remark-stringify"; -import remarkMdx from "remark-mdx"; +import { PRODUCT_SLOGANS } from "../../lib/constants"; export const revalidate = false; @@ -18,27 +16,29 @@ export async function GET(): Promise { "!./content/docs/telemetry.mdx", ]); - const scan = files.map(async (file) => { + const scan = files.sort().map(async (file) => { const fileContent = await fs.readFile(file); - const { content, data } = matter(fileContent.toString()); + const { data } = matter(fileContent.toString()); - const processed = await processContent(content); - return `file: ${file} - meta: ${JSON.stringify(data, null, 2)} - - ${processed}`; + return `- [${data.title}](${file + .replace("./content/docs", "") + .replace(/\.mdx$/, ".md")}): ${data.description}`; }); const scanned = await Promise.all(scan); - return new Response(scanned.join("\n\n")); -} + const header = ` +# Turborepo documentation + +Generated at: ${new Date().toUTCString()} + +## Turborepo + +> ${PRODUCT_SLOGANS.turbo} + +## Docs -async function processContent(content: string): Promise { - const file = await remark() - .use(remarkMdx) - .use(remarkStringify) - .process(content); +`; - return String(file); + return new Response(header.concat(scanned.join("\n"))); } diff --git a/docs/site/next.config.ts b/docs/site/next.config.ts index 4fcc45aaf5f10..938417286b987 100644 --- a/docs/site/next.config.ts +++ b/docs/site/next.config.ts @@ -40,8 +40,17 @@ const config: NextConfig = { source: "/api/feedback", destination: "https://vercel.com/api/feedback", }, + { + source: "/docs/:path*.mdx", + destination: "/llms.mdx/:path*", + }, ] - : undefined, + : [ + { + source: "/docs/:path*.md", + destination: "/llms.md/:path*", + }, + ], }; }, // Next.js still expects these to return Promises even without await From 5fd64ce1b620debe260179232bb964b289f7f1d6 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 04:54:39 -0600 Subject: [PATCH 2/7] WIP 37607 --- docs/site/app/lib/llms-utils.ts | 37 ++++++++++++++++++++++ docs/site/app/llms-full.txt/route.ts | 37 ++++++---------------- docs/site/app/llms.md/[[...slug]]/route.ts | 4 +-- docs/site/app/llms.txt/route.ts | 22 +++++-------- 4 files changed, 57 insertions(+), 43 deletions(-) create mode 100644 docs/site/app/lib/llms-utils.ts diff --git a/docs/site/app/lib/llms-utils.ts b/docs/site/app/lib/llms-utils.ts new file mode 100644 index 0000000000000..21220d1156749 --- /dev/null +++ b/docs/site/app/lib/llms-utils.ts @@ -0,0 +1,37 @@ +import * as fs from "node:fs/promises"; +import fg from "fast-glob"; +import matter from "gray-matter"; +import { remark } from "remark"; +import remarkStringify from "remark-stringify"; +import remarkMdx from "remark-mdx"; + +export const DEFAULT_IGNORED_FILES = [ + "!./content/docs/acknowledgments.mdx", + "!./content/docs/community.mdx", + "!./content/docs/telemetry.mdx", +]; + +export async function scanDocumentationFiles( + patterns: Array = ["./content/docs/**/*.mdx"], + ignorePatterns: Array = DEFAULT_IGNORED_FILES +) { + return fg([...patterns, ...ignorePatterns]); +} + +export async function parseFileContent(filePath: string) { + const fileContent = await fs.readFile(filePath); + return matter(fileContent.toString()); +} + +export async function processMarkdownContent(content: string): Promise { + const file = await remark() + .use(remarkMdx) + .use(remarkStringify) + .process(content); + + return String(file); +} + +export function formatFilePath(filePath: string): string { + return filePath.replace("./content/docs", "").replace(/\.mdx$/, ".md"); +} diff --git a/docs/site/app/llms-full.txt/route.ts b/docs/site/app/llms-full.txt/route.ts index 97b80f0fa0924..a9146e8203ddc 100644 --- a/docs/site/app/llms-full.txt/route.ts +++ b/docs/site/app/llms-full.txt/route.ts @@ -1,31 +1,23 @@ // This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. -import * as fs from "node:fs/promises"; -import fg from "fast-glob"; -import matter from "gray-matter"; -import { remark } from "remark"; -import remarkStringify from "remark-stringify"; -import remarkMdx from "remark-mdx"; +import { + scanDocumentationFiles, + parseFileContent, + processMarkdownContent, + formatFilePath, +} from "../lib/llms-utils"; export const revalidate = false; export async function GET(): Promise { // all scanned content - const files = await fg([ - "./content/docs/**/*.mdx", - "!./content/docs/acknowledgments.mdx", - "!./content/docs/community.mdx", - "!./content/docs/telemetry.mdx", - ]); + const files = await scanDocumentationFiles(); const scan = files.map(async (file) => { - const fileContent = await fs.readFile(file); - const { content, data } = matter(fileContent.toString()); + const { content, data } = await parseFileContent(file); - const processed = await processContent(content); - return `- [${data.title}](${file - .replace("./content/docs", "") - .replace(/\.mdx$/, ".md")}): ${data.description} + const processed = await processMarkdownContent(content); + return `- [${data.title}](${formatFilePath(file)}): ${data.description} ${processed}`; }); @@ -34,12 +26,3 @@ export async function GET(): Promise { return new Response(scanned.join("\n\n")); } - -async function processContent(content: string): Promise { - const file = await remark() - .use(remarkMdx) - .use(remarkStringify) - .process(content); - - return String(file); -} diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts index c051453ad497b..ee5d1e00b2206 100644 --- a/docs/site/app/llms.md/[[...slug]]/route.ts +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -2,7 +2,7 @@ import * as fs from "node:fs/promises"; import { notFound } from "next/navigation"; -import { type NextRequest, NextResponse } from "next/server"; +import { type NextRequest } from "next/server"; import { repoDocsPages } from "../../source"; export const revalidate = false; @@ -16,7 +16,7 @@ export async function GET( if (!page) notFound(); const fileContent = await fs.readFile(page.data._file.absolutePath); - return new NextResponse(fileContent); + return new Response(fileContent); } export function generateStaticParams() { diff --git a/docs/site/app/llms.txt/route.ts b/docs/site/app/llms.txt/route.ts index 185740ac4c5d7..c5358d13a0b18 100644 --- a/docs/site/app/llms.txt/route.ts +++ b/docs/site/app/llms.txt/route.ts @@ -1,28 +1,22 @@ // This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. -import * as fs from "node:fs/promises"; -import fg from "fast-glob"; -import matter from "gray-matter"; +import { + scanDocumentationFiles, + parseFileContent, + formatFilePath, +} from "../lib/llms-utils"; import { PRODUCT_SLOGANS } from "../../lib/constants"; export const revalidate = false; export async function GET(): Promise { // all scanned content - const files = await fg([ - "./content/docs/**/*.mdx", - "!./content/docs/acknowledgments.mdx", - "!./content/docs/community.mdx", - "!./content/docs/telemetry.mdx", - ]); + const files = await scanDocumentationFiles(); const scan = files.sort().map(async (file) => { - const fileContent = await fs.readFile(file); - const { data } = matter(fileContent.toString()); + const { data } = await parseFileContent(file); - return `- [${data.title}](${file - .replace("./content/docs", "") - .replace(/\.mdx$/, ".md")}): ${data.description}`; + return `- [${data.title}](${formatFilePath(file)}): ${data.description}`; }); const scanned = await Promise.all(scan); From cda2c59cb32d845c04ea56ff3a277b5ca95bc6b0 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 04:56:06 -0600 Subject: [PATCH 3/7] WIP d0c48 --- docs/site/next.config.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/site/next.config.ts b/docs/site/next.config.ts index 938417286b987..054bd5288d8ea 100644 --- a/docs/site/next.config.ts +++ b/docs/site/next.config.ts @@ -6,6 +6,11 @@ import { REDIRECTS_FOR_V2_DOCS } from "./lib/redirects/v2-docs.mjs"; const withMDX = createMDX(); const vercelToolbar = withVercelToolbar(); +const llmMarkdownRedirects = { + source: "/docs/:path*.md", + destination: "/llms.md/:path*", +}; + const config: NextConfig = { reactStrictMode: true, images: { @@ -40,17 +45,9 @@ const config: NextConfig = { source: "/api/feedback", destination: "https://vercel.com/api/feedback", }, - { - source: "/docs/:path*.mdx", - destination: "/llms.mdx/:path*", - }, + llmMarkdownRedirects, ] - : [ - { - source: "/docs/:path*.md", - destination: "/llms.md/:path*", - }, - ], + : [llmMarkdownRedirects], }; }, // Next.js still expects these to return Promises even without await From d330a66954473b325adfc60d3ea9b0f2c035726f Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 05:02:53 -0600 Subject: [PATCH 4/7] WIP 01bf2 --- docs/site/app/llms.md/[[...slug]]/route.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts index ee5d1e00b2206..3fa3e5923df2e 100644 --- a/docs/site/app/llms.md/[[...slug]]/route.ts +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -1,9 +1,9 @@ // This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. -import * as fs from "node:fs/promises"; import { notFound } from "next/navigation"; import { type NextRequest } from "next/server"; import { repoDocsPages } from "../../source"; +import { parseFileContent, processMarkdownContent } from "../../lib/llms-utils"; export const revalidate = false; @@ -15,8 +15,9 @@ export async function GET( const page = repoDocsPages.getPage(slug); if (!page) notFound(); - const fileContent = await fs.readFile(page.data._file.absolutePath); - return new Response(fileContent); + const { content } = await parseFileContent(page.data._file.absolutePath); + const txt = await processMarkdownContent(content); + return new Response(txt); } export function generateStaticParams() { From 01c70cd3891a4e61e49781a70e884784aeca879f Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 05:08:04 -0600 Subject: [PATCH 5/7] WIP ad4df --- docs/site/app/llms.md/[[...slug]]/route.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts index 3fa3e5923df2e..ffab114e902e8 100644 --- a/docs/site/app/llms.md/[[...slug]]/route.ts +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -15,9 +15,18 @@ export async function GET( const page = repoDocsPages.getPage(slug); if (!page) notFound(); - const { content } = await parseFileContent(page.data._file.absolutePath); + const { data, content } = await parseFileContent( + page.data._file.absolutePath + ); const txt = await processMarkdownContent(content); - return new Response(txt); + + const header = ` +# ${data.title} +Description: ${data.description} + +`; + + return new Response(header.concat(txt)); } export function generateStaticParams() { From 9e5dbbd9dc16441232344ad2d1dd861d8d8ae659 Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 05:08:14 -0600 Subject: [PATCH 6/7] WIP 85cab --- docs/site/app/llms.md/[[...slug]]/route.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts index ffab114e902e8..f7295336e08e9 100644 --- a/docs/site/app/llms.md/[[...slug]]/route.ts +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -20,8 +20,7 @@ export async function GET( ); const txt = await processMarkdownContent(content); - const header = ` -# ${data.title} + const header = `# ${data.title} Description: ${data.description} `; From dc16753b71974a126b0af19e4713da3572e4843e Mon Sep 17 00:00:00 2001 From: Anthony Shew Date: Fri, 29 Aug 2025 08:07:17 -0600 Subject: [PATCH 7/7] Update docs/site/app/llms-full.txt/route.ts Co-authored-by: vercel[bot] <35613825+vercel[bot]@users.noreply.github.com> --- docs/site/app/llms-full.txt/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/site/app/llms-full.txt/route.ts b/docs/site/app/llms-full.txt/route.ts index a9146e8203ddc..fbce1cd31deda 100644 --- a/docs/site/app/llms-full.txt/route.ts +++ b/docs/site/app/llms-full.txt/route.ts @@ -19,7 +19,7 @@ export async function GET(): Promise { const processed = await processMarkdownContent(content); return `- [${data.title}](${formatFilePath(file)}): ${data.description} - ${processed}`; +${processed}`; }); const scanned = await Promise.all(scan);