diff --git a/docs/site/app/lib/llms-utils.ts b/docs/site/app/lib/llms-utils.ts new file mode 100644 index 0000000000000..21220d1156749 --- /dev/null +++ b/docs/site/app/lib/llms-utils.ts @@ -0,0 +1,37 @@ +import * as fs from "node:fs/promises"; +import fg from "fast-glob"; +import matter from "gray-matter"; +import { remark } from "remark"; +import remarkStringify from "remark-stringify"; +import remarkMdx from "remark-mdx"; + +export const DEFAULT_IGNORED_FILES = [ + "!./content/docs/acknowledgments.mdx", + "!./content/docs/community.mdx", + "!./content/docs/telemetry.mdx", +]; + +export async function scanDocumentationFiles( + patterns: Array = ["./content/docs/**/*.mdx"], + ignorePatterns: Array = DEFAULT_IGNORED_FILES +) { + return fg([...patterns, ...ignorePatterns]); +} + +export async function parseFileContent(filePath: string) { + const fileContent = await fs.readFile(filePath); + return matter(fileContent.toString()); +} + +export async function processMarkdownContent(content: string): Promise { + const file = await remark() + .use(remarkMdx) + .use(remarkStringify) + .process(content); + + return String(file); +} + +export function formatFilePath(filePath: string): string { + return filePath.replace("./content/docs", "").replace(/\.mdx$/, ".md"); +} diff --git a/docs/site/app/llms-full.txt/route.ts b/docs/site/app/llms-full.txt/route.ts new file mode 100644 index 0000000000000..fbce1cd31deda --- /dev/null +++ b/docs/site/app/llms-full.txt/route.ts @@ -0,0 +1,28 @@ +// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. + +import { + scanDocumentationFiles, + parseFileContent, + processMarkdownContent, + formatFilePath, +} from "../lib/llms-utils"; + +export const revalidate = false; + +export async function GET(): Promise { + // all scanned content + const files = await scanDocumentationFiles(); + + const scan = files.map(async (file) => { + const { content, data } = await parseFileContent(file); + + const processed = await processMarkdownContent(content); + return `- [${data.title}](${formatFilePath(file)}): ${data.description} + +${processed}`; + }); + + const scanned = await Promise.all(scan); + + return new Response(scanned.join("\n\n")); +} diff --git a/docs/site/app/llms.md/[[...slug]]/route.ts b/docs/site/app/llms.md/[[...slug]]/route.ts new file mode 100644 index 0000000000000..f7295336e08e9 --- /dev/null +++ b/docs/site/app/llms.md/[[...slug]]/route.ts @@ -0,0 +1,33 @@ +// This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. + +import { notFound } from "next/navigation"; +import { type NextRequest } from "next/server"; +import { repoDocsPages } from "../../source"; +import { parseFileContent, processMarkdownContent } from "../../lib/llms-utils"; + +export const revalidate = false; + +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ slug?: Array }> } +) { + const { slug } = await params; + const page = repoDocsPages.getPage(slug); + if (!page) notFound(); + + const { data, content } = await parseFileContent( + page.data._file.absolutePath + ); + const txt = await processMarkdownContent(content); + + const header = `# ${data.title} +Description: ${data.description} + +`; + + return new Response(header.concat(txt)); +} + +export function generateStaticParams() { + return repoDocsPages.generateParams(); +} diff --git a/docs/site/app/llms.txt/route.ts b/docs/site/app/llms.txt/route.ts index 51b756ac47fc2..c5358d13a0b18 100644 --- a/docs/site/app/llms.txt/route.ts +++ b/docs/site/app/llms.txt/route.ts @@ -1,44 +1,38 @@ // This file is mostly a copy-paste from https://fumadocs.vercel.app/docs/ui/llms. -import * as fs from "node:fs/promises"; -import fg from "fast-glob"; -import matter from "gray-matter"; -import { remark } from "remark"; -import remarkStringify from "remark-stringify"; -import remarkMdx from "remark-mdx"; +import { + scanDocumentationFiles, + parseFileContent, + formatFilePath, +} from "../lib/llms-utils"; +import { PRODUCT_SLOGANS } from "../../lib/constants"; export const revalidate = false; export async function GET(): Promise { // all scanned content - const files = await fg([ - "./content/docs/**/*.mdx", - "!./content/docs/acknowledgments.mdx", - "!./content/docs/community.mdx", - "!./content/docs/telemetry.mdx", - ]); - - const scan = files.map(async (file) => { - const fileContent = await fs.readFile(file); - const { content, data } = matter(fileContent.toString()); - - const processed = await processContent(content); - return `file: ${file} - meta: ${JSON.stringify(data, null, 2)} - - ${processed}`; + const files = await scanDocumentationFiles(); + + const scan = files.sort().map(async (file) => { + const { data } = await parseFileContent(file); + + return `- [${data.title}](${formatFilePath(file)}): ${data.description}`; }); const scanned = await Promise.all(scan); - return new Response(scanned.join("\n\n")); -} + const header = ` +# Turborepo documentation + +Generated at: ${new Date().toUTCString()} + +## Turborepo + +> ${PRODUCT_SLOGANS.turbo} + +## Docs -async function processContent(content: string): Promise { - const file = await remark() - .use(remarkMdx) - .use(remarkStringify) - .process(content); +`; - return String(file); + return new Response(header.concat(scanned.join("\n"))); } diff --git a/docs/site/next.config.ts b/docs/site/next.config.ts index 4fcc45aaf5f10..054bd5288d8ea 100644 --- a/docs/site/next.config.ts +++ b/docs/site/next.config.ts @@ -6,6 +6,11 @@ import { REDIRECTS_FOR_V2_DOCS } from "./lib/redirects/v2-docs.mjs"; const withMDX = createMDX(); const vercelToolbar = withVercelToolbar(); +const llmMarkdownRedirects = { + source: "/docs/:path*.md", + destination: "/llms.md/:path*", +}; + const config: NextConfig = { reactStrictMode: true, images: { @@ -40,8 +45,9 @@ const config: NextConfig = { source: "/api/feedback", destination: "https://vercel.com/api/feedback", }, + llmMarkdownRedirects, ] - : undefined, + : [llmMarkdownRedirects], }; }, // Next.js still expects these to return Promises even without await