diff --git a/collector/utils/extensions/YoutubeTranscript/index.js b/collector/utils/extensions/YoutubeTranscript/index.js index f6f970e4b9..c5fe997406 100644 --- a/collector/utils/extensions/YoutubeTranscript/index.js +++ b/collector/utils/extensions/YoutubeTranscript/index.js @@ -14,7 +14,7 @@ const { validYoutubeVideoUrl } = require("../../url"); /** * Fetch the transcript content for a YouTube video * @param {string} url - The URL of the YouTube video - * @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: Object}>} - The transcript content for the YouTube video + * @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: TranscriptMetadata}>} - The transcript content for the YouTube video */ async function fetchVideoTranscriptContent({ url }) { if (!validYoutubeVideoUrl(url)) { @@ -64,11 +64,20 @@ async function fetchVideoTranscriptContent({ url }) { }; } +/** + * @typedef {Object} TranscriptMetadata + * @property {string} title - The title of the video + * @property {string} author - The author of the video + * @property {string} description - The description of the video + * @property {string} view_count - The view count of the video + * @property {string} source - The source of the video (videoId) + */ + /** * @typedef {Object} TranscriptAsDocument * @property {boolean} success - Whether the transcript was successful * @property {string|null} reason - The reason for the transcript - * @property {{title: string, author: string, destination: string}} data - The data from the transcript + * @property {TranscriptMetadata} metadata - The metadata from the transcript */ /** @@ -104,11 +113,12 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) { } const { content, metadata } = transcriptResults; + if (options.parseOnly) { return { success: true, reason: null, - content, + content: buildTranscriptContentWithMetadata(content, metadata), documents: [], saveAsDocument: options.parseOnly, data: {}, @@ -154,6 +164,33 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) { }; } +/** + * Generate the transcript content and metadata into a single string + * + * Why? For ephemeral documents where we just want the content, we want to include the metadata as keys in the content + * so that the LLM has context about the video, this gives it a better understanding of the video + * and allows it to use the metadata in the conversation if relevant. + * Examples: + * - How many views does have? + * - Checkout and tell me the key points and if it is performing well + * - Summarize this video ? -> description could have links and references + * @param {string} content - The content of the transcript + * @param {TranscriptMetadata} metadata - The metadata from the transcript + * @returns {string} - The concatenated transcript content and metadata + */ +function buildTranscriptContentWithMetadata(content = "", metadata = {}) { + const VALID_METADATA_KEYS = ["title", "author", "description", "view_count"]; + if (!content || !metadata || Object.keys(metadata).length === 0) + return content; + + let contentWithMetadata = ""; + VALID_METADATA_KEYS.forEach((key) => { + if (!metadata[key]) return; + contentWithMetadata += `<${key}>${metadata[key]}`; + }); + return `${contentWithMetadata}\nTranscript:\n${content}`; +} + module.exports = { loadYouTubeTranscript, fetchVideoTranscriptContent,