θΏ™ζ˜―indexlocζδΎ›ηš„ζœεŠ‘οΌŒδΈθ¦θΎ“ε…₯任何密码
Skip to content

Refactor loadYouTubeTranscript() to include YouTube Video Metadata in Content When parseOnly is true #4552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 15, 2025
Merged

Refactor loadYouTubeTranscript() to include YouTube Video Metadata in Content When parseOnly is true #4552

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8b4df30
Enhance YouTube transcript loading to include video metadata in parse…
angelplusultra Oct 15, 2025
5a2be57
extract to function
timothycarambat Oct 15, 2025
03e5eeb
Merge branch 'master' into 4551-feat-include-youtube-video-metadata-i…
timothycarambat Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 40 additions & 3 deletions collector/utils/extensions/YoutubeTranscript/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ const { validYoutubeVideoUrl } = require("../../url");
/**
* Fetch the transcript content for a YouTube video
* @param {string} url - The URL of the YouTube video
* @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: Object}>} - The transcript content for the YouTube video
* @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: TranscriptMetadata}>} - The transcript content for the YouTube video
*/
async function fetchVideoTranscriptContent({ url }) {
if (!validYoutubeVideoUrl(url)) {
Expand Down Expand Up @@ -64,11 +64,20 @@ async function fetchVideoTranscriptContent({ url }) {
};
}

/**
* @typedef {Object} TranscriptMetadata
* @property {string} title - The title of the video
* @property {string} author - The author of the video
* @property {string} description - The description of the video
* @property {string} view_count - The view count of the video
* @property {string} source - The source of the video (videoId)
*/

/**
* @typedef {Object} TranscriptAsDocument
* @property {boolean} success - Whether the transcript was successful
* @property {string|null} reason - The reason for the transcript
* @property {{title: string, author: string, destination: string}} data - The data from the transcript
* @property {TranscriptMetadata} metadata - The metadata from the transcript
*/

/**
Expand Down Expand Up @@ -104,11 +113,12 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) {
}

const { content, metadata } = transcriptResults;

if (options.parseOnly) {
return {
success: true,
reason: null,
content,
content: buildTranscriptContentWithMetadata(content, metadata),
documents: [],
saveAsDocument: options.parseOnly,
data: {},
Expand Down Expand Up @@ -154,6 +164,33 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) {
};
}

/**
* Generate the transcript content and metadata into a single string
*
* Why? For ephemeral documents where we just want the content, we want to include the metadata as keys in the content
* so that the LLM has context about the video, this gives it a better understanding of the video
* and allows it to use the metadata in the conversation if relevant.
* Examples:
* - How many views does <LINK> have?
* - Checkout <LINK> and tell me the key points and if it is performing well
* - Summarize this video <LINK>? -> description could have links and references
* @param {string} content - The content of the transcript
* @param {TranscriptMetadata} metadata - The metadata from the transcript
* @returns {string} - The concatenated transcript content and metadata
*/
function buildTranscriptContentWithMetadata(content = "", metadata = {}) {
const VALID_METADATA_KEYS = ["title", "author", "description", "view_count"];
if (!content || !metadata || Object.keys(metadata).length === 0)
return content;

let contentWithMetadata = "";
VALID_METADATA_KEYS.forEach((key) => {
if (!metadata[key]) return;
contentWithMetadata += `<${key}>${metadata[key]}</${key}>`;
});
return `${contentWithMetadata}\nTranscript:\n${content}`;
}

module.exports = {
loadYouTubeTranscript,
fetchVideoTranscriptContent,
Expand Down