Mintplex-Labs · timothycarambat · Jul 23, 2024 · Jul 10, 2024 · Jul 10, 2024 · Jul 10, 2024
diff --git a/.github/workflows/dev-build.yaml b/.github/workflows/dev-build.yaml
@@ -6,7 +6,7 @@ concurrency:
 
 on:
   push:
-    branches: ['1915-docker-perms'] # master branch only. Do not modify.
+    branches: ['-dev'] # put your current branch to create a build. Core team only.
     paths-ignore:
       - '**.md'
       - 'cloud-deployments/*'

diff --git a/collector/extensions/index.js b/collector/extensions/index.js
@@ -1,5 +1,6 @@
 const { setDataSigner } = require("../middleware/setDataSigner");
 const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
+const { resolveRepoLoader, resolveRepoLoaderFunction } = require("../utils/extensions/RepoLoader");
 const { reqBody } = require("../utils/http");
 const { validURL } = require("../utils/url");
 const RESYNC_METHODS = require("./resync");
@@ -28,15 +29,16 @@ function extensions(app) {
   )
 
   app.post(
-    "/ext/github-repo",
+    "/ext/:repo_platform-repo",
     [verifyPayloadIntegrity, setDataSigner],
     async function (request, response) {
       try {
-        const { loadGithubRepo } = require("../utils/extensions/GithubRepo");
-        const { success, reason, data } = await loadGithubRepo(
+        const loadRepo = resolveRepoLoaderFunction(request.params.repo_platform);
+        const { success, reason, data } = await loadRepo(
           reqBody(request),
           response,
         );
+        console.log({ success, reason, data })
         response.status(200).json({
           success,
           reason,
@@ -56,12 +58,12 @@ function extensions(app) {
 
   // gets all branches for a specific repo
   app.post(
-    "/ext/github-repo/branches",
+    "/ext/:repo_platform-repo/branches",
     [verifyPayloadIntegrity],
     async function (request, response) {
       try {
-        const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader");
-        const allBranches = await new GithubRepoLoader(
+        const RepoLoader = resolveRepoLoader(request.params.repo_platform);
+        const allBranches = await new RepoLoader(
           reqBody(request)
         ).getRepoBranches();
         response.status(200).json({

diff --git a/collector/extensions/resync/index.js b/collector/extensions/resync/index.js
@@ -86,7 +86,7 @@ async function resyncGithub({ chunkSource }, response) {
     // Github file data is `payload` encrypted (might contain PAT). So we need to expand its
     // encrypted payload back into query params so we can reFetch the page with same access token/params.
     const source = response.locals.encryptionWorker.expandPayload(chunkSource);
-    const { fetchGithubFile } = require("../../utils/extensions/GithubRepo");
+    const { fetchGithubFile } = require("../../utils/extensions/RepoLoader/GithubRepo");
     const { success, reason, content } = await fetchGithubFile({
       repoUrl: `https:${source.pathname}`, // need to add back the real protocol
       branch: source.searchParams.get('branch'),

diff --git a/collector/package.json b/collector/package.json
@@ -32,6 +32,7 @@
     "mammoth": "^1.6.0",
     "mbox-parser": "^1.0.1",
     "mime": "^3.0.0",
+    "minimatch": "5.1.0",
     "moment": "^2.29.4",
     "multer": "^1.4.5-lts.1",
     "node-html-parser": "^6.1.13",
@@ -50,4 +51,4 @@
     "nodemon": "^2.0.22",
     "prettier": "^2.4.1"
   }
-}
+}
diff --git a/...extensions/GithubRepo/RepoLoader/index.js → ...RepoLoader/GithubRepo/RepoLoader/index.js b/...extensions/GithubRepo/RepoLoader/index.js → ...RepoLoader/GithubRepo/RepoLoader/index.js
@@ -1,4 +1,21 @@
-class RepoLoader {
+/**
+ * @typedef {Object} RepoLoaderArgs
+ * @property {string} repo - The GitHub repository URL.
+ * @property {string} [branch] - The branch to load from (optional).
+ * @property {string} [accessToken] - GitHub access token for authentication (optional).
+ * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional).
+ */
+
+/**
+ * @class
+ * @classdesc Loads and manages GitHub repository content.
+ */
+class GitHubRepoLoader {
+  /**
+   * Creates an instance of RepoLoader.
+   * @param {RepoLoaderArgs} [args] - The configuration options.
+   * @returns {GitHubRepoLoader}
+   */
   constructor(args = {}) {
     this.ready = false;
     this.repo = args?.repo;
@@ -67,6 +84,10 @@ class RepoLoader {
     return;
   }
 
+  /**
+   * Initializes the RepoLoader instance.
+   * @returns {Promise<RepoLoader>} The initialized RepoLoader instance.
+   */
   async init() {
     if (!this.#validGithubUrl()) return;
     await this.#validBranch();
@@ -75,6 +96,11 @@ class RepoLoader {
     return this;
   }
 
+  /**
+   * Recursively loads the repository content.
+   * @returns {Promise<Array<Object>>} An array of loaded documents.
+   * @throws {Error} If the RepoLoader is not in a ready state.
+   */
   async recursiveLoader() {
     if (!this.ready) throw new Error("[Github Loader]: not in ready state!");
     const {
@@ -109,7 +135,10 @@ class RepoLoader {
     }, []);
   }
 
-  // Get all branches for a given repo.
+  /**
+   * Retrieves all branches for the repository.
+   * @returns {Promise<string[]>} An array of branch names.
+   */
   async getRepoBranches() {
     if (!this.#validGithubUrl() || !this.author || !this.project) return [];
     await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight
@@ -151,6 +180,11 @@ class RepoLoader {
     return this.#branchPrefSort(this.branches);
   }
 
+  /**
+   * Fetches the content of a single file from the repository.
+   * @param {string} sourceFilePath - The path to the file in the repository.
+   * @returns {Promise<string|null>} The content of the file, or null if fetching fails.
+   */
   async fetchSingleFile(sourceFilePath) {
     try {
       return fetch(
@@ -182,4 +216,4 @@ class RepoLoader {
   }
 }
 
-module.exports = RepoLoader;
+module.exports = GitHubRepoLoader;
diff --git a/...ctor/utils/extensions/GithubRepo/index.js → ...extensions/RepoLoader/GithubRepo/index.js b/...ctor/utils/extensions/GithubRepo/index.js → ...extensions/RepoLoader/GithubRepo/index.js
@@ -3,8 +3,8 @@ const fs = require("fs");
 const path = require("path");
 const { default: slugify } = require("slugify");
 const { v4 } = require("uuid");
-const { writeToServerDocuments } = require("../../files");
-const { tokenizeString } = require("../../tokenizer");
+const { writeToServerDocuments } = require("../../../files");
+const { tokenizeString } = require("../../../tokenizer");
 
 /**
  * Load in a Github Repo recursively or just the top level if no PAT is provided
@@ -42,7 +42,7 @@ async function loadGithubRepo(args, response) {
     process.env.NODE_ENV === "development"
       ? path.resolve(
           __dirname,
-          `../../../../server/storage/documents/${outFolder}`
+          `../../../../../server/storage/documents/${outFolder}`
         )
       : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);