Mintplex-Labs · timothycarambat · Apr 5, 2024 · Apr 4, 2024 · Apr 4, 2024 · Apr 4, 2024
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
@@ -39,12 +39,6 @@ const GeneralVectorDatabase = lazy(
   () => import("@/pages/GeneralSettings/VectorDatabase")
 );
 const GeneralSecurity = lazy(() => import("@/pages/GeneralSettings/Security"));
-const DataConnectors = lazy(
-  () => import("@/pages/GeneralSettings/DataConnectors")
-);
-const DataConnectorSetup = lazy(
-  () => import("@/pages/GeneralSettings/DataConnectors/Connectors")
-);
 const WorkspaceSettings = lazy(() => import("@/pages/WorkspaceSettings"));
 const EmbedConfigSetup = lazy(
   () => import("@/pages/GeneralSettings/EmbedConfigs")
@@ -145,15 +139,6 @@ export default function App() {
                 path="/settings/workspaces"
                 element={<ManagerRoute Component={AdminWorkspaces} />}
               />
-              <Route
-                path="/settings/data-connectors"
-                element={<ManagerRoute Component={DataConnectors} />}
-              />
-              <Route
-                path="/settings/data-connectors/:connector"
-                element={<ManagerRoute Component={DataConnectorSetup} />}
-              />
-
               {/* Onboarding Flow */}
               <Route path="/onboarding" element={<OnboardingFlow />} />
               <Route path="/onboarding/:step" element={<OnboardingFlow />} />

diff --git a/frontend/src/components/DataConnectorOption/index.jsx b/frontend/src/components/DataConnectorOption/index.jsx
@@ -1,6 +1,3 @@
-import paths from "@/utils/paths";
-import ConnectorImages from "./media";
-
 export default function DataConnectorOption({ slug }) {
   if (!DATA_CONNECTORS.hasOwnProperty(slug)) return null;
   const { path, image, name, description, link } = DATA_CONNECTORS[slug];
@@ -26,22 +23,3 @@ export default function DataConnectorOption({ slug }) {
     </a>
   );
 }
-
-export const DATA_CONNECTORS = {
-  github: {
-    name: "GitHub Repo",
-    path: paths.settings.dataConnectors.github(),
-    image: ConnectorImages.github,
-    description:
-      "Import an entire public or private Github repository in a single click.",
-    link: "https://github.com",
-  },
-  "youtube-transcript": {
-    name: "YouTube Transcript",
-    path: paths.settings.dataConnectors.youtubeTranscript(),
-    image: ConnectorImages.youtube,
-    description:
-      "Import the transcription of an entire YouTube video from a link.",
-    link: "https://youtube.com",
-  },
-};
diff --git a/frontend/src/components/DataConnectorOption/media/github.png b/frontend/src/components/DataConnectorOption/media/github.png
diff --git a/frontend/src/components/DataConnectorOption/media/github.svg b/frontend/src/components/DataConnectorOption/media/github.svg
diff --git a/frontend/src/components/DataConnectorOption/media/index.js b/frontend/src/components/DataConnectorOption/media/index.js
@@ -1,5 +1,5 @@
-import Github from "./github.png";
-import YouTube from "./youtube.png";
+import Github from "./github.svg";
+import YouTube from "./youtube.svg";
 
 const ConnectorImages = {
   github: Github,

diff --git a/frontend/src/components/DataConnectorOption/media/youtube.png b/frontend/src/components/DataConnectorOption/media/youtube.png
diff --git a/frontend/src/components/DataConnectorOption/media/youtube.svg b/frontend/src/components/DataConnectorOption/media/youtube.svg
diff --git a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/ConnectorOption/index.jsx b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/ConnectorOption/index.jsx
@@ -0,0 +1,25 @@
+export default function ConnectorOption({
+  slug,
+  selectedConnector,
+  setSelectedConnector,
+  image,
+  name,
+  description,
+}) {
+  return (
+    <button
+      onClick={() => setSelectedConnector(slug)}
+      className={`flex text-left gap-x-3.5 items-center py-2 px-4 hover:bg-white/10 ${
+        selectedConnector === slug ? "bg-white/10" : ""
+      } rounded-lg cursor-pointer w-full`}
+    >
+      <img src={image} alt={name} className="w-[40px] h-[40px] rounded-md" />
+      <div className="flex flex-col">
+        <div className="text-white font-bold text-[14px]">{name}</div>
+        <div>
+          <p className="text-[12px] text-white/60">{description}</p>
+        </div>
+      </div>
+    </button>
+  );
+}
diff --git a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Github/index.jsx b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Github/index.jsx
@@ -0,0 +1,271 @@
+import React, { useEffect, useState } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import pluralize from "pluralize";
+import { TagsInput } from "react-tag-input-component";
+import { Warning } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+
+const DEFAULT_BRANCHES = ["main", "master"];
+export default function GithubOptions() {
+  const [loading, setLoading] = useState(false);
+  const [repo, setRepo] = useState(null);
+  const [accessToken, setAccessToken] = useState(null);
+  const [ignores, setIgnores] = useState([]);
+
+  const [settings, setSettings] = useState({
+    repo: null,
+    accessToken: null,
+  });
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = new FormData(e.target);
+
+    try {
+      setLoading(true);
+      showToast(
+        "Fetching all files for repo - this may take a while.",
+        "info",
+        { clear: true, autoClose: false }
+      );
+      const { data, error } = await System.dataConnectors.github.collect({
+        repo: form.get("repo"),
+        accessToken: form.get("accessToken"),
+        branch: form.get("branch"),
+        ignorePaths: ignores,
+      });
+
+      if (!!error) {
+        showToast(error, "error", { clear: true });
+        setLoading(false);
+        return;
+      }
+
+      showToast(
+        `${data.files} ${pluralize("file", data.files)} collected from ${
+          data.author
+        }/${data.repo}:${data.branch}. Output folder is ${data.destination}.`,
+        "success",
+        { clear: true }
+      );
+      e.target.reset();
+      setLoading(false);
+      return;
+    } catch (e) {
+      console.error(e);
+      showToast(e.message, "error", { clear: true });
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pb-6 pb-16">
+        <form className="w-full" onSubmit={handleSubmit}>
+          <div className="w-full flex flex-col py-2">
+            <div className="w-full flex flex-col gap-4">
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white text-sm font-bold">
+                    GitHub Repo URL
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    Url of the GitHub repo you wish to collect.
+                  </p>
+                </div>
+                <input
+                  type="url"
+                  name="repo"
+                  className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="https://github.com/Mintplex-Labs/anything-llm"
+                  required={true}
+                  autoComplete="off"
+                  onChange={(e) => setRepo(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, repo })}
+                  spellCheck={false}
+                />
+              </div>
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white font-bold text-sm flex gap-x-2 items-center">
+                    <p className="font-bold text-white">Github Access Token</p>{" "}
+                    <p className="text-xs text-white/50 font-light flex items-center">
+                      optional
+                      {!accessToken && (
+                        <Warning
+                          size={14}
+                          className="ml-1 text-orange-500 cursor-pointer"
+                          data-tooltip-id="access-token-tooltip"
+                          data-tooltip-place="right"
+                        />
+                      )}
+                      <Tooltip
+                        delayHide={300}
+                        id="access-token-tooltip"
+                        className="max-w-xs"
+                        clickable={true}
+                      >
+                        <p className="text-sm">
+                          Without a{" "}
+                          <a
+                            href="https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens"
+                            rel="noreferrer"
+                            target="_blank"
+                            className="underline"
+                            onClick={(e) => e.stopPropagation()}
+                          >
+                            Personal Access Token
+                          </a>
+                          , the GitHub API may limit the number of files that
+                          can be collected due to rate limits. You can{" "}
+                          <a
+                            href="https://github.com/settings/personal-access-tokens/new"
+                            rel="noreferrer"
+                            target="_blank"
+                            className="underline"
+                            onClick={(e) => e.stopPropagation()}
+                          >
+                            create a temporary Access Token
+                          </a>{" "}
+                          to avoid this issue.
+                        </p>
+                      </Tooltip>
+                    </p>
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    Access Token to prevent rate limiting.
+                  </p>
+                </div>
+                <input
+                  type="text"
+                  name="accessToken"
+                  className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="github_pat_1234_abcdefg"
+                  required={false}
+                  autoComplete="off"
+                  spellCheck={false}
+                  onChange={(e) => setAccessToken(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, accessToken })}
+                />
+              </div>
+              <GitHubBranchSelection
+                repo={settings.repo}
+                accessToken={settings.accessToken}
+              />
+            </div>
+
+            <div className="flex flex-col w-full py-4 pr-10">
+              <div className="flex flex-col gap-y-1 mb-4">
+                <label className="text-white text-sm flex gap-x-2 items-center">
+                  <p className="text-white text-sm font-bold">File Ignores</p>
+                </label>
+                <p className="text-xs font-normal text-white/50">
+                  List in .gitignore format to ignore specific files during
+                  collection. Press enter after each entry you want to save.
+                </p>
+              </div>
+              <TagsInput
+                value={ignores}
+                onChange={setIgnores}
+                name="ignores"
+                placeholder="!*.js, images/*, .DS_Store, bin/*"
+                classNames={{
+                  tag: "bg-blue-300/10 text-zinc-800",
+                  input:
+                    "flex bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white",
+                }}
+              />
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-y-2 w-full pr-10">
+            <button
+              type="submit"
+              disabled={loading}
+              className="mt-2 w-full justify-center border border-slate-200 px-4 py-2 rounded-lg text-[#222628] text-sm font-bold items-center flex gap-x-2 bg-slate-200 hover:bg-slate-300 hover:text-slate-800 disabled:bg-slate-300 disabled:cursor-not-allowed"
+            >
+              {loading ? "Collecting files..." : "Submit"}
+            </button>
+            {loading && (
+              <p className="text-xs text-white/50">
+                Once complete, all files will be available for embedding into
+                workspaces in the document picker.
+              </p>
+            )}
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}
+
+function GitHubBranchSelection({ repo, accessToken }) {
+  const [allBranches, setAllBranches] = useState(DEFAULT_BRANCHES);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function fetchAllBranches() {
+      if (!repo) {
+        setAllBranches(DEFAULT_BRANCHES);
+        setLoading(false);
+        return;
+      }
+
+      setLoading(true);
+      const { branches } = await System.dataConnectors.github.branches({
+        repo,
+        accessToken,
+      });
+      setAllBranches(branches.length > 0 ? branches : DEFAULT_BRANCHES);
+      setLoading(false);
+    }
+    fetchAllBranches();
+  }, [repo, accessToken]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <div className="flex flex-col gap-y-1 mb-4">
+          <label className="text-white text-sm font-bold">Branch</label>
+          <p className="text-xs font-normal text-white/50">
+            Branch you wish to collect files from.
+          </p>
+        </div>
+        <select
+          name="branch"
+          required={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available branches --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <div className="flex flex-col gap-y-1 mb-4">
+        <label className="text-white text-sm font-bold">Branch</label>
+        <p className="text-xs font-normal text-white/50">
+          Branch you wish to collect files from.
+        </p>
+      </div>
+      <select
+        name="branch"
+        required={true}
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {allBranches.map((branch) => {
+          return (
+            <option key={branch} value={branch}>
+              {branch}
+            </option>
+          );
+        })}
+      </select>
+    </div>
+  );
+}