njfio · njfio · Apr 25, 2024 · Apr 25, 2024 · Apr 25, 2024 · Apr 25, 2024
diff --git a/amber.yaml b/amber.yaml
@@ -13,9 +13,15 @@ secrets:
 - name: AMBER_FLUENT_GEMINI_API_KEY_01
   sha256: dcc3fc4b33e620c9d1c1fb81b1c21ec36eaa0a3d71137949e17120b996decbb7
   cipher: 0e05274547adb1cb064de584a1e69190edec52b22bf1059c8bd83d02146ed0732b7bcb3d9b01481d1b17432b66b6fc594f617aeecc5e4cb2a699539a6db7713f19da92356846b47b7324c7de089d550be4110a304de482
+- name: AMBER_FLUENT_GITHUB_PAT_KEY_01
+  sha256: ba3ab7e0594b847ce15f159c0c81137bf04228c61dbc030b023deae7ce0faefc
+  cipher: 8f3834349b01542ab105c97f4b42bbabbf4d42c93c56b3975c29ddcf04d6e45b6331110cedacba97cd185d9581d27ed1cf06cd04c7fc8246cc3fd8e66ab2a083d85dba17391499e0bc064c8f57338a8ba3e983bcb1c436d5f2bc5f28d698e1e51dadb7805402cd86d1d90accb0a47488540ab184dc081a1350876505fa774ba2b79e87bfa5ca4241ea07126a2d
 - name: AMBER_FLUENT_GROQ_API_KEY_01
   sha256: 89f7aeec36590e47a2eab221b2381dd70343ae36b13e8e17a9f6dc62f1cb80be
   cipher: 6ff9c99f465f1354cc8a107d5cbbd1165be08dd1c0a77b67bbb22dd4866dcb2a2bb48622edeb0bfcc1f3b100b1a83871e6ea90dde44ee9b4909641af7bfd69790b45518884bb686cc22045441ba48d668d87b3893869bc25bf1c21777d2c132322349f387b8730b3
+- name: AMBER_FLUENT_LANGSMITH_KEY_01
+  sha256: ee941a5c4b1e1d8f66f9aff29f528367b43decebfd302ba733e14e4db14758ba
+  cipher: ce521a396babf8658dc8b7378317b86f50de481e1b18e2838109ed53c5e2aa15f9702c313253fc20f71cbc84f3a1f04106786b120106921e09f2df78cd01dc3703f9a9408c66342762ec3409c57d04cd40a60613
 - name: AMBER_FLUENT_MISTRAL_KEY_01
   sha256: 9731bc831969cfa36c45cbd7fb6b757c9cfbba91a96a64a847f9b6205d84fd2a
   cipher: 1c0bd5f0a66ee85221bbd179cf342b191105e36413a90d3740a6e5c5ef716939df7144ebb1a1bbb9eb068664c76c335bc455c6bfe6b3ebb7bc40fdcea97cb4e4ce46eef06d95f127ecafe3a985a0ca56
@@ -25,6 +31,9 @@ secrets:
 - name: AMBER_FLUENT_PERPLEXITY_API_KEY_01
   sha256: 4bca46bf6cbe53f3d3aec1d7d8a1fe1e5b97b61678ca005be9c5878f6237cb85
   cipher: 07b6db3a91a9fffe0a3a71dab5f97d6c7b87bbe14bb5fc5e14426806a36a8c3983d2e87d5779fa7936438358d0c631cd75f1bb26881ad3326f53fc2279311a7b104ac417734785303d5aa848e7a4e85ab3cb19bf1912410557b95e600c597ac1aab0d37e43
+- name: AMBER_FLUENT_PINECONE_API_KEY_01
+  sha256: 544bf9c05a941a1b523f4be0afe6499b5047e34cd42506479e02268023949847
+  cipher: e1e95d74456bc5569c2269eacb43d0ceefd4e22ef1a06d03496bf7bbe964154b2e4a1168382e8ae8fe602425d4e610c28341a779afb8b9ee8b128b75f0fb9a4dc8fa329e6e63ffb9c82ea9c723052667552c0ff3
 - name: AMBER_FLUENT_SEARCHAPI_KEY_ID_01
   sha256: 3b378ff8071fe87a6311f10bcdd3f6171e1a0d08be1e4c4a70561ad9f2b24ec6
   cipher: 09aad10b270d089bbc5ec6ee160d217b65100c4e2bd0a8c06e16134efaa69868458fbd0b5b4559e43f6fb49c3bb691aabbc37c67ddf2d0257245b2068cc46139103b8b00b2d932c6

diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
-reqwest = { version = "0.11", features = ["json", "blocking"] }
+reqwest = { version = "0.11", features = ["json", "blocking", "multipart", "stream"] }
 tokio = { version = "1", features = ["full"] }
 clap = { version = "3.0", features = ["derive"] }
 tokio-console = "0.1" # For debugging async tasks
@@ -24,4 +24,7 @@ termimad = "0.14"
 serde_yaml = "0.9.34+deprecated"
 chrono = "0.4.38"
 anyhow = "1.0.68"
-colored = "2.1.0"
+colored = "2.1.0"
+tokio-util = "0.7.10"
+thiserror = "1.0.58"
+mime_guess = "2.0.4"
diff --git a/fluent_cli/config.json b/fluent_cli/config.json
@@ -9,7 +9,7 @@
     "sessionId": "",
     "bearer_token": "AMBER_REPO_CLOUD_FLUENT_DEMO_KEY",
     "overrideConfig": {
-      "chainName": "GPT4ImageUploadChain",
+      "chainName": "GPT4ImageUploadRepoCloud",
       "sessionId": "AMBER_FLUENT_SESSION_ID_01",
       "openAIApiKey": "AMBER_FLUENT_OPENAI_API_KEY_01",
       "searchApiKey": "AMBER_FLUENT_SEARCHAPI_KEY_ID_01",
@@ -19,6 +19,60 @@
     },
     "timeout_ms": 500000
   },
+  {
+    "name": "GPT4ToolAgentWithTextFileUpsertRepoCloud",
+    "protocol": "https",
+    "hostname": "9d81nz4o.rpcld.co",
+    "port": 443,
+    "chat_id": "85e4e8b3-28c9-4405-b4d9-9ec270cbabfe",
+    "request_path": "/api/v1/prediction/",
+    "upsert_path": "/api/v1/vector/upsert/",
+    "sessionId": "",
+    "bearer_token": "AMBER_REPO_CLOUD_FLUENT_DEMO_KEY",
+    "overrideConfig": {
+      "chainName": "GPT4ImageUploadChain",
+      "sessionId": "AMBER_FLUENT_SESSION_ID_01",
+      "openAIApiKey": "AMBER_FLUENT_OPENAI_API_KEY_01",
+      "searchApiKey": "AMBER_FLUENT_SEARCHAPI_KEY_ID_01",
+      "accessToken": "AMBER_FLUENT_GITHUB_PAT_KEY_01"
+    },
+    "timeout_ms": 500000
+  },
+  {
+    "name": "GPT4ToolAgentWithUpsertRepoCloud",
+    "protocol": "https",
+    "hostname": "9d81nz4o.rpcld.co",
+    "port": 443,
+    "chat_id": "931b1268-561d-494d-8f37-b5f5caac4466",
+    "request_path": "/api/v1/prediction/",
+    "upsert_path": "/api/v1/vector/upsert/",
+    "sessionId": "",
+    "bearer_token": "AMBER_REPO_CLOUD_FLUENT_DEMO_KEY",
+    "overrideConfig": {
+      "chainName": "GPT4ToolAgentWithUpsertRepoCloud",
+      "sessionId": "AMBER_FLUENT_SESSION_ID_01",
+      "openAIApiKey": "AMBER_FLUENT_OPENAI_API_KEY_01",
+      "searchApiKey": "AMBER_FLUENT_SEARCHAPI_KEY_ID_01",
+      "accessToken": "AMBER_FLUENT_GITHUB_PAT_KEY_01",
+      "pineconeApiKey": "AMBER_FLUENT_PINECONE_API_KEY_01",
+      "repoLink": "https://github.com/njfio/fluent_cli/",
+      "branch":  "main",
+      "recursive": true,
+      "maxRetries" : 3,
+      "language":{
+        "codeTextSplitter_0": "rust"
+      },
+      "pineconeIndex": "fluentcli",
+      "modelName": {
+        "chatOpenAI_0": "gpt-4-turbo-preview",
+        "openAIEmbeddings_1": "text-embedding-3-small"
+      },
+      "name": "fluentcli_retriever",
+      "description": "fluentcli source code retriever",
+      "returnSourceDocuments": true
+    },
+    "timeout_ms": 500000
+  },
   {
     "name": "GPT4FunctionAgentWithMemoryAndBrowsingRepoCloud",
     "protocol": "https",
@@ -234,6 +288,7 @@
     "port": 443,
     "chat_id": "9634304b-6b2a-44cf-ad61-e1afadaf7bf1",
     "request_path": "/api/v1/prediction/",
+    "upsert_path": "/api/v1/upsert/",
     "sessionId": "",
     "bearer_token": "AMBER_REPO_CLOUD_FLUENT_DEMO_KEY",
     "overrideConfig": {

diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs
@@ -1,6 +1,6 @@
 use log::{debug, error};
 use std::env;
-use reqwest::{Client};
+use reqwest::{Client, multipart};
 use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION};
 use serde_json::{json, Value};
 use std::time::Duration;
@@ -246,17 +246,146 @@ pub(crate) fn build_request_payload(question: &str, context: Option<&str>) -> Va
 
 
 
+use thiserror::Error;
+use reqwest::Error as ReqwestError;
+use serde_json::Error as SerdeJsonError;
+use std::io::Error as IoError;
+
+#[derive(Error, Debug)]
+pub enum MyError {
+    #[error("Network error: {0}")]
+    Network(#[from] ReqwestError),
+
+    #[error("JSON error: {0}")]
+    Json(#[from] SerdeJsonError),
+
+    #[error("I/O error: {0}")]
+    Io(#[from] IoError),
+
+    #[error("Other error: {0}")]
+    Other(String),
+}
+
+
+
+// Function returns Result<(), serde_json::Error
+
+
+
+use std::error::Error as StdError;  // Import the StdError trait for `source` method
+
+
+
+fn to_json_error<E: std::fmt::Display>(error: E) -> SerdeError {
+    // Simulate a JSON parsing error
+    let faulty_json = format!("{{: \"{}\"", error); // Intentionally malformed JSON
+    serde_json::from_str::<Value>(&faulty_json).unwrap_err()
+}
+
+
+fn to_serde_json_error<E: std::fmt::Display>(err: E) -> serde_json::Error {
+    serde_json::Error::custom(err.to_string())
+}
+
+
+pub async fn upsert_with_json(api_url: &str, flow: &FlowConfig, payload: serde_json::Value) -> Result<()> {
+    let bearer_token = if flow.bearer_token.starts_with("AMBER_") {
+        env::var(&flow.bearer_token[6..]).unwrap_or_else(|_| flow.bearer_token.clone())
+    } else {
+        flow.bearer_token.clone()
+    };
+
+    debug!("Bearer token: {}", bearer_token);
+
+    let client = reqwest::Client::new();
+    debug!("Sending to URL: {}", api_url);
+    debug!("Payload: {:?}", payload);
+    eprintln!("Upserting with JSON: {:?}", payload);
+    let response = client.post(api_url)
+        .header("Content-Type", "application/json")
+        .header("Authorization", format!("Bearer {}", bearer_token))
+        .json(&payload)
+        .send()
+        .await
+        .map_err(to_serde_json_error)?;
+
+    if response.status().is_success() {
+        let response_json: serde_json::Value = response.json().await.map_err(to_serde_json_error)?;
+        debug!("Success response: {:#?}", response_json);
+        println!("Success: {:#?}", response_json);
+    } else {
+        let error_message = format!("Failed to upsert data: Status code: {}", response.status());
+        return Err(serde_json::Error::custom(error_message));
+    }
+
+    Ok(())
+}
+
+pub async fn upload_files(api_url: &str, file_paths: Vec<&str>) -> Result<()> {
+    let client = Client::new();
+    let mut form = Form::new();
+    let mut file_paths_clone = file_paths.clone();
+
+    for file_path in file_paths {
+        let path = Path::new(file_path);
+        debug!("File path: {}", path.display());
+        let mime_type = mime_guess::from_path(path).first_or_octet_stream().essence_str().to_string(); // Convert to String here
+        debug!("MIME type: {}", mime_type);
+        let mut file = match File::open(path).await {
+            Ok(f) => f,
+            Err(e) => return Err(to_serde_json_error(e)),
+        };
+        debug!("File opened: {}", file_path);
+        let mut buffer = Vec::new();
+        if let Err(e) = file.read_to_end(&mut buffer).await {
+            return Err(to_serde_json_error(e));
+        }
+
+        let part = Part::bytes(buffer)
+            .file_name(path.file_name().unwrap().to_str().unwrap().to_owned())
+            .mime_str(&mime_type).map_err(to_serde_json_error)?; // Use a reference to the owned String
+
+        form = form.part("files", part);
+    }
+    debug!("Form: {:?}", form);
+    debug!("API URL: {}", api_url);
+    debug!("File paths: {:?}", file_paths_clone);
+    let response = client.post(api_url)
+        .multipart(form)
+        .send()
+        .await
+        .map_err(to_serde_json_error)?;
+
+    debug!("Response: {:?}", response);
+    if !response.status().is_success() {
+        return Err(to_serde_json_error(format!("Failed to upload files: Status code: {}", response.status())));
+    }
+
+    let response_json: serde_json::Value = response.json().await.map_err(to_serde_json_error)?;
+    println!("{:#?}", response_json);
+
+    Ok(())
+}
+
+
+
+
+
 use tokio::fs::File as TokioFile; // Alias to avoid confusion with std::fs::File
 use tokio::io::{AsyncReadExt as TokioAsyncReadExt, Result as IoResult};
 use base64::encode;
 use std::collections::HashMap;
-use std::error::Error;
+
 use std::io::ErrorKind;
 use std::path::Path;
 use pulldown_cmark::{Event, Parser, Tag};
 use regex::Regex;
+use reqwest::multipart::{Form, Part};
+use serde::de::Error;
+
 use termimad::{FmtText, MadSkin};
 use termimad::minimad::once_cell::sync::Lazy;
+use tokio_util::codec::{BytesCodec, FramedRead};
 
 
 pub(crate) async fn prepare_payload(flow: &FlowConfig, question: &str, file_path: Option<&str>, actual_final_context: Option<String>) -> IoResult<Value> {

diff --git a/fluent_cli/src/config.rs b/fluent_cli/src/config.rs
@@ -15,6 +15,7 @@ pub struct FlowConfig {
     pub port: u16,
     pub chat_id: String,
     pub request_path: String,
+    pub upsert_path: Option<String>,
     #[serde(rename = "sessionId")]
     pub session_id: String,
     pub bearer_token: String,