From b97f5420644c83a6e269c4f4c5859fda5aaaf91b Mon Sep 17 00:00:00 2001 From: OhOhWonOne <7220+njfio@users.noreply.github.com> Date: Sat, 20 Apr 2024 23:38:33 -0400 Subject: [PATCH 1/5] first round of initial output parsing. Added initial parsing for code blocks. --- fluent_cli/Cargo.toml | 1 + fluent_cli/src/client.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml index ce6dc4c..da8d1c7 100644 --- a/fluent_cli/Cargo.toml +++ b/fluent_cli/Cargo.toml @@ -18,3 +18,4 @@ atty = "0.2.14" amber = "0.2.0" base64 = "0.21.7" infer = "0.8.0" +pulldown-cmark = "0.9.0" \ No newline at end of file diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs index e7e5a7d..3c4f70c 100644 --- a/fluent_cli/src/client.rs +++ b/fluent_cli/src/client.rs @@ -58,6 +58,7 @@ pub fn handle_response(response_body: &str) -> Result<()> { Err(_) => parsed_output.question.clone(), // If parsing fails, use the original string }; + let code_blocks = extract_code_blocks(&parsed_output.text); // Print parsed data or further process it as needed println!("\n\n"); println!("\tText:\n\t{}\n", parsed_output.text); @@ -65,10 +66,44 @@ pub fn handle_response(response_body: &str) -> Result<()> { println!("\tChat ID: {}", parsed_output.chat_id); println!("\tSession ID: {}", parsed_output.session_id); println!("\tMemory Type: {:?}", parsed_output.memory_type); + println!("\tCode blocks: {:?}", code_blocks); Ok(()) } + +fn extract_code_blocks(markdown_content: &str) -> String { + let parser = Parser::new(markdown_content); + let mut in_code_block = false; + let mut code_blocks = String::new(); + let mut current_block = String::new(); + + for event in parser { + match event { + Event::Start(Tag::CodeBlock(_)) => { + in_code_block = true; + current_block.clear(); // Ensure the current block is empty when starting a new block + }, + Event::Text(text) if in_code_block => { + if !current_block.is_empty() { + current_block.push('\n'); // Add a newline to separate lines within the same block + } + current_block.push_str(&text); // Append text to the current block + }, + Event::End(Tag::CodeBlock(_)) => { + in_code_block = false; + if !code_blocks.is_empty() { + code_blocks.push('\n'); // Add a newline to separate different blocks + } + code_blocks.push_str(¤t_block); // Add the complete block to the result string + }, + _ => {} + } + } + + code_blocks +} + pub fn parse_fluent_cli_output(json_data: &str) -> Result { let output: FluentCliOutput = serde_json::from_str(json_data)?; Ok(output) @@ -143,6 +178,7 @@ use tokio::io::{AsyncReadExt as TokioAsyncReadExt, Result as IoResult}; use base64::encode; use std::collections::HashMap; use std::path::Path; +use pulldown_cmark::{Event, Parser, Tag}; pub(crate) async fn prepare_payload(flow: &FlowConfig, question: &str, file_path: Option<&str>, actual_final_context: Option) -> IoResult { From fd1003c9ca48b3932016012476aa0381ce4ca535 Mon Sep 17 00:00:00 2001 From: OhOhWonOne <7220+njfio@users.noreply.github.com> Date: Sun, 21 Apr 2024 00:21:58 -0400 Subject: [PATCH 2/5] code blocks are getting identified and capable of output. --- fluent_cli/Cargo.toml | 3 ++- fluent_cli/src/client.rs | 38 +++++++++----------------------------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml index da8d1c7..570619f 100644 --- a/fluent_cli/Cargo.toml +++ b/fluent_cli/Cargo.toml @@ -18,4 +18,5 @@ atty = "0.2.14" amber = "0.2.0" base64 = "0.21.7" infer = "0.8.0" -pulldown-cmark = "0.9.0" \ No newline at end of file +pulldown-cmark = "0.9.0" +regex = "1.10.4" \ No newline at end of file diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs index 3c4f70c..d03e072 100644 --- a/fluent_cli/src/client.rs +++ b/fluent_cli/src/client.rs @@ -72,38 +72,17 @@ pub fn handle_response(response_body: &str) -> Result<()> { } -fn extract_code_blocks(markdown_content: &str) -> String { - let parser = Parser::new(markdown_content); - let mut in_code_block = false; - let mut code_blocks = String::new(); - let mut current_block = String::new(); - - for event in parser { - match event { - Event::Start(Tag::CodeBlock(_)) => { - in_code_block = true; - current_block.clear(); // Ensure the current block is empty when starting a new block - }, - Event::Text(text) if in_code_block => { - if !current_block.is_empty() { - current_block.push('\n'); // Add a newline to separate lines within the same block - } - current_block.push_str(&text); // Append text to the current block - }, - Event::End(Tag::CodeBlock(_)) => { - in_code_block = false; - if !code_blocks.is_empty() { - code_blocks.push('\n'); // Add a newline to separate different blocks - } - code_blocks.push_str(¤t_block); // Add the complete block to the result string - }, - _ => {} - } - } - code_blocks +fn extract_code_blocks(markdown_content: &str) -> Vec { + let re = Regex::new(r"```[\w]*\n([\s\S]*?)\n```").unwrap(); + re.captures_iter(markdown_content) + .map(|cap| { + cap[1].trim().to_string() // Trim to remove leading/trailing whitespace + }) + .collect() } + pub fn parse_fluent_cli_output(json_data: &str) -> Result { let output: FluentCliOutput = serde_json::from_str(json_data)?; Ok(output) @@ -179,6 +158,7 @@ use base64::encode; use std::collections::HashMap; use std::path::Path; use pulldown_cmark::{Event, Parser, Tag}; +use regex::Regex; pub(crate) async fn prepare_payload(flow: &FlowConfig, question: &str, file_path: Option<&str>, actual_final_context: Option) -> IoResult { From 50c27c255d5eb99bbafe93798a7e7f2b2ae8ce3f Mon Sep 17 00:00:00 2001 From: OhOhWonOne <7220+njfio@users.noreply.github.com> Date: Sun, 21 Apr 2024 00:50:24 -0400 Subject: [PATCH 3/5] added quick skin of the markdown. --- fluent_cli/Cargo.toml | 3 ++- fluent_cli/src/client.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml index 570619f..33f341f 100644 --- a/fluent_cli/Cargo.toml +++ b/fluent_cli/Cargo.toml @@ -19,4 +19,5 @@ amber = "0.2.0" base64 = "0.21.7" infer = "0.8.0" pulldown-cmark = "0.9.0" -regex = "1.10.4" \ No newline at end of file +regex = "1.10.4" +termimad = "0.14" diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs index d03e072..1eb3963 100644 --- a/fluent_cli/src/client.rs +++ b/fluent_cli/src/client.rs @@ -59,18 +59,27 @@ pub fn handle_response(response_body: &str) -> Result<()> { }; let code_blocks = extract_code_blocks(&parsed_output.text); + // Print parsed data or further process it as needed println!("\n\n"); - println!("\tText:\n\t{}\n", parsed_output.text); - println!("\tQuestion:\n\t{}", question_text); + println!("\tResponse Text:\n{}\n", parsed_output.text); + println!("\tQuestion:\n{}", question_text); println!("\tChat ID: {}", parsed_output.chat_id); println!("\tSession ID: {}", parsed_output.session_id); println!("\tMemory Type: {:?}", parsed_output.memory_type); println!("\tCode blocks: {:?}", code_blocks); + println!("\tPretty printed text:\n"); + pretty_print_markdown(&parsed_output.text); Ok(()) } +fn pretty_print_markdown(markdown_content: &str) { + let skin = MadSkin::default(); // Default skin with basic Markdown styling + skin.print_text(markdown_content); +} + + fn extract_code_blocks(markdown_content: &str) -> Vec { @@ -159,6 +168,7 @@ use std::collections::HashMap; use std::path::Path; use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; +use termimad::MadSkin; pub(crate) async fn prepare_payload(flow: &FlowConfig, question: &str, file_path: Option<&str>, actual_final_context: Option) -> IoResult { From 6b16219f48fc8bf618823068a752dcae8e4f86e9 Mon Sep 17 00:00:00 2001 From: OhOhWonOne <7220+njfio@users.noreply.github.com> Date: Sun, 21 Apr 2024 11:18:45 -0400 Subject: [PATCH 4/5] working -m markdown and -parsed and -z all output option --- fluent_cli/Cargo.toml | 1 + fluent_cli/src/client.rs | 59 +++++++++++++++++++++++++--------------- fluent_cli/src/main.rs | 17 +++++++++++- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml index 33f341f..52df737 100644 --- a/fluent_cli/Cargo.toml +++ b/fluent_cli/Cargo.toml @@ -21,3 +21,4 @@ infer = "0.8.0" pulldown-cmark = "0.9.0" regex = "1.10.4" termimad = "0.14" +serde_yaml = "0.9.34+deprecated" diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs index 1eb3963..7075af6 100644 --- a/fluent_cli/src/client.rs +++ b/fluent_cli/src/client.rs @@ -13,6 +13,8 @@ use tokio::fs::File; use tokio::io; use tokio::io::AsyncReadExt; use crate::client; +use serde_yaml::to_string as to_yaml; // Add serde_yaml to your Cargo.toml if not already included + #[derive(Serialize, Deserialize, Debug)] struct FluentCliOutput { @@ -49,34 +51,46 @@ struct Upload { mime: String, } +#[derive(Debug)] +struct ResponseOutput { + response_text: String, + question: String, + chat_id: String, + session_id: String, + memory_type: Option, + code_blocks: Option>, // Only populated if `--parse-code-output` is present + pretty_text: Option, // Only populated if `--parse-code-output` is not present +} -pub fn handle_response(response_body: &str) -> Result<()> { - let parsed_output: FluentCliOutput = serde_json::from_str(response_body)?; - let question_parsed: Result = serde_json::from_str(&parsed_output.question); - let question_text = match question_parsed { - Ok(q) => q.question, - Err(_) => parsed_output.question.clone(), // If parsing fails, use the original string - }; - let code_blocks = extract_code_blocks(&parsed_output.text); +pub fn handle_response(response_body: &str, matches: &clap::ArgMatches) -> Result<()> { + let parsed_output: FluentCliOutput = serde_json::from_str(response_body)?; - // Print parsed data or further process it as needed - println!("\n\n"); - println!("\tResponse Text:\n{}\n", parsed_output.text); - println!("\tQuestion:\n{}", question_text); - println!("\tChat ID: {}", parsed_output.chat_id); - println!("\tSession ID: {}", parsed_output.session_id); - println!("\tMemory Type: {:?}", parsed_output.memory_type); - println!("\tCode blocks: {:?}", code_blocks); - println!("\tPretty printed text:\n"); - pretty_print_markdown(&parsed_output.text); + if matches.is_present("full-output") { + // Serialize the complete output to YAML and print to stdout + let json_output = serde_json::to_string(&parsed_output)?; + println!("{}", json_output); + } else if matches.is_present("parse-code-output") { + // Extract and print code blocks to stdout + let code_blocks = extract_code_blocks(&parsed_output.text); + for block in code_blocks { + println!("{}", block); + } + } else if matches.is_present("markdown-output") { + let pretty_text = pretty_format_markdown(&parsed_output.text); + eprintln!("{:?}", pretty_text); // Print to stderr + } else { + // Pretty-print markdown to stderr and output raw text to stdout + println!("{}", parsed_output.text); // Print to stdout + } Ok(()) } -fn pretty_print_markdown(markdown_content: &str) { - let skin = MadSkin::default(); // Default skin with basic Markdown styling - skin.print_text(markdown_content); +fn pretty_format_markdown(markdown_content: &str) { + let skin = MadSkin::default(); // Assuming `termimad` is used + let formatted = skin.print_text(markdown_content); // Render to a string + formatted } @@ -168,7 +182,8 @@ use std::collections::HashMap; use std::path::Path; use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; -use termimad::MadSkin; +use termimad::{FmtText, MadSkin}; +use termimad::minimad::once_cell::sync::Lazy; pub(crate) async fn prepare_payload(flow: &FlowConfig, question: &str, file_path: Option<&str>, actual_final_context: Option) -> IoResult { diff --git a/fluent_cli/src/main.rs b/fluent_cli/src/main.rs index 7efe0b4..b1ac0bd 100644 --- a/fluent_cli/src/main.rs +++ b/fluent_cli/src/main.rs @@ -71,6 +71,21 @@ async fn main() -> Result<(), Box> { .short('g') // Assigns a short flag .help("Generates a bash autocomplete script") .takes_value(false)) + .arg(Arg::new("parse-code-output") + .long("parse-code-output") + .short('p') // Assigns a short flag + .help("Extracts and displays only the code blocks from the response") + .takes_value(false)) + .arg(Arg::new("full-output") + .long("full-output") + .short('z') // Assigns a short flag + .help("Outputs all response data in JSON format") + .takes_value(false)) + .arg(Arg::new("markdown-output") + .long("markdown-output") + .short('m') // Assigns a short flag + .help("Outputs the response to the terminal in stylized markdown. Do not use for pipelines") + .takes_value(false)) .get_matches(); @@ -156,7 +171,7 @@ async fn main() -> Result<(), Box> { let payload = crate::client::prepare_payload(&flow, request, file_path, actual_final_context_clone ).await?; let response = crate::client::send_request(&flow, &payload).await?; - handle_response(response.as_str())?; + handle_response(response.as_str(), &matches)?; Ok(()) } From 03aa5315fd3dd6b0405a4ea5685368f1651cb3f3 Mon Sep 17 00:00:00 2001 From: OhOhWonOne <7220+njfio@users.noreply.github.com> Date: Sun, 21 Apr 2024 14:18:19 -0400 Subject: [PATCH 5/5] Uploads, Output Modifiers (parsed, markdown, full, Download), Secure Configuration, Transparent Run Time Security Modifiers. Incredible! --- fluent_cli/Cargo.toml | 2 + fluent_cli/src/client.rs | 127 ++++++++++++++++++++++++++++++++------- fluent_cli/src/config.rs | 2 + fluent_cli/src/main.rs | 12 +++- 4 files changed, 120 insertions(+), 23 deletions(-) diff --git a/fluent_cli/Cargo.toml b/fluent_cli/Cargo.toml index 52df737..c723d1d 100644 --- a/fluent_cli/Cargo.toml +++ b/fluent_cli/Cargo.toml @@ -22,3 +22,5 @@ pulldown-cmark = "0.9.0" regex = "1.10.4" termimad = "0.14" serde_yaml = "0.9.34+deprecated" +chrono = "0.4.38" +anyhow = "1.0.68" \ No newline at end of file diff --git a/fluent_cli/src/client.rs b/fluent_cli/src/client.rs index 7075af6..63ca715 100644 --- a/fluent_cli/src/client.rs +++ b/fluent_cli/src/client.rs @@ -1,6 +1,6 @@ use log::{debug, error}; use std::env; -use reqwest::{Client, Error}; +use reqwest::{Client}; use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use serde_json::{json, Value}; use std::time::Duration; @@ -62,31 +62,64 @@ struct ResponseOutput { pretty_text: Option, // Only populated if `--parse-code-output` is not present } - -pub fn handle_response(response_body: &str, matches: &clap::ArgMatches) -> Result<()> { - let parsed_output: FluentCliOutput = serde_json::from_str(response_body)?; - - if matches.is_present("full-output") { - // Serialize the complete output to YAML and print to stdout - let json_output = serde_json::to_string(&parsed_output)?; - println!("{}", json_output); - } else if matches.is_present("parse-code-output") { - // Extract and print code blocks to stdout - let code_blocks = extract_code_blocks(&parsed_output.text); - for block in code_blocks { - println!("{}", block); +use serde_json::Error as SerdeError; + +pub async fn handle_response(response_body: &str, matches: &clap::ArgMatches) -> Result<()> { + // Parse the response body, handle error properly here instead of unwrapping + debug!("Response body: {}", response_body); + let result = serde_json::from_str::(response_body); + + + let response_text = match result { + Ok(parsed_output) => { + // If parsing is successful, use the parsed data + debug!("{:?}", parsed_output); + if let Some(directory) = matches.value_of("download-media") { + let urls = extract_urls(response_body); // Assume extract_urls can handle any text + download_media(urls, directory).await; + } + if matches.is_present("markdown-output") { + let pretty_text = pretty_format_markdown(&parsed_output.text); // Ensure text is obtained correctly + eprintln!("{:?}", pretty_text); + } else if matches.is_present("parse-code-output") { + let code_blocks = extract_code_blocks(&parsed_output.text); + for block in code_blocks { + println!("{}", block); + } + } else if matches.is_present("full-output") { + println!("{}", response_body); // Output the text used, whether parsed or raw + } else { + println!("{}", parsed_output.text); // Output the text used, whether parsed or raw, but only if the --markdown-output flag is not set").text; + } + }, + Err(e) => { + // If there's an error parsing the JSON, print the error and the raw response body + eprintln!("Raw Webhook Output"); + if let Some(directory) = matches.value_of("download-media") { + let urls = extract_urls(response_body); // Assume extract_urls can handle any text + download_media(urls, directory).await; + } + println!("{}", response_body); + response_body.to_string(); } - } else if matches.is_present("markdown-output") { - let pretty_text = pretty_format_markdown(&parsed_output.text); - eprintln!("{:?}", pretty_text); // Print to stderr - } else { - // Pretty-print markdown to stderr and output raw text to stdout - println!("{}", parsed_output.text); // Print to stdout - } + }; Ok(()) } + + + + + + +fn extract_urls(text: &str) -> Vec { + let url_regex = Regex::new(r"https?://[^\s]+").unwrap(); + url_regex.find_iter(text) + .map(|mat| mat.as_str().to_string()) + .collect() +} + fn pretty_format_markdown(markdown_content: &str) { let skin = MadSkin::default(); // Assuming `termimad` is used let formatted = skin.print_text(markdown_content); // Render to a string @@ -112,6 +145,56 @@ pub fn parse_fluent_cli_output(json_data: &str) -> Result { } +use reqwest; + +use tokio::io::AsyncWriteExt; + +use chrono::Local; + + +use anyhow::{Context}; + +// Correct definition of the function returning a Result with a boxed dynamic error + + + +async fn download_media(urls: Vec, directory: &str) { + let client = reqwest::Client::new(); + + for url in urls { + match client.get(&url).send().await { + Ok(response) => { + if response.status().is_success() { + match response.bytes().await { + Ok(content) => { + let path = Path::new(&url); + let filename = if let Some(name) = path.file_name() { + format!("{}-{}.{}", name.to_string_lossy(), Local::now().format("%Y%m%d%H%M%S"), path.extension().unwrap_or_default().to_string_lossy()) + } else { + format!("download-{}.dat", Local::now().format("%Y%m%d%H%M%S")) + }; + let filepath = Path::new(directory).join(filename); + + match File::create(filepath).await { + Ok(mut file) => { + if let Err(e) = file.write_all(&content).await { + eprintln!("Failed to write to file: {}", e); + } + }, + Err(e) => eprintln!("Failed to create file: {}", e), + } + }, + Err(e) => eprintln!("Failed to read bytes from response: {}", e), + } + } else { + eprintln!("Failed to download {}: {}", url, response.status()); + } + }, + Err(e) => eprintln!("Failed to send request: {}", e), + } + } +} + // Change the signature to accept a simple string for `question` pub async fn send_request(flow: &FlowConfig, payload: &Value) -> reqwest::Result { @@ -179,9 +262,11 @@ use tokio::fs::File as TokioFile; // Alias to avoid confusion with std::fs::File use tokio::io::{AsyncReadExt as TokioAsyncReadExt, Result as IoResult}; use base64::encode; use std::collections::HashMap; +use std::io::ErrorKind; use std::path::Path; use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; +use serde::de::Error; use termimad::{FmtText, MadSkin}; use termimad::minimad::once_cell::sync::Lazy; diff --git a/fluent_cli/src/config.rs b/fluent_cli/src/config.rs index 0780c45..44119c3 100644 --- a/fluent_cli/src/config.rs +++ b/fluent_cli/src/config.rs @@ -193,3 +193,5 @@ complete -F autocomplete_flows fluent_cli "#) } + + diff --git a/fluent_cli/src/main.rs b/fluent_cli/src/main.rs index b1ac0bd..379ca58 100644 --- a/fluent_cli/src/main.rs +++ b/fluent_cli/src/main.rs @@ -12,6 +12,8 @@ use tokio::io::{self, AsyncReadExt}; use crate::client::handle_response; use crate::config::{EnvVarGuard, generate_bash_autocomplete_script}; +use anyhow::Result; + // use env_logger; // Uncomment this when you are using it to initialize logs #[tokio::main] @@ -86,6 +88,12 @@ async fn main() -> Result<(), Box> { .short('m') // Assigns a short flag .help("Outputs the response to the terminal in stylized markdown. Do not use for pipelines") .takes_value(false)) + .arg(Arg::new("download-media") + .long("download-media") + .short('d') // Assigns a short flag + .help("Downloads all media files listed in the output to a specified directory") + .takes_value(true) + .value_name("DIRECTORY")) .get_matches(); @@ -170,8 +178,8 @@ async fn main() -> Result<(), Box> { let payload = crate::client::prepare_payload(&flow, request, file_path, actual_final_context_clone ).await?; let response = crate::client::send_request(&flow, &payload).await?; - - handle_response(response.as_str(), &matches)?; + debug!("Handling Response"); + handle_response(response.as_str(), &matches).await?; Ok(()) }