From a4cc7375c98fa56e53a91d98df1e2cc7ebcc69e2 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 5 Mar 2024 16:56:30 -0800 Subject: [PATCH 1/7] Add migrations for running anthropic --- .../migrations/20240306005619_add_anthropic/migration.sql | 2 ++ libs/superagent/prisma/schema.prisma | 1 + 2 files changed, 3 insertions(+) create mode 100644 libs/superagent/prisma/migrations/20240306005619_add_anthropic/migration.sql diff --git a/libs/superagent/prisma/migrations/20240306005619_add_anthropic/migration.sql b/libs/superagent/prisma/migrations/20240306005619_add_anthropic/migration.sql new file mode 100644 index 000000000..79d6cce25 --- /dev/null +++ b/libs/superagent/prisma/migrations/20240306005619_add_anthropic/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "LLMProvider" ADD VALUE 'ANTHROPIC'; diff --git a/libs/superagent/prisma/schema.prisma b/libs/superagent/prisma/schema.prisma index ebfc77782..5306ea927 100644 --- a/libs/superagent/prisma/schema.prisma +++ b/libs/superagent/prisma/schema.prisma @@ -22,6 +22,7 @@ enum LLMProvider { HUGGINGFACE PERPLEXITY TOGETHER_AI + ANTHROPIC } enum LLMModel { From 76fd724033e838426fef95786df27058a79e61b5 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 5 Mar 2024 17:29:14 -0800 Subject: [PATCH 2/7] Add support for Anthropic LLMs --- libs/superagent/app/agents/llm.py | 1 + .../app/api/workflow_configs/saml_schema.py | 2 ++ libs/superagent/poetry.lock | 8 ++++---- libs/superagent/pyproject.toml | 2 +- libs/ui/config/site.ts | 13 +++++++++++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/libs/superagent/app/agents/llm.py b/libs/superagent/app/agents/llm.py index dc8a290b0..289446364 100644 --- a/libs/superagent/app/agents/llm.py +++ b/libs/superagent/app/agents/llm.py @@ -1,4 +1,5 @@ import logging +import litellm from typing import Optional from langchain.schema.messages import AIMessage diff --git a/libs/superagent/app/api/workflow_configs/saml_schema.py b/libs/superagent/app/api/workflow_configs/saml_schema.py index 8b3a3ad86..257c5691c 100644 --- a/libs/superagent/app/api/workflow_configs/saml_schema.py +++ b/libs/superagent/app/api/workflow_configs/saml_schema.py @@ -134,6 +134,7 @@ class LLMAgentTool(BaseAgentToolModel, LLMAgent): SAML_OSS_LLM_PROVIDERS = [ LLMProvider.PERPLEXITY.value, LLMProvider.TOGETHER_AI.value, + LLMProvider.ANTHROPIC.value, ] @@ -143,6 +144,7 @@ class Workflow(BaseModel): # ~~OSS LLM providers~~ perplexity: Optional[LLMAgent] together_ai: Optional[LLMAgent] + anthropic: Optional[LLMAgent] llm: Optional[LLMAgent] = Field( description="Deprecated! Use LLM providers instead. e.g. `perplexity` or `together_ai`" ) diff --git a/libs/superagent/poetry.lock b/libs/superagent/poetry.lock index afe1ac263..8caa0353d 100644 --- a/libs/superagent/poetry.lock +++ b/libs/superagent/poetry.lock @@ -2259,13 +2259,13 @@ requests = ">=2,<3" [[package]] name = "litellm" -version = "1.27.6" +version = "1.29.3" description = "Library to easily interface with LLM API providers" optional = false python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" files = [ - {file = "litellm-1.27.6-py3-none-any.whl", hash = "sha256:a9b2831065d89b9ea02e8abe07a1e09e280ccebcd8e1a0fcbeba89561174c6fc"}, - {file = "litellm-1.27.6.tar.gz", hash = "sha256:6a9a80fa463549846b7d0af565df593f5c565d2572527998688ba797f6ba69bf"}, + {file = "litellm-1.29.3-py3-none-any.whl", hash = "sha256:73a79f50c22919a2b5e834123b676270cf99673f70247e9b5ce964bebb65b73a"}, + {file = "litellm-1.29.3.tar.gz", hash = "sha256:0b2d8ed698d24183b1f2520070bb1c28e62b554899731a8ce36351f4349622d4"}, ] [package.dependencies] @@ -5927,4 +5927,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1, <3.12" -content-hash = "360ca47b78aa00dd95b3575f5b04d748ad7dfae34aa52e2e8c6eb2e92bfee1f1" +content-hash = "b34a793a28448f4aaacc857ac0235ff4b8703dbb1dbfe6f290a8cd7d451ad70f" diff --git a/libs/superagent/pyproject.toml b/libs/superagent/pyproject.toml index c4d110e83..4b8c2d285 100644 --- a/libs/superagent/pyproject.toml +++ b/libs/superagent/pyproject.toml @@ -50,7 +50,7 @@ openai = "^1.1.1" langchain-experimental = "^0.0.37" pydub = "^0.25.1" algoliasearch = "^3.0.0" -litellm = "^1.14.1" +litellm = "^1.29.3" weaviate-client = "^3.25.3" qdrant-client = "^1.6.9" vecs = "^0.4.2" diff --git a/libs/ui/config/site.ts b/libs/ui/config/site.ts index 3ef3731af..a173cbe0a 100644 --- a/libs/ui/config/site.ts +++ b/libs/ui/config/site.ts @@ -434,6 +434,19 @@ export const siteConfig = { }, ], }, + { + disabled: false, + formDescription: "Please enter your Anthropic API key.", + provider: "ANTHROPIC", + name: "Anthropic", + metadata: [ + { + key: "apiKey", + type: "input", + label: "Anthropic API Key", + }, + ], + }, { disabled: true, formDescription: "Please enter your HF API key.", From 1742fc479515c9389964c9a0e3f1d82094d7479e Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 5 Mar 2024 21:08:47 -0800 Subject: [PATCH 3/7] Add support for anthropic model --- libs/superagent/poetry.lock | 8 ++++---- libs/superagent/pyproject.toml | 2 +- libs/ui/app/workflows/[id]/chat.tsx | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libs/superagent/poetry.lock b/libs/superagent/poetry.lock index 8caa0353d..7d0812c6b 100644 --- a/libs/superagent/poetry.lock +++ b/libs/superagent/poetry.lock @@ -2259,13 +2259,13 @@ requests = ">=2,<3" [[package]] name = "litellm" -version = "1.29.3" +version = "1.29.4" description = "Library to easily interface with LLM API providers" optional = false python-versions = ">=3.8, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*" files = [ - {file = "litellm-1.29.3-py3-none-any.whl", hash = "sha256:73a79f50c22919a2b5e834123b676270cf99673f70247e9b5ce964bebb65b73a"}, - {file = "litellm-1.29.3.tar.gz", hash = "sha256:0b2d8ed698d24183b1f2520070bb1c28e62b554899731a8ce36351f4349622d4"}, + {file = "litellm-1.29.4-py3-none-any.whl", hash = "sha256:014b03fd37864d12acb095511f42bb46b74bf77a0c7086eb5d7d3ea0a27cc238"}, + {file = "litellm-1.29.4.tar.gz", hash = "sha256:14a3e5c5aaa042b2a732374f56260afd7761625d8ee6ac38f6e1de1c5ee5f792"}, ] [package.dependencies] @@ -5927,4 +5927,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1, <3.12" -content-hash = "b34a793a28448f4aaacc857ac0235ff4b8703dbb1dbfe6f290a8cd7d451ad70f" +content-hash = "e82f732de8734cd7b820d427a7e56a793eabf9377b33faee7e8d2b613b946b89" diff --git a/libs/superagent/pyproject.toml b/libs/superagent/pyproject.toml index 4b8c2d285..e00a2b583 100644 --- a/libs/superagent/pyproject.toml +++ b/libs/superagent/pyproject.toml @@ -50,7 +50,7 @@ openai = "^1.1.1" langchain-experimental = "^0.0.37" pydub = "^0.25.1" algoliasearch = "^3.0.0" -litellm = "^1.29.3" +litellm = "^1.29.4" weaviate-client = "^3.25.3" qdrant-client = "^1.6.9" vecs = "^0.4.2" diff --git a/libs/ui/app/workflows/[id]/chat.tsx b/libs/ui/app/workflows/[id]/chat.tsx index bc93cb9be..e1c2ce47a 100644 --- a/libs/ui/app/workflows/[id]/chat.tsx +++ b/libs/ui/app/workflows/[id]/chat.tsx @@ -141,6 +141,7 @@ export default function Chat({ resetState() }, async onmessage(event) { + console.log(event) if (event.id) currentEventId = event.id if (event.event === "function_call") { From fe23de84146178e59bfdcd857b89bd700607db2c Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Wed, 6 Mar 2024 00:08:39 -0800 Subject: [PATCH 4/7] Rewrite streaming funcitonality --- libs/superagent/app/utils/callbacks.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/libs/superagent/app/utils/callbacks.py b/libs/superagent/app/utils/callbacks.py index d462f6f8c..d136fee3f 100644 --- a/libs/superagent/app/utils/callbacks.py +++ b/libs/superagent/app/utils/callbacks.py @@ -60,36 +60,27 @@ async def on_llm_error(self, *args: Any, **kwargs: Any) -> None: # noqa async def aiter(self) -> AsyncIterator[str]: while not self.queue.empty() or not self.done.is_set(): - # Wait for the next token in the queue, - # but stop waiting if the done event is set - done, other = await asyncio.wait( + done, pending = await asyncio.wait( [ - # NOTE: If you add other tasks here, update the code below, - # which assumes each set has exactly one task each asyncio.ensure_future(self.queue.get()), asyncio.ensure_future(self.done.wait()), ], return_when=asyncio.FIRST_COMPLETED, timeout=self.TIMEOUT_SECONDS, ) - # if we the timeout has been reached - if not done or not other: + if not done: logger.warning(f"{self.TIMEOUT_SECONDS} seconds of timeout reached") self.done.set() break - # Cancel the other task - if other: - other.pop().cancel() + for future in pending: + future.cancel() - # Extract the value of the first completed task token_or_done = cast(Union[str, Literal[True]], done.pop().result()) - # If the extracted value is the boolean True, the done event was set - if token_or_done is True: + if token_or_done is True and self.queue.empty(): break - # Otherwise, the extracted value is a token, which we yield yield token_or_done From 0607dcb9ce820ae32894e2b375c5fcb8f9cebf33 Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Wed, 6 Mar 2024 20:32:39 +0400 Subject: [PATCH 5/7] fix: streaming --- libs/superagent/app/agents/llm.py | 18 +----------------- libs/superagent/app/utils/callbacks.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/libs/superagent/app/agents/llm.py b/libs/superagent/app/agents/llm.py index 289446364..761d0df21 100644 --- a/libs/superagent/app/agents/llm.py +++ b/libs/superagent/app/agents/llm.py @@ -1,9 +1,6 @@ import logging -import litellm from typing import Optional -from langchain.schema.messages import AIMessage -from langchain.schema.output import ChatGeneration, LLMResult from litellm import acompletion from app.agents.base import AgentBase @@ -123,20 +120,7 @@ async def ainvoke(self, input, *_, **kwargs): output += token await streaming.on_llm_new_token(token) - await streaming.on_llm_end( - response=LLMResult( - generations=[ - [ - ChatGeneration( - message=AIMessage( - content=output, - ) - ) - ] - ], - ) - ) - + await streaming.on_llm_end() return { **function_calling_res, "input": input, diff --git a/libs/superagent/app/utils/callbacks.py b/libs/superagent/app/utils/callbacks.py index d136fee3f..53462096a 100644 --- a/libs/superagent/app/utils/callbacks.py +++ b/libs/superagent/app/utils/callbacks.py @@ -27,7 +27,7 @@ def always_verbose(self) -> bool: return True def __init__(self) -> None: - self.queue = asyncio.Queue() + self.queue = asyncio.Queue(maxsize=5) self.done = asyncio.Event() async def on_chat_model_start( @@ -44,16 +44,16 @@ async def on_llm_start(self) -> None: async def on_llm_new_token(self, token: str, **kwargs: Any) -> None: # noqa if token is not None and token != "": - self.queue.put_nowait(token) + has_put = False + while not has_put: + try: + await self.queue.put(token) + has_put = True + except asyncio.QueueFull: + continue async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: # noqa - # TODO: - # This should be removed when Langchain has merged - # https://github.com/langchain-ai/langchain/pull/9536 - for gen_list in response.generations: - for gen in gen_list: - if gen.message.content != "": - self.done.set() + self.done.set() async def on_llm_error(self, *args: Any, **kwargs: Any) -> None: # noqa self.done.set() @@ -78,8 +78,8 @@ async def aiter(self) -> AsyncIterator[str]: token_or_done = cast(Union[str, Literal[True]], done.pop().result()) - if token_or_done is True and self.queue.empty(): - break + if token_or_done is True: + continue yield token_or_done From f2a5f6cf6c4f6be62dd84a94f2f79be72137e590 Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Wed, 6 Mar 2024 22:21:50 +0400 Subject: [PATCH 6/7] chore: small fix --- libs/superagent/app/utils/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/superagent/app/utils/callbacks.py b/libs/superagent/app/utils/callbacks.py index 53462096a..fdd1e9a15 100644 --- a/libs/superagent/app/utils/callbacks.py +++ b/libs/superagent/app/utils/callbacks.py @@ -52,7 +52,7 @@ async def on_llm_new_token(self, token: str, **kwargs: Any) -> None: # noqa except asyncio.QueueFull: continue - async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: # noqa + async def on_llm_end(self, *args: Any, **kwargs: Any) -> None: # noqa self.done.set() async def on_llm_error(self, *args: Any, **kwargs: Any) -> None: # noqa From ce81fa0a0d762da437f25bb7538584c75cc0e3f0 Mon Sep 17 00:00:00 2001 From: alisalim17 Date: Wed, 6 Mar 2024 22:52:02 +0400 Subject: [PATCH 7/7] fix: small errors --- libs/superagent/app/agents/llm.py | 21 ++++++++++++++++----- libs/superagent/app/utils/callbacks.py | 10 ++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/libs/superagent/app/agents/llm.py b/libs/superagent/app/agents/llm.py index 761d0df21..de436185d 100644 --- a/libs/superagent/app/agents/llm.py +++ b/libs/superagent/app/agents/llm.py @@ -4,6 +4,7 @@ from litellm import acompletion from app.agents.base import AgentBase +from app.utils.callbacks import CustomAsyncIteratorCallbackHandler from app.utils.llm import LLM_REVERSE_MAPPING from app.utils.prisma import prisma from prisma.enums import AgentType, LLMProvider @@ -66,8 +67,9 @@ async def get_agent(self): class CustomAgentExecutor: async def ainvoke(self, input, *_, **kwargs): - function_calling_res = {"output": ""} + function_calling_res = {} + print("agent_config.tools", agent_config, input) if len(agent_config.tools) > 0: function_calling = await FunctionCalling( enable_streaming=False, @@ -111,16 +113,25 @@ async def ainvoke(self, input, *_, **kwargs): output = "" if enable_streaming: - streaming = kwargs["config"]["callbacks"][0] - await streaming.on_llm_start() + streaming_callback = None + for callback in kwargs["config"]["callbacks"]: + if isinstance(callback, CustomAsyncIteratorCallbackHandler): + streaming_callback = callback + + if not streaming_callback: + raise Exception("Streaming Callback not found") + await streaming_callback.on_llm_start() async for chunk in res: token = chunk.choices[0].delta.content if token: output += token - await streaming.on_llm_new_token(token) + await streaming_callback.on_llm_new_token(token) + + streaming_callback.done.set() + else: + output = res.choices[0].message.content - await streaming.on_llm_end() return { **function_calling_res, "input": input, diff --git a/libs/superagent/app/utils/callbacks.py b/libs/superagent/app/utils/callbacks.py index fdd1e9a15..01daf67b3 100644 --- a/libs/superagent/app/utils/callbacks.py +++ b/libs/superagent/app/utils/callbacks.py @@ -52,8 +52,14 @@ async def on_llm_new_token(self, token: str, **kwargs: Any) -> None: # noqa except asyncio.QueueFull: continue - async def on_llm_end(self, *args: Any, **kwargs: Any) -> None: # noqa - self.done.set() + async def on_llm_end(self, response, **kwargs: Any) -> None: # noqa + # TODO: + # This should be removed when Langchain has merged + # https://github.com/langchain-ai/langchain/pull/9536 + for gen_list in response.generations: + for gen in gen_list: + if gen.message.content != "": + self.done.set() async def on_llm_error(self, *args: Any, **kwargs: Any) -> None: # noqa self.done.set()