diff --git a/fern/mdx/deploy/local.mdx b/fern/mdx/deploy/local.mdx index 21062b7b7..c8c2a866b 100644 --- a/fern/mdx/deploy/local.mdx +++ b/fern/mdx/deploy/local.mdx @@ -41,6 +41,11 @@ MEMORY_API_URL=https://memory.superagent.sh # OPTIONAL VARIABLES +# AgentOps +# Automatically track costs, session replays, and agent benchmarks. Sign up for a key at [agentops.ai](https://agentops.ai). # Using the default org key in .env.example will include added premium features. +AGENTOPS_API_KEY= +AGENTOPS_ORG_KEY= + # Langfuse observability LANGFUSE_PUBLIC_KEY= LANGFUSE_SECRET_KEY= diff --git a/libs/superagent/.env.example b/libs/superagent/.env.example index d779a6b12..2f3ce4689 100644 --- a/libs/superagent/.env.example +++ b/libs/superagent/.env.example @@ -33,6 +33,9 @@ ASTRA_DB_KEYSPACE_NAME= E2B_API_KEY=e2b_21b611cdf96fad06a6a819708734be20cfe8b777 # Replicate LLM/tool REPLICATE_API_TOKEN= +# AgentOps session tracking +AGENTOPS_API_KEY= +AGENTOPS_ORG_KEY=843bf677-e691-45ad-97cf-909e99f9ad83 # Langfuse tracing LANGFUSE_PUBLIC_KEY= LANGFUSE_SECRET_KEY= @@ -42,7 +45,11 @@ LANGCHAIN_TRACING_V2=False LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" LANGCHAIN_API_KEY= LANGSMITH_PROJECT_ID= +# Agentops tracking +AGENTOPS_API_KEY= +AGENTOPS_ORG_KEY= # Finetunes LAMINI_API_KEY= # Tracking SEGMENT_WRITE_KEY= + diff --git a/libs/superagent/app/agents/base.py b/libs/superagent/app/agents/base.py index 552ab281f..03e1cb37b 100644 --- a/libs/superagent/app/agents/base.py +++ b/libs/superagent/app/agents/base.py @@ -1,5 +1,10 @@ from typing import Any, List, Optional +from agentops.langchain_callback_handler import ( + AsyncCallbackHandler, + LangchainCallbackHandler, +) + from app.models.request import LLMParams from app.utils.streaming import CustomAsyncIteratorCallbackHandler from prisma.models import Agent, AgentDatasource, AgentLLM, AgentTool @@ -18,9 +23,11 @@ def __init__( enable_streaming: bool = False, output_schema: str = None, callback: CustomAsyncIteratorCallbackHandler = None, + session_tracker: LangchainCallbackHandler | AsyncCallbackHandler = None, llm_params: Optional[LLMParams] = {}, agent_config: Agent = None, ): + self.session_tracker = session_tracker self.agent_id = agent_id self.session_id = session_id self.enable_streaming = enable_streaming @@ -53,6 +60,7 @@ async def get_agent(self): enable_streaming=self.enable_streaming, output_schema=self.output_schema, callback=self.callback, + session_tracker=self.session_tracker, llm_params=self.llm_params, ) else: @@ -64,6 +72,7 @@ async def get_agent(self): enable_streaming=self.enable_streaming, output_schema=self.output_schema, callback=self.callback, + session_tracker=self.session_tracker, ) return await agent.get_agent(config=self.agent_config) diff --git a/libs/superagent/app/agents/langchain.py b/libs/superagent/app/agents/langchain.py index ffefab759..6b813c63d 100644 --- a/libs/superagent/app/agents/langchain.py +++ b/libs/superagent/app/agents/langchain.py @@ -111,12 +111,18 @@ async def _get_llm(self, agent_llm: AgentLLM, model: str) -> Any: **(self.llm_params.dict() if self.llm_params else {}), } + callbacks = [] + if self.enable_streaming: + callbacks.append(self.callback) + if self.session_tracker: + callbacks.append(self.session_tracker) + if agent_llm.llm.provider == "OPENAI": return ChatOpenAI( model=LLM_MAPPING[model], openai_api_key=agent_llm.llm.apiKey, streaming=self.enable_streaming, - callbacks=[self.callback] if self.enable_streaming else [], + callbacks=callbacks, **(agent_llm.llm.options if agent_llm.llm.options else {}), **(llm_params), ) @@ -124,7 +130,7 @@ async def _get_llm(self, agent_llm: AgentLLM, model: str) -> Any: return AzureChatOpenAI( api_key=agent_llm.llm.apiKey, streaming=self.enable_streaming, - callbacks=[self.callback] if self.enable_streaming else [], + callbacks=callbacks, **(agent_llm.llm.options if agent_llm.llm.options else {}), **(llm_params), ) diff --git a/libs/superagent/app/api/agents.py b/libs/superagent/app/api/agents.py index f2c4f8086..e47e9b931 100644 --- a/libs/superagent/app/api/agents.py +++ b/libs/superagent/app/api/agents.py @@ -4,6 +4,7 @@ from typing import AsyncIterable import segment.analytics as analytics +from agentops.langchain_callback_handler import AsyncLangchainCallbackHandler from decouple import config from fastapi import APIRouter, Depends from fastapi.responses import JSONResponse, StreamingResponse @@ -146,7 +147,6 @@ async def get(agent_id: str, api_user=Depends(get_current_api_user)): "llms": {"include": {"llm": True}}, }, ) - print(data) for llm in data.llms: llm.llm.options = json.dumps(llm.llm.options) for tool in data.tools: @@ -233,6 +233,17 @@ async def invoke( ) langfuse_handler = trace.get_langchain_handler() + agentops_api_key = config("AGENTOPS_API_KEY", default=None) + agentops_org_key = config("AGENTOPS_ORG_KEY", default=None) + + agentops_handler = None + if agentops_api_key and agentops_org_key: + agentops_handler = AsyncLangchainCallbackHandler( + api_key=agentops_api_key, + org_key=agentops_org_key, + tags=[agent_id, str(body.sessionId)], + ) + agent_config = await prisma.agent.find_unique_or_raise( where={"id": agent_id}, include={ @@ -281,6 +292,9 @@ def track_agent_invocation(result): if langfuse_handler: agentCallbacks.append(langfuse_handler) + if agentops_handler: + agentCallbacks.append(agentops_handler) + async def send_message( agent: LLMChain | AgentExecutor, content: str, @@ -337,6 +351,7 @@ async def send_message( enable_streaming=enable_streaming, output_schema=output_schema, callback=callback, + session_tracker=agentops_handler, llm_params=body.llm_params, agent_config=agent_config, ).get_agent() @@ -348,6 +363,7 @@ async def send_message( return StreamingResponse(generator, media_type="text/event-stream") logging.info("Streaming not enabled. Invoking agent synchronously...") + output = await agent.acall( inputs={"input": input}, tags=[agent_id], callbacks=agentCallbacks ) diff --git a/libs/superagent/app/api/workflows.py b/libs/superagent/app/api/workflows.py index eff8d978b..94041b44e 100644 --- a/libs/superagent/app/api/workflows.py +++ b/libs/superagent/app/api/workflows.py @@ -4,6 +4,7 @@ from typing import AsyncIterable import segment.analytics as analytics +from agentops.langchain_callback_handler import AsyncLangchainCallbackHandler from decouple import config from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse @@ -174,10 +175,18 @@ async def invoke( input = body.input enable_streaming = body.enableStreaming + agentops_api_key = config("AGENTOPS_API_KEY") + agentops_org_key = config("AGENTOPS_ORG_KEY") + + agentops_handler = AsyncLangchainCallbackHandler( + api_key=agentops_api_key, org_key=agentops_org_key, tags=[session_id] + ) + workflow = WorkflowBase( workflow=workflowData, enable_streaming=enable_streaming, callbacks=[workflowStep["callback"] for workflowStep in workflowSteps], + session_tracker=agentops_handler, session_id=session_id, ) @@ -221,6 +230,10 @@ async def send_message() -> AsyncIterable[str]: input, ) + # End session + agentops_handler.ao_client.end_session( + "Success", end_state_reason="Workflow completed" + ) return {"success": True, "data": output} diff --git a/libs/superagent/app/workflows/base.py b/libs/superagent/app/workflows/base.py index bdaeb4976..7c12370bb 100644 --- a/libs/superagent/app/workflows/base.py +++ b/libs/superagent/app/workflows/base.py @@ -1,6 +1,12 @@ import asyncio from typing import Any, List +from agentops.langchain_callback_handler import ( + AsyncCallbackHandler, + LangchainCallbackHandler, +) +from decouple import config + from app.agents.base import AgentBase from app.utils.prisma import prisma from app.utils.streaming import CustomAsyncIteratorCallbackHandler @@ -13,11 +19,13 @@ def __init__( workflow: Workflow, callbacks: List[CustomAsyncIteratorCallbackHandler], session_id: str, + session_tracker: LangchainCallbackHandler | AsyncCallbackHandler = None, enable_streaming: bool = False, ): self.workflow = workflow self.enable_streaming = enable_streaming self.session_id = session_id + self.session_tracker = session_tracker self.callbacks = callbacks async def arun(self, input: Any): @@ -41,6 +49,7 @@ async def arun(self, input: Any): agent_id=step.agentId, enable_streaming=True, callback=self.callbacks[stepIndex], + session_tracker=self.session_tracker, session_id=self.session_id, agent_config=agent_config, ).get_agent() diff --git a/libs/superagent/poetry.lock b/libs/superagent/poetry.lock index bf699f106..74fc24657 100644 --- a/libs/superagent/poetry.lock +++ b/libs/superagent/poetry.lock @@ -12,6 +12,27 @@ files = [ {file = "aenum-3.1.15.tar.gz", hash = "sha256:8cbd76cd18c4f870ff39b24284d3ea028fbe8731a58df3aa581e434c575b9559"}, ] +[[package]] +name = "agentops" +version = "0.0.20" +description = "Python SDK for developing AI agent evals and observability" +optional = false +python-versions = ">=3.7" +files = [ + {file = "agentops-0.0.20-py3-none-any.whl", hash = "sha256:ba20fc48902434858f28e3c4a7febe56d275a28bd33378868e7fcde2f53f2430"}, + {file = "agentops-0.0.20.tar.gz", hash = "sha256:d50623b03b410c8c88718c29ea271304681e1305b5c05ba824edb92d18aab4f8"}, +] + +[package.dependencies] +langchain = {version = ">=0.0.354", optional = true, markers = "extra == \"langchain\""} +packaging = ">=23.1,<24.0" +pydantic = ">=1.9.0" +requests = "2.31.0" + +[package.extras] +dev = ["pytest (==7.4.0)", "requests-mock (==1.11.0)"] +langchain = ["langchain (>=0.0.354)"] + [[package]] name = "aiohttp" version = "3.9.1" @@ -4261,24 +4282,24 @@ python-versions = ">=3.6" files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d92f81886165cb14d7b067ef37e142256f1c6a90a65cd156b063a43da1708cfd"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, - {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:b5edda50e5e9e15e54a6a8a0070302b00c518a9d32accc2346ad6c984aacd279"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, - {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:7048c338b6c86627afb27faecf418768acb6331fc24cfa56c93e8c9780f815fa"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, - {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, @@ -4286,7 +4307,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, - {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3fcc54cb0c8b811ff66082de1680b4b14cf8a81dce0d4fbf665c2265a81e07a1"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, @@ -4294,7 +4315,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, - {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:665f58bfd29b167039f714c6998178d27ccd83984084c286110ef26b230f259f"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, @@ -4302,7 +4323,7 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, - {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:9eb5dee2772b0f704ca2e45b1713e4e5198c18f515b52743576d196348f374d3"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, @@ -5733,4 +5754,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8.1, <3.12" -content-hash = "c138986962a0cf1e696eb298c0a10d61b07a613eb883bdc0132615d3603b0a53" +content-hash = "0cf9c77ee7490127838309bbea18c2dfa70288bbd42298914cd6a8dd26504591" diff --git a/libs/superagent/pyproject.toml b/libs/superagent/pyproject.toml index 25ccd8c18..dde2f6cb9 100644 --- a/libs/superagent/pyproject.toml +++ b/libs/superagent/pyproject.toml @@ -59,6 +59,7 @@ weaviate-client = "^3.25.3" qdrant-client = "^1.6.9" langfuse = "^2.6.3" vecs = "^0.4.2" +agentops = {extras = ["langchain"], version = "^0.0.20"} [build-system]