diff --git a/README.md b/README.md index 06adfba..10973ac 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Visit the Github: [![FastAPI MCP LangGraph Template](https://img.shields.io/gith - [![FastAPI](https://img.shields.io/github/stars/fastapi/fastapi?logo=fastapi&label=fastapi)](https://github.com/fastapi/fastapi) for Python backend API - [![SQLModel](https://img.shields.io/github/stars/fastapi/sqlmodel?logo=sqlmodel&label=SQLModel)](https://github.com/fastapi/sqlmodel) for Python SQL database interactions (ORM + Validation). - Wrapper of [![SQLAlchemy](https://img.shields.io/github/stars/sqlalchemy/sqlalchemy?logo=sqlalchemy&label=SQLAlchemy)](https://github.com/sqlalchemy/sqlalchemy) +- [![LangFuse](https://img.shields.io/github/stars/langfuse/langfuse?logo=langfuse&label=LangFuse)](https://github.com/langfuse/langfuse) for LLM Observability and LLM Metrics - [![Pydantic](https://img.shields.io/github/stars/pydantic/pydantic?logo=pydantic&label=Pydantic)](https://github.com/pydantic/pydantic) for Data Validation and Settings Management. - [![Supabase](https://img.shields.io/github/stars/supabase/supabase?logo=supabase&label=Supabase)](https://github.com/supabase/supabase) for DB RBAC - [![PostgreSQL](https://img.shields.io/github/stars/postgres/postgres?logo=postgresql&label=Postgres)](https://github.com/postgres/postgres) Relational DB @@ -56,7 +57,6 @@ Visit the Github: [![FastAPI MCP LangGraph Template](https://img.shields.io/gith ### Planned Features -- [![LangFuse](https://img.shields.io/github/stars/langfuse/langfuse?logo=langfuse&label=LangFuse)](https://github.com/langfuse/langfuse) for LLM Observability and LLM Metrics - [![Prometheus](https://img.shields.io/github/stars/prometheus/prometheus?logo=prometheus&label=Prometheus)](https://github.com/prometheus/prometheus) for scraping Metrics - [![Grafana](https://img.shields.io/github/stars/prometheus/prometheus?logo=grafana&label=Grafana)](https://github.com/grafana/grafana) for visualizing Metrics - [![Auth0](https://img.shields.io/badge/Auth0-white?logo=auth0)](https://auth0.com/docs) SaaS for Authentication and Authorization with OIDC & JWT via OAuth 2.0 @@ -199,6 +199,13 @@ Add your following API keys and value to the respective file: `./envs/backend.en ```bash OPENAI_API_KEY=sk-proj-... POSTGRES_DSN=postgresql://postgres... + +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +LANGFUSE_HOST=https://cloud.langfuse.com + +ENVIRONMENT=production + YOUTUBE_API_KEY=... ``` diff --git a/backend/api/core/agent/orchestration.py b/backend/api/core/agent/orchestration.py index 84a9b5a..dedd7b4 100644 --- a/backend/api/core/agent/orchestration.py +++ b/backend/api/core/agent/orchestration.py @@ -10,6 +10,7 @@ from langgraph.prebuilt import ToolNode, tools_condition from api.core.agent.prompts import SYSTEM_PROMPT +from api.core.dependencies import LangfuseHandlerDep class State(MessagesState): @@ -70,7 +71,8 @@ def get_graph( return graph_factory(worker_node, tools, checkpointer, name) -def get_config(): +def get_config(langfuse_handler: LangfuseHandlerDep): return dict( configurable=dict(thread_id="1"), + callbacks=[langfuse_handler], ) diff --git a/backend/api/core/config.py b/backend/api/core/config.py index 2ba0ca3..ef07752 100644 --- a/backend/api/core/config.py +++ b/backend/api/core/config.py @@ -33,5 +33,11 @@ def checkpoint_conn_str(self) -> str: # with specifying psycopg driver explicitly return self.postgres_dsn.encoded_string() + langfuse_public_key: str = "" + langfuse_secret_key: str = "" + langfuse_host: str = "https://cloud.langfuse.com" + + environment: str = "development" + settings = Settings() diff --git a/backend/api/core/dependencies.py b/backend/api/core/dependencies.py index 20218e7..eddaecc 100644 --- a/backend/api/core/dependencies.py +++ b/backend/api/core/dependencies.py @@ -4,6 +4,7 @@ from fastapi import Depends from langchain_mcp_adapters.tools import load_mcp_tools from langchain_openai import ChatOpenAI +from langfuse.callback import CallbackHandler from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from api.core.agent.persistence import checkpointer_context @@ -47,3 +48,17 @@ async def setup_graph() -> AsyncGenerator[Resource]: tools=tools, session=session, ) + + +def get_langfuse_handler() -> CallbackHandler: + + return CallbackHandler( + public_key=settings.langfuse_public_key, + secret_key=settings.langfuse_secret_key, + host=settings.langfuse_host, + session_id=settings.environment, + environment=settings.environment, + ) + + +LangfuseHandlerDep = Annotated[CallbackHandler, Depends(get_langfuse_handler)] diff --git a/backend/api/routers/llms.py b/backend/api/routers/llms.py index b7035a8..ffba42c 100644 --- a/backend/api/routers/llms.py +++ b/backend/api/routers/llms.py @@ -8,7 +8,7 @@ from starlette.responses import Response from api.core.agent.orchestration import get_config, get_graph -from api.core.dependencies import LLMDep, setup_graph +from api.core.dependencies import LangfuseHandlerDep, LLMDep, setup_graph from api.core.logs import print, uvicorn router = APIRouter(tags=["chat"]) @@ -26,13 +26,17 @@ async def completions(query: str, llm: LLMDep) -> Response: @router.get("/chat/agent") -async def agent(query: str, llm: LLMDep) -> Response: +async def agent( + query: str, + llm: LLMDep, + langfuse_handler: LangfuseHandlerDep, +) -> Response: """Stream LangGraph completions as Server-Sent Events (SSE). This endpoint streams LangGraph-generated events in real-time, allowing the client to receive responses as they are processed, useful for agent-based workflows. """ - return EventSourceResponse(stream_graph(query, llm)) + return EventSourceResponse(stream_graph(query, llm, langfuse_handler)) async def stream_completions( @@ -57,17 +61,23 @@ async def checkpointer_setup(pool): async def stream_graph( query: str, llm: LLMDep, + langfuse_handler: LangfuseHandlerDep, ) -> AsyncGenerator[dict[str, str], None]: async with setup_graph() as resource: graph = get_graph( llm, tools=resource.tools, checkpointer=resource.checkpointer, + name="math_agent", ) - config = get_config() + config = get_config(langfuse_handler) events = dict(messages=[HumanMessage(content=query)]) - async for event in graph.astream_events(events, config, version="v2"): - if event.get("event").endswith("end"): - print(event) + async for event in graph.astream_events( + events, + config, + version="v2", + stream_mode="updates", + ): yield dict(data=event) + print(event) diff --git a/docs/langfuse.md b/docs/langfuse.md index 9b42905..6280766 100644 --- a/docs/langfuse.md +++ b/docs/langfuse.md @@ -5,3 +5,41 @@ Learn more [here](https://langfuse.com/) [![LangFuse](https://img.shields.io/github/stars/langfuse/langfuse?logo=langfuse&label=LangFuse)](https://github.com/langfuse/langfuse) + +Below is an outline of the steps involved in a simple Math Agent. Key elements illustrated include: + +- A visual breakdown of each step—e.g., when the agent invokes a tool and when control returns to the agent +- Inputs and outputs at each stage of the process: + - **User to Agent**: The user asks a natural language question — e.g., `What is 1 + 1?` + - **Agent to Tool**: The agent decides to call a calculator tool with structured arguments — e.g., `args: { a: 1, b: 1 }`. + - **Tool to Agent**: The tool executes the operation and returns the result — e.g., `2`. + - **Agent to User**: The agent responds with the final answer in natural language — e.g., `1 + 1 = 2`. +- The full chat history throughout the interaction +- Latency and cost associated with each node + +### Step 1: Math Agent (User to Agent → Agent to Tool) + +This section shows: + +- **User to Agent**: The user asks a natural language question — e.g., `What is 1 + 1?` +- **Agent to Tool**: The agent decides to call a calculator tool with structured arguments — e.g., `args: { a: 1, b: 1 }`. +- **Full Chat History Throughout the Interaction**: You can inspect earlier user-agent messages. For instance: + +```text +User: reply only no +Agent: No. +``` + +In this example, the agent responded directly without calling any tools. + +### Step 2: Tool Call (Tool to Agent) + +This section shows: + + - **Tool to Agent**: The tool executes the operation and returns the result — e.g., `2`. + +### Step 3: Math Agent (Agent to User) + +This section shows: + + - **Agent to User**: The agent responds with the final answer in natural language — e.g., `1 + 1 = 2`. diff --git a/envs/backend.env b/envs/backend.env index 4fa624a..c6edf59 100644 --- a/envs/backend.env +++ b/envs/backend.env @@ -1,3 +1,9 @@ OPENAI_API_KEY= # do not specify driver (do not specify `+psycopg`) POSTGRES_DSN= + +LANGFUSE_PUBLIC_KEY=pk-lf- +LANGFUSE_SECRET_KEY=sk-lf- +LANGFUSE_HOST=https://cloud.langfuse.com + +ENVIRONMENT=production diff --git a/nginx/nginx.conf b/nginx/nginx.conf index a75d671..1dfb4e9 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -23,6 +23,9 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_set_header Host $host; + proxy_read_timeout 600; + proxy_send_timeout 600; + proxy_buffering off; proxy_redirect off; proxy_pass http://api/;