Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ dependencies = [
"anthropic>=0.62.0",
"litellm>=1.75.3",
"python-dotenv>=1.1.1",
"llm-prices>=1.3.0",
]

[project.optional-dependencies]
Expand Down
91 changes: 86 additions & 5 deletions src/agentlab/llm/tracking.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from functools import cache, partial
from typing import Optional

import llm_prices
import requests

langchain_community = importlib.util.find_spec("langchain_community")
Expand All @@ -21,6 +22,8 @@

TRACKER = threading.local()

# Fallback cache factors, only used for models that llm-prices has no row for. Models in the
# llm-prices table are priced with their own listed cache read and cache write prices.
ANTHROPIC_CACHE_PRICING_FACTOR = {
"cache_read_tokens": 0.1, # Cost for 5 min ephemeral cache. See Pricing Here: https://docs.anthropic.com/en/docs/about-claude/pricing#model-pricing
"cache_write_tokens": 1.25,
Expand Down Expand Up @@ -163,6 +166,14 @@ def get_pricing_anthropic():
return res


def _token_count(obj, name: str) -> int:
"""Returns the integer token count `obj.name`, or 0 when it is missing or not an int."""
value = getattr(obj, name, 0)
if isinstance(value, bool) or not isinstance(value, int):
return 0
return value


def get_pricing_litellm(model_name):
"""Returns a dictionary of model pricing for a LiteLLM model."""
try:
Expand Down Expand Up @@ -232,7 +243,21 @@ def fetch_pricing_information_from_provider(self) -> Optional[dict]:
return pricing_fn()

def set_pricing_attributes(self) -> None:
"""Set the pricing attributes for the model based on the provider."""
"""Set the pricing attributes for the model based on the provider.

OpenAI and Anthropic models are priced from the llm-prices table first, which also
gives the model's own cache read and cache write prices for the effective cost. Models
it has no row for (and every other provider) use the provider pricing below.
"""
self._llm_price = None
if self._pricing_api in ("openai", "anthropic"):
price = llm_prices.lookup_provider(self._pricing_api, self.model_name)
if price is not None and price.input is not None and price.output is not None:
self._llm_price = price
self.input_cost = price.input_per_token
self.output_cost = price.output_per_token
return

model_to_price_dict = self.fetch_pricing_information_from_provider()
model_costs = model_to_price_dict.get(self.model_name) if model_to_price_dict else None
if model_costs:
Expand Down Expand Up @@ -299,6 +324,45 @@ def get_effective_cost(self, response):
)
return 0.0

def _get_effective_cost_from_llm_prices(
self, new_input_tokens, output_tokens, cache_read_tokens, cache_write_5m, cache_write_1h
) -> Optional[float]:
"""Price one response with the model's llm-prices row.

Cache reads and cache writes are billed at the model's listed prices, and the
long-context prices apply above the model's threshold.

Args:
new_input_tokens: Uncached input tokens.
output_tokens: Output tokens, reasoning tokens included.
cache_read_tokens: Input tokens served from the prompt cache.
cache_write_5m: Input tokens written to the cache with a 5-minute lifetime.
cache_write_1h: Input tokens written to the cache with a 1-hour lifetime.

Returns:
Optional[float]: The cost in USD, or None when the model has no llm-prices row or
a price the usage needs is not listed.
"""
price = getattr(self, "_llm_price", None)
counts = (
new_input_tokens,
output_tokens,
cache_read_tokens,
cache_write_5m,
cache_write_1h,
)
if price is None or not all(isinstance(c, int) and not isinstance(c, bool) for c in counts):
return None
usage = llm_prices.Usage(
input=max(new_input_tokens, 0),
output=max(output_tokens, 0),
cache_read=max(cache_read_tokens, 0),
cache_write_5m=max(cache_write_5m, 0),
cache_write_1h=max(cache_write_1h, 0),
)
cost = price.cost_usage(usage)
return None if cost is None else cost.total

def get_effective_cost_from_antrophic_api(self, response) -> float:
"""
Get the effective cost from the Anthropic API response.
Expand All @@ -314,10 +378,22 @@ def get_effective_cost_from_antrophic_api(self, response) -> float:
float: The effective cost calculated from the response.
"""
usage = getattr(response, "usage", {})
new_input_tokens = getattr(usage, "input_tokens", 0) # new input tokens
output_tokens = getattr(usage, "output_tokens", 0)
cache_read_tokens = getattr(usage, "cache_input_tokens", 0)
cache_write_tokens = getattr(usage, "cache_creation_input_tokens", 0)
new_input_tokens = _token_count(usage, "input_tokens") # new input tokens
output_tokens = _token_count(usage, "output_tokens")
cache_read_tokens = _token_count(usage, "cache_read_input_tokens")
cache_write_tokens = _token_count(usage, "cache_creation_input_tokens")
# Newer SDKs split cache writes by lifetime; the total alone means 5-minute writes.
cache_creation = getattr(usage, "cache_creation", None)
cache_write_1h = min(
_token_count(cache_creation, "ephemeral_1h_input_tokens"), cache_write_tokens
)
cache_write_5m = cache_write_tokens - cache_write_1h

effective_cost = self._get_effective_cost_from_llm_prices(
new_input_tokens, output_tokens, cache_read_tokens, cache_write_5m, cache_write_1h
)
if effective_cost is not None:
return effective_cost

cache_read_cost = self.input_cost * ANTHROPIC_CACHE_PRICING_FACTOR["cache_read_tokens"]
cache_write_cost = self.input_cost * ANTHROPIC_CACHE_PRICING_FACTOR["cache_write_tokens"]
Expand Down Expand Up @@ -373,6 +449,11 @@ def get_effective_cost_from_openai_api(self, response) -> float:
else:
logging.warning(f"Unsupported API type: {api_type}. Defaulting cost to 0.0.")
return 0.0
effective_cost = self._get_effective_cost_from_llm_prices(
new_input_tokens, output_tokens, cached_input_tokens, 0, 0
)
if effective_cost is not None:
return effective_cost
cache_read_cost = self.input_cost * OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"]
effective_cost = (
self.input_cost * new_input_tokens
Expand Down
2 changes: 1 addition & 1 deletion tests/llm/test_response_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ def create_mock_anthropic_response(
response.usage = MagicMock()
response.usage.input_tokens = input_tokens
response.usage.output_tokens = output_tokens
response.usage.cache_input_tokens = 0
response.usage.cache_read_input_tokens = 0
response.usage.cache_creation_input_tokens = 0
return response

Expand Down
134 changes: 134 additions & 0 deletions tests/llm/test_tracking.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
import os
import time
from functools import partial
from types import SimpleNamespace
from unittest.mock import patch

import pytest
from anthropic.types import Usage as AnthropicUsage
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
from openai.types.responses.response_usage import (
InputTokensDetails,
OutputTokensDetails,
ResponseUsage,
)

import agentlab.llm.tracking as tracking
from agentlab.llm.chat_api import (
Expand All @@ -12,6 +21,11 @@
make_system_message,
make_user_message,
)
from agentlab.llm.response_api import (
ClaudeResponseModelArgs,
OpenAIChatModelArgs,
OpenAIResponseModelArgs,
)


def test_get_action_decorator():
Expand Down Expand Up @@ -54,6 +68,126 @@ def test_get_pricing_openai():
assert all(isinstance(pricing[model]["completion"], float) for model in pricing)


# List prices in USD per 1M tokens, copied by hand from the providers' pricing pages
# (https://platform.openai.com/docs/pricing, https://docs.anthropic.com/en/docs/about-claude/pricing)
# so that the effective costs below are checked against the published numbers.
GPT_5 = {"input": 1.25, "cached_input": 0.125, "output": 10.0}
CLAUDE_OPUS_5_5 = {"input": 4.0, "cache_read": 0.20, "cache_write_5m": 5.0, "output": 20.0}
CLAUDE_SONNET_4_6 = {
"input": 3.0,
"cache_read": 0.30,
"cache_write_5m": 3.75,
"cache_write_1h": 6.0,
"output": 15.0,
}


def _make_openai_model(args_cls, model_name):
with patch("agentlab.llm.response_api.OpenAI"):
return args_cls(model_name=model_name).make_model()


def _make_claude_model(model_name):
with patch("agentlab.llm.response_api.Anthropic"):
return ClaudeResponseModelArgs(model_name=model_name).make_model()


def test_openai_chat_completion_effective_cost_gpt5_cached_tokens():
model = _make_openai_model(OpenAIChatModelArgs, "gpt-5")
assert model.input_cost == pytest.approx(GPT_5["input"] / 1e6)
assert model.output_cost == pytest.approx(GPT_5["output"] / 1e6)

# 100k prompt tokens, 80k of them served from the prompt cache
usage = CompletionUsage(
prompt_tokens=100_000,
completion_tokens=2_000,
total_tokens=102_000,
prompt_tokens_details=PromptTokensDetails(cached_tokens=80_000),
)
response = SimpleNamespace(usage=usage)
expected = (
20_000 * GPT_5["input"] + 80_000 * GPT_5["cached_input"] + 2_000 * GPT_5["output"]
) / 1e6
assert model.get_effective_cost(response) == pytest.approx(expected) # $0.055
# the fixed 0.5 cache factor priced the same call at $0.095
assert expected == pytest.approx(0.055)


def test_openai_responses_api_effective_cost_gpt5_cached_tokens():
model = _make_openai_model(OpenAIResponseModelArgs, "gpt-5")
usage = ResponseUsage(
input_tokens=100_000,
input_tokens_details=InputTokensDetails(cached_tokens=80_000),
output_tokens=2_000,
output_tokens_details=OutputTokensDetails(reasoning_tokens=1_500),
total_tokens=102_000,
)
response = SimpleNamespace(usage=usage)
expected = (
20_000 * GPT_5["input"] + 80_000 * GPT_5["cached_input"] + 2_000 * GPT_5["output"]
) / 1e6
assert model.get_effective_cost(response) == pytest.approx(expected)


def test_anthropic_effective_cost_claude_opus_5_5_cache():
model = _make_claude_model("claude-opus-5-5")
assert model.input_cost == pytest.approx(CLAUDE_OPUS_5_5["input"] / 1e6)
assert model.output_cost == pytest.approx(CLAUDE_OPUS_5_5["output"] / 1e6)

usage = AnthropicUsage(
input_tokens=1_000,
cache_read_input_tokens=50_000,
cache_creation_input_tokens=4_000,
output_tokens=1_500,
)
response = SimpleNamespace(usage=usage)
expected = (
1_000 * CLAUDE_OPUS_5_5["input"]
+ 50_000 * CLAUDE_OPUS_5_5["cache_read"]
+ 4_000 * CLAUDE_OPUS_5_5["cache_write_5m"]
+ 1_500 * CLAUDE_OPUS_5_5["output"]
) / 1e6
assert model.get_effective_cost(response) == pytest.approx(expected) # $0.064


def test_anthropic_effective_cost_one_hour_cache_writes():
model = _make_claude_model("claude-sonnet-4-6")
# newer anthropic SDKs split cache writes by lifetime
usage = SimpleNamespace(
input_tokens=500,
cache_read_input_tokens=10_000,
cache_creation_input_tokens=3_000,
cache_creation=SimpleNamespace(
ephemeral_5m_input_tokens=1_000, ephemeral_1h_input_tokens=2_000
),
output_tokens=800,
)
expected = (
500 * CLAUDE_SONNET_4_6["input"]
+ 10_000 * CLAUDE_SONNET_4_6["cache_read"]
+ 1_000 * CLAUDE_SONNET_4_6["cache_write_5m"]
+ 2_000 * CLAUDE_SONNET_4_6["cache_write_1h"]
+ 800 * CLAUDE_SONNET_4_6["output"]
) / 1e6
assert model.get_effective_cost(SimpleNamespace(usage=usage)) == pytest.approx(expected)


def test_effective_cost_falls_back_for_models_without_llm_prices_row():
with patch("agentlab.llm.response_api.OpenAI"):
model = OpenAIChatModelArgs(model_name="my-org/my-finetuned-model").make_model()
assert model._llm_price is None
model.input_cost, model.output_cost = 1e-6, 2e-6
usage = CompletionUsage(
prompt_tokens=1_000,
completion_tokens=100,
total_tokens=1_100,
prompt_tokens_details=PromptTokensDetails(cached_tokens=400),
)
factor = tracking.OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"]
expected = 600 * 1e-6 + 400 * 1e-6 * factor + 100 * 2e-6
assert model.get_effective_cost(SimpleNamespace(usage=usage)) == pytest.approx(expected)


def call_llm():
if hasattr(tracking.TRACKER, "instance") and isinstance(
tracking.TRACKER.instance, tracking.LLMTracker
Expand Down
11 changes: 11 additions & 0 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.