Implement LLM-WP-0007: Kimi K3 default and EUR spend reporting
Add moonshotai/kimi-k3 as OpenRouter basemodel default after live smoke, USD→EUR cost conversion, append-only usage ledger, and CLI run/cost/spend week commands with token and euro reporting.
This commit is contained in:
parent
f3121c3f1f
commit
09aa1f3604
21 changed files with 1396 additions and 103 deletions
|
|
@ -1,8 +1,10 @@
|
|||
import json
|
||||
from datetime import datetime, timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from llm_connect.cli import main
|
||||
from llm_connect.quality import QualityLedger, QualityObservation
|
||||
from llm_connect.usage import UsageEvent, UsageLedger
|
||||
|
||||
|
||||
def test_rates_show_json_outputs_default_registry(capsys):
|
||||
|
|
@ -11,6 +13,7 @@ def test_rates_show_json_outputs_default_registry(capsys):
|
|||
payload = json.loads(capsys.readouterr().out)
|
||||
|
||||
assert payload["openai/gpt-4o-mini"]["prompt_per_1k"] == 0.00015
|
||||
assert payload["moonshotai/kimi-k3"]["prompt_per_1k"] == 0.003
|
||||
|
||||
|
||||
def test_classes_show_lists_builtins(capsys):
|
||||
|
|
@ -52,3 +55,116 @@ def test_classes_fit_reads_quality_ledger(tmp_path, capsys):
|
|||
payload = json.loads(capsys.readouterr().out)
|
||||
|
||||
assert payload["entity-extraction"]["params"]["tokens_per_entity"] == 70
|
||||
|
||||
|
||||
def test_run_with_mock_reports_cost_and_writes_ledger(tmp_path, capsys):
|
||||
ledger = tmp_path / "usage.jsonl"
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"run",
|
||||
"hello world",
|
||||
"--provider",
|
||||
"mock",
|
||||
"--model",
|
||||
"moonshotai/kimi-k3",
|
||||
"--ledger",
|
||||
str(ledger),
|
||||
"--eur-per-usd",
|
||||
"1.0",
|
||||
"--json",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert payload["content"]
|
||||
assert payload["usage"]["total_tokens"] > 0
|
||||
assert payload["cost_usd"] is not None
|
||||
assert payload["cost_eur"] is not None
|
||||
events = UsageLedger(ledger).read_all()
|
||||
assert len(events) == 1
|
||||
assert events[0].source == "cli"
|
||||
assert events[0].model_id == "moonshotai/kimi-k3"
|
||||
|
||||
|
||||
def test_cost_estimate_cli(capsys):
|
||||
assert (
|
||||
main(
|
||||
[
|
||||
"cost",
|
||||
"estimate",
|
||||
"--model",
|
||||
"moonshotai/kimi-k3",
|
||||
"--prompt-tokens",
|
||||
"1000",
|
||||
"--completion-tokens",
|
||||
"1000",
|
||||
"--eur-per-usd",
|
||||
"1",
|
||||
"--json",
|
||||
]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert payload["cost_usd"] == 0.018
|
||||
assert payload["cost_eur"] == 0.018
|
||||
|
||||
|
||||
def test_spend_week_current_and_last(tmp_path, capsys, monkeypatch):
|
||||
ledger_path = tmp_path / "usage.jsonl"
|
||||
ledger = UsageLedger(ledger_path)
|
||||
berlin = ZoneInfo("Europe/Berlin")
|
||||
current_start = datetime(2026, 8, 3, 0, 0, tzinfo=berlin)
|
||||
current_end = datetime(2026, 8, 5, 12, 0, tzinfo=berlin)
|
||||
last_start = datetime(2026, 7, 27, 0, 0, tzinfo=berlin)
|
||||
last_end = current_start
|
||||
|
||||
def _fake_week_window(which="current", *, now=None, tz=None):
|
||||
if which == "last":
|
||||
return last_start, last_end
|
||||
return current_start, current_end
|
||||
|
||||
monkeypatch.setattr("llm_connect.cli.week_window", _fake_week_window)
|
||||
|
||||
ledger.append(
|
||||
UsageEvent(
|
||||
provider="mock",
|
||||
model_id="moonshotai/kimi-k3",
|
||||
prompt_tokens=100,
|
||||
completion_tokens=20,
|
||||
total_tokens=120,
|
||||
cost_usd=0.01,
|
||||
cost_eur=0.009,
|
||||
cost_source="rate_table:moonshotai/kimi-k3",
|
||||
source="cli",
|
||||
recorded_at=datetime(2026, 8, 4, 9, 0, tzinfo=berlin),
|
||||
)
|
||||
)
|
||||
ledger.append(
|
||||
UsageEvent(
|
||||
provider="mock",
|
||||
model_id="moonshotai/kimi-k3",
|
||||
prompt_tokens=50,
|
||||
completion_tokens=10,
|
||||
total_tokens=60,
|
||||
cost_usd=0.005,
|
||||
cost_eur=0.0045,
|
||||
cost_source="rate_table:moonshotai/kimi-k3",
|
||||
source="cli",
|
||||
recorded_at=datetime(2026, 7, 29, 9, 0, tzinfo=berlin), # previous week
|
||||
)
|
||||
)
|
||||
|
||||
assert main(["spend", "week", "--ledger", str(ledger_path), "--json"]) == 0
|
||||
current = json.loads(capsys.readouterr().out)
|
||||
assert current["event_count"] == 1
|
||||
assert current["total_tokens"] == 120
|
||||
assert current["cost_eur"] == 0.009
|
||||
|
||||
assert main(["spend", "week", "--last", "--ledger", str(ledger_path), "--json"]) == 0
|
||||
last = json.loads(capsys.readouterr().out)
|
||||
assert last["event_count"] == 1
|
||||
assert last["total_tokens"] == 60
|
||||
|
|
|
|||
|
|
@ -10,12 +10,16 @@ def test_known_model_cost_matches_lefevre_smoke_budget():
|
|||
assert estimate.cost_source == "rate_table:openai/gpt-4o-mini"
|
||||
assert estimate.cost_usd == pytest.approx(0.0087)
|
||||
assert estimate.cost_usd == pytest.approx(0.009, rel=0.2)
|
||||
assert estimate.cost_eur is not None
|
||||
assert estimate.fx_source is not None
|
||||
|
||||
|
||||
def test_unknown_model_returns_unknown_without_zeroing_cost():
|
||||
estimate = estimate_cost("unknown/model", 100, 50)
|
||||
|
||||
assert estimate == CostEstimate(cost_usd=None, cost_source="unknown")
|
||||
assert estimate.cost_usd is None
|
||||
assert estimate.cost_eur is None
|
||||
assert estimate.cost_source == "unknown"
|
||||
|
||||
|
||||
def test_registry_override_controls_estimate():
|
||||
|
|
@ -29,21 +33,40 @@ def test_registry_override_controls_estimate():
|
|||
}
|
||||
)
|
||||
|
||||
estimate = estimate_cost("vendor/model", 1_000, 500, registry=registry)
|
||||
estimate = estimate_cost("vendor/model", 1_000, 500, registry=registry, fx=1.0)
|
||||
|
||||
assert estimate.cost_usd == pytest.approx(2.0)
|
||||
assert estimate.prompt_cost_usd == pytest.approx(1.0)
|
||||
assert estimate.completion_cost_usd == pytest.approx(1.0)
|
||||
assert estimate.cost_eur == pytest.approx(2.0)
|
||||
|
||||
|
||||
def test_zero_tokens_are_valid_and_cost_zero_for_known_model():
|
||||
estimate = CostModel().estimate_cost("openai/gpt-4o-mini", 0, 0)
|
||||
estimate = CostModel(fx=1.0).estimate_cost("openai/gpt-4o-mini", 0, 0)
|
||||
|
||||
assert estimate.cost_usd == 0
|
||||
assert estimate.prompt_cost_usd == 0
|
||||
assert estimate.completion_cost_usd == 0
|
||||
assert estimate.cost_eur == 0
|
||||
|
||||
|
||||
def test_negative_tokens_are_rejected():
|
||||
with pytest.raises(ValueError, match="prompt_tokens"):
|
||||
estimate_cost("openai/gpt-4o-mini", -1, 0)
|
||||
|
||||
|
||||
def test_kimi_k3_rate_and_eur_conversion():
|
||||
estimate = estimate_cost("moonshotai/kimi-k3", 1_000, 1_000, fx=0.92)
|
||||
|
||||
# $0.003 + $0.015 = $0.018; ×0.92 = €0.01656
|
||||
assert estimate.cost_usd == pytest.approx(0.018)
|
||||
assert estimate.cost_eur == pytest.approx(0.01656)
|
||||
assert estimate.cost_source == "rate_table:moonshotai/kimi-k3"
|
||||
|
||||
|
||||
def test_apply_fx_false_skips_eur():
|
||||
estimate = estimate_cost("moonshotai/kimi-k3", 100, 0, apply_fx=False)
|
||||
|
||||
assert estimate.cost_usd is not None
|
||||
assert estimate.cost_eur is None
|
||||
assert estimate.fx_source is None
|
||||
|
|
|
|||
40
tests/test_fx.py
Normal file
40
tests/test_fx.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
import pytest
|
||||
|
||||
from llm_connect.fx import DEFAULT_EUR_PER_USD, FxRate, resolve_fx_rate
|
||||
|
||||
|
||||
def test_resolve_fx_prefers_explicit_rate():
|
||||
rate = resolve_fx_rate(0.85)
|
||||
|
||||
assert rate is not None
|
||||
assert rate.eur_per_usd == pytest.approx(0.85)
|
||||
assert rate.source == "explicit"
|
||||
assert rate.usd_to_eur(10.0) == pytest.approx(8.5)
|
||||
|
||||
|
||||
def test_resolve_fx_reads_env(monkeypatch):
|
||||
monkeypatch.setenv("LLM_CONNECT_EUR_PER_USD", "0.9")
|
||||
|
||||
rate = resolve_fx_rate()
|
||||
|
||||
assert rate is not None
|
||||
assert rate.eur_per_usd == pytest.approx(0.9)
|
||||
assert rate.source == "env:LLM_CONNECT_EUR_PER_USD"
|
||||
|
||||
|
||||
def test_resolve_fx_falls_back_to_snapshot():
|
||||
rate = resolve_fx_rate(env={})
|
||||
|
||||
assert rate is not None
|
||||
assert rate.eur_per_usd == pytest.approx(DEFAULT_EUR_PER_USD)
|
||||
assert rate.source.startswith("snapshot:")
|
||||
|
||||
|
||||
def test_invalid_env_rate_raises():
|
||||
with pytest.raises(ValueError, match="LLM_CONNECT_EUR_PER_USD"):
|
||||
resolve_fx_rate(env={"LLM_CONNECT_EUR_PER_USD": "0"})
|
||||
|
||||
|
||||
def test_fx_rate_rejects_non_positive():
|
||||
with pytest.raises(ValueError, match="eur_per_usd"):
|
||||
FxRate(eur_per_usd=-1, source="x")
|
||||
|
|
@ -61,3 +61,21 @@ def test_wp_0006_profile_primitives_are_exported_from_package_root():
|
|||
for name in expected_names:
|
||||
assert hasattr(llm_connect, name)
|
||||
assert name in llm_connect.__all__
|
||||
|
||||
|
||||
def test_wp_0007_spend_primitives_are_exported_from_package_root():
|
||||
expected_names = [
|
||||
"FxRate",
|
||||
"resolve_fx_rate",
|
||||
"UsageEvent",
|
||||
"UsageLedger",
|
||||
"UsageSummary",
|
||||
"default_usage_ledger_path",
|
||||
"event_from_response",
|
||||
"maybe_record_usage",
|
||||
"week_window",
|
||||
]
|
||||
|
||||
for name in expected_names:
|
||||
assert hasattr(llm_connect, name)
|
||||
assert name in llm_connect.__all__
|
||||
|
|
|
|||
|
|
@ -7,9 +7,12 @@ def test_default_registry_contains_openrouter_seed_models():
|
|||
registry = ModelRateRegistry.default()
|
||||
rates = registry.all()
|
||||
|
||||
assert len(rates) >= 9
|
||||
assert len(rates) >= 10
|
||||
assert rates["openai/gpt-4o-mini"].captured_at == "2026-05-17"
|
||||
assert rates["openai/gpt-4o-mini"].source_url == "https://openrouter.ai/models"
|
||||
assert rates["moonshotai/kimi-k3"].prompt_per_1k == 0.003
|
||||
assert rates["moonshotai/kimi-k3"].completion_per_1k == 0.015
|
||||
assert rates["moonshotai/kimi-k3"].captured_at == "2026-08-03"
|
||||
|
||||
|
||||
def test_from_yaml_loads_package_shape(tmp_path):
|
||||
|
|
|
|||
127
tests/test_usage.py
Normal file
127
tests/test_usage.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_connect.models import LLMResponse
|
||||
from llm_connect.usage import (
|
||||
UsageEvent,
|
||||
UsageLedger,
|
||||
default_usage_ledger_path,
|
||||
event_from_response,
|
||||
maybe_record_usage,
|
||||
suppress_auto_usage_record,
|
||||
week_window,
|
||||
)
|
||||
|
||||
|
||||
def test_default_ledger_path_env_and_xdg():
|
||||
assert default_usage_ledger_path(env={"LLM_CONNECT_USAGE_LEDGER": "/tmp/u.jsonl"}) == (
|
||||
__import__("pathlib").Path("/tmp/u.jsonl")
|
||||
)
|
||||
path = default_usage_ledger_path(env={"XDG_DATA_HOME": "/tmp/xdg"})
|
||||
assert path.as_posix().endswith("llm-connect/usage.jsonl")
|
||||
assert "xdg" in path.as_posix()
|
||||
|
||||
|
||||
def test_usage_ledger_append_and_sum_range(tmp_path):
|
||||
ledger = UsageLedger(tmp_path / "usage.jsonl")
|
||||
berlin = ZoneInfo("Europe/Berlin")
|
||||
monday = datetime(2026, 8, 3, 10, 0, tzinfo=berlin) # Monday
|
||||
sunday = datetime(2026, 8, 2, 12, 0, tzinfo=berlin) # previous Sunday
|
||||
|
||||
ledger.append(
|
||||
UsageEvent(
|
||||
provider="openrouter",
|
||||
model_id="moonshotai/kimi-k3",
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
total_tokens=150,
|
||||
cost_usd=0.001,
|
||||
cost_eur=0.00092,
|
||||
cost_source="rate_table:moonshotai/kimi-k3",
|
||||
fx_source="snapshot:2026-08-03",
|
||||
source="cli",
|
||||
recorded_at=monday,
|
||||
)
|
||||
)
|
||||
ledger.append(
|
||||
UsageEvent(
|
||||
provider="openrouter",
|
||||
model_id="moonshotai/kimi-k3",
|
||||
prompt_tokens=10,
|
||||
completion_tokens=5,
|
||||
total_tokens=15,
|
||||
cost_usd=0.0001,
|
||||
cost_eur=0.000092,
|
||||
cost_source="rate_table:moonshotai/kimi-k3",
|
||||
source="cli",
|
||||
recorded_at=sunday,
|
||||
)
|
||||
)
|
||||
|
||||
start, end = week_window(
|
||||
"current",
|
||||
now=datetime(2026, 8, 5, 18, 0, tzinfo=berlin),
|
||||
tz="Europe/Berlin",
|
||||
)
|
||||
summary = ledger.sum_range(start, end)
|
||||
|
||||
assert summary.event_count == 1
|
||||
assert summary.prompt_tokens == 100
|
||||
assert summary.total_tokens == 150
|
||||
assert summary.cost_eur == pytest.approx(0.00092)
|
||||
|
||||
last_start, last_end = week_window(
|
||||
"last",
|
||||
now=datetime(2026, 8, 5, 18, 0, tzinfo=berlin),
|
||||
tz="Europe/Berlin",
|
||||
)
|
||||
last_summary = ledger.sum_range(last_start, last_end)
|
||||
assert last_summary.event_count == 1
|
||||
assert last_summary.total_tokens == 15
|
||||
|
||||
|
||||
def test_week_window_current_and_last():
|
||||
berlin = ZoneInfo("Europe/Berlin")
|
||||
now = datetime(2026, 8, 5, 15, 30, tzinfo=berlin) # Wednesday
|
||||
start, end = week_window("current", now=now, tz="Europe/Berlin")
|
||||
assert start.weekday() == 0
|
||||
assert start.hour == 0
|
||||
assert end == now
|
||||
|
||||
last_start, last_end = week_window("last", now=now, tz="Europe/Berlin")
|
||||
assert (last_end - last_start).days == 7
|
||||
assert last_end == start
|
||||
|
||||
|
||||
def test_event_from_response_estimates_kimi_cost():
|
||||
response = LLMResponse(
|
||||
content="hi",
|
||||
model="moonshotai/kimi-k3",
|
||||
usage={"prompt_tokens": 1000, "completion_tokens": 1000, "total_tokens": 2000},
|
||||
)
|
||||
event = event_from_response(response, provider="openrouter", source="cli", fx=1.0)
|
||||
|
||||
assert event.cost_usd == pytest.approx(0.018)
|
||||
assert event.cost_eur == pytest.approx(0.018)
|
||||
assert event.cost_source == "rate_table:moonshotai/kimi-k3"
|
||||
|
||||
|
||||
def test_maybe_record_usage_opt_in(tmp_path, monkeypatch):
|
||||
ledger = tmp_path / "usage.jsonl"
|
||||
monkeypatch.setenv("LLM_CONNECT_USAGE_LEDGER", str(ledger))
|
||||
response = LLMResponse(
|
||||
content="x",
|
||||
model="moonshotai/kimi-k3",
|
||||
usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
metadata={"provider": "openrouter"},
|
||||
)
|
||||
|
||||
event = maybe_record_usage(response, provider="openrouter", source="library")
|
||||
assert event is not None
|
||||
assert ledger.is_file()
|
||||
assert UsageLedger(ledger).read_all()[0].source == "library"
|
||||
|
||||
with suppress_auto_usage_record():
|
||||
assert maybe_record_usage(response, provider="openrouter", source="library") is None
|
||||
Loading…
Add table
Add a link
Reference in a new issue