17 lines
467 B
Python
17 lines
467 B
Python
|
"""
|
||
|
|
Rough token estimation for backends that don't return usage data.
|
||
|
|
|
||
|
|
Uses the ~4 characters per token heuristic common across English LLM tokenizers.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
def estimate_tokens(text: str) -> int:
|
||
|
|
"""Estimate the number of tokens in *text*.
|
||
|
|
|
||
|
|
This is intentionally coarse — it is only used by the Claude Code CLI
|
||
|
|
adapter where real token counts are unavailable.
|
||
|
|
"""
|
||
|
|
if not text:
|
||
|
|
return 0
|
||
|
|
return max(1, len(text) // 4)
|