Benchmark reference

audiotext.benchmark

BenchmarkRecord dataclass

Source code in src/audiotext/benchmark.py 
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
@dataclass(frozen=True)
class BenchmarkRecord:
    audio_path: str
    model: str
    language: str
    cpu_threads: int
    num_workers: int
    beam_size: int
    vad_filter: bool
    wall_seconds: float
    cpu_seconds: float
    cpu_percent: float
    load_seconds: float
    transcribe_seconds: float
    total_seconds: float
    cache_hit: bool
    peak_rss_bytes: int
    text_preview: str

    def to_dict(self) -> dict[str, object]:
        return asdict(self)

audio_path instance-attribute 

audio_path

model instance-attribute 

model

language instance-attribute 

language

cpu_threads instance-attribute 

cpu_threads

num_workers instance-attribute 

num_workers

beam_size instance-attribute 

beam_size

vad_filter instance-attribute 

vad_filter

wall_seconds instance-attribute 

wall_seconds

cpu_seconds instance-attribute 

cpu_seconds

cpu_percent instance-attribute 

cpu_percent

load_seconds instance-attribute 

load_seconds

transcribe_seconds instance-attribute 

transcribe_seconds

total_seconds instance-attribute 

total_seconds

cache_hit instance-attribute 

cache_hit

peak_rss_bytes instance-attribute 

peak_rss_bytes

text_preview instance-attribute 

text_preview

__init__ 

__init__(
    audio_path,
    model,
    language,
    cpu_threads,
    num_workers,
    beam_size,
    vad_filter,
    wall_seconds,
    cpu_seconds,
    cpu_percent,
    load_seconds,
    transcribe_seconds,
    total_seconds,
    cache_hit,
    peak_rss_bytes,
    text_preview,
)

to_dict 

to_dict()
Source code in src/audiotext/benchmark.py 
32
33
def to_dict(self) -> dict[str, object]:
    return asdict(self)

run_benchmark_case 

run_benchmark_case(
    *,
    manager,
    audio_path,
    model,
    language,
    cpu_threads,
    num_workers,
    beam_size,
    vad_filter,
)
Source code in src/audiotext/benchmark.py 
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def run_benchmark_case(
    *,
    manager: ModelManager,
    audio_path: Path,
    model: str,
    language: str,
    cpu_threads: int,
    num_workers: int,
    beam_size: int,
    vad_filter: bool,
) -> BenchmarkRecord:
    runtime = RuntimeOptions(preset=model, cpu_threads=cpu_threads, num_workers=num_workers)
    options = normalize_options(
        overrides={
            "language": language,
            "beam_size": beam_size,
            "best_of": 1,
            "vad_filter": vad_filter,
            "condition_on_previous_text": False,
        }
    )
    cpu_started = _cpu_seconds()
    started = time.perf_counter()
    result, timing = manager.transcribe(runtime, str(audio_path), options)
    wall_seconds = time.perf_counter() - started
    cpu_seconds = _cpu_seconds() - cpu_started
    return BenchmarkRecord(
        audio_path=str(audio_path),
        model=model,
        language=language,
        cpu_threads=cpu_threads,
        num_workers=num_workers,
        beam_size=beam_size,
        vad_filter=vad_filter,
        wall_seconds=round(wall_seconds, 3),
        cpu_seconds=round(cpu_seconds, 3),
        cpu_percent=round((cpu_seconds / wall_seconds * 100.0) if wall_seconds > 0 else 0.0, 1),
        load_seconds=float(timing["load_seconds"]),
        transcribe_seconds=float(timing["transcribe_seconds"]),
        total_seconds=float(timing["total_seconds"]),
        cache_hit=bool(timing["cache_hit"]),
        peak_rss_bytes=current_rss_bytes(),
        text_preview=_preview(result.text),
    )

render_markdown 

render_markdown(records)
Source code in src/audiotext/benchmark.py 
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
def render_markdown(records: list[BenchmarkRecord]) -> str:
    lines = [
        "| Audio | Model | Lang | Threads | Workers | Beam | VAD | Wall s | CPU s | CPU % | Load s | Transcribe s | Cache | Peak RSS MB | Preview |",
        "| --- | --- | --- | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | --- | ---: | --- |",
    ]
    for record in records:
        lines.append(
            "| "
            + " | ".join(
                [
                    _md(record.audio_path),
                    _md(record.model),
                    _md(record.language),
                    str(record.cpu_threads),
                    str(record.num_workers),
                    str(record.beam_size),
                    "yes" if record.vad_filter else "no",
                    f"{record.wall_seconds:.3f}",
                    f"{record.cpu_seconds:.3f}",
                    f"{record.cpu_percent:.1f}",
                    f"{record.load_seconds:.3f}",
                    f"{record.transcribe_seconds:.3f}",
                    "yes" if record.cache_hit else "no",
                    f"{record.peak_rss_bytes / (1024 * 1024):.1f}",
                    _md(record.text_preview),
                ]
            )
            + " |"
        )
    return "\n".join(lines) + "\n"