Appearance
Realtime example
A complete realtime transcription client in one file, built on the official OpenAI Python SDK. It streams an audio file at real speed, the way a microphone would, and prints the transcript while the audio is still being sent.
The other endpoints need only a few lines each; their pages show the code: text to speech, speech to text and models. Realtime is the one worth a full client, because it has to send audio and receive text at the same time.
Run it
bash
pip install "openai[realtime]>=2.43"
export XVOICE_API_KEY=xv_...
python realtime.py speech.wav
python realtime.py meeting.m4a --language en --segment 10It needs ffmpeg on your PATH, only to stand in for a microphone: ffmpeg decodes the file to the wire format, 24 kHz mono PCM16. A live client captures that format directly and needs nothing extra.
Run python realtime.py --help for every option. --segment sets how often the script commits a turn, since the server does not detect the end of speech for you.
What to reuse
sendpaces audio at real time and commits every--segmentseconds. Replace the file and the timer with your microphone and your own end-of-speech logic.receiveandSessionhandle transcripts for several turns at once, keyed byitem_id, because a long turn can finish after a short one that followed it.streamruns the two together, so anerrorevent stops the sender straight away instead of after the whole file has gone out.
realtime.py
python
"""Realtime speech to text: a live session over a WebSocket.
pip install "openai[realtime]>=2.43"
export XVOICE_API_KEY=xv_...
python realtime.py speech.wav
python realtime.py meeting.m4a --language en --segment 10
This opens a session and feeds audio while it is still being produced, which is what a
phone call or a browser microphone does: the first words come back before the speaker
has finished.
Here ffmpeg stands in for the capture device -- it decodes the file to the wire
format and the script paces it at real speed. A live client sends the same bytes
straight from its microphone and needs no ffmpeg at all.
Two things differ from the file endpoints and catch people out:
* **There is no voice activity detection.** The server never decides that an
utterance has ended, so nothing comes back until you send
`input_audio_buffer.commit`. `--segment` controls how often this script does that.
* **The audio format is fixed:** 24 kHz mono PCM16, little endian, base64 inside
JSON. There is no container and no header, so there is nothing to negotiate.
"""
import argparse
import asyncio
import base64
import json
import os
import shutil
import sys
import time
from collections.abc import AsyncIterator
from contextlib import aclosing
from dataclasses import dataclass, field
from pathlib import Path
from openai import APIConnectionError, APIError, AsyncOpenAI
DEFAULT_BASE_URL = "https://api.xvoice.dev/v1"
# The wire format. These are protocol constants, not settings: the endpoint accepts
# nothing else, and a rate mismatch is rejected rather than resampled.
SAMPLE_RATE = 24_000
BYTES_PER_SAMPLE = 2
FRAME_MS = 20
FRAME_BYTES = SAMPLE_RATE * BYTES_PER_SAMPLE * FRAME_MS // 1000 # 960
TRANSCRIPTION = "conversation.item.input_audio_transcription"
class SessionError(Exception):
"""Something ended the session: a server `error` event, or audio we could not read."""
def closed(exc: BaseException) -> str | None:
"""Render a WebSocket close, if that is what this exception is.
The server always sends an `error` event before closing, and websockets delivers
queued frames before it raises, so reaching this means the connection dropped
without one: the deadline, a proxy, or the network.
Duck-typed on `websockets.exceptions.ConnectionClosed`, which is the SDK's
transport rather than part of its API.
"""
frame = getattr(exc, "rcvd", None) or getattr(exc, "sent", None)
if frame is None:
return None
return f"the server closed the session (code {frame.code}): {frame.reason or 'no reason given'}"
def explain(error: dict) -> str:
return (
f"{error.get('code', 'unknown')}: {error.get('message', '')} "
f"(request_id: {error.get('request_id')})"
)
def refused(exc: BaseException) -> str | None:
"""Render a session refused *before* the WebSocket upgrade, if that is what this is.
A bad key or an empty balance is answered as an ordinary HTTP response carrying
the usual error envelope, which the SDK raises as its transport's exception
rather than an `APIError`. Duck-typed on `websockets.exceptions.InvalidStatus`,
for the same reason as `closed`.
"""
response = getattr(exc, "response", None)
status = getattr(response, "status_code", None)
body = getattr(response, "body", None)
if status is None or body is None:
return None
try:
error = json.loads(body or b"{}").get("error", {})
except ValueError:
error = {}
return f"HTTP {status} {explain(error)}"
# ----------------------------------------------------------------------- the audio
async def frames(path: Path) -> AsyncIterator[bytes]:
"""Decode any audio file to 20 ms frames of 24 kHz mono PCM16.
Frame size is a choice, not a requirement -- the server takes appends of any even
length up to 15 MB. 20 ms is what capture devices hand you and what keeps the
round trip short; larger frames trade latency for fewer messages.
"""
process = await asyncio.create_subprocess_exec(
"ffmpeg", "-v", "error", "-i", str(path),
"-f", "s16le", "-acodec", "pcm_s16le", "-ac", "1", "-ar", str(SAMPLE_RATE), "-",
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
) # fmt: skip
assert process.stdout is not None
buffered = b""
try:
while chunk := await process.stdout.read(FRAME_BYTES * 16):
buffered += chunk
while len(buffered) >= FRAME_BYTES:
yield buffered[:FRAME_BYTES]
buffered = buffered[FRAME_BYTES:]
if buffered:
yield buffered # ffmpeg's output is always sample-aligned
_, errors = await process.communicate()
if process.returncode:
raise SessionError(f"ffmpeg could not read {path}:\n{errors.decode().strip()}")
finally:
if process.returncode is None:
process.kill()
# ------------------------------------------------------------------ the transcript
@dataclass
class Turn:
number: int
streamed: str = ""
transcript: str = ""
seconds: float = 0.0
@dataclass
class Session:
"""What has been said so far, and how much of it is still being transcribed.
Utterances are concurrent: committing frees the input buffer at once and the
server keeps transcribing in the background, so a long turn can finish after a short
one that followed it. Everything is therefore keyed by `item_id`, never held as a
single "current" turn.
"""
turns: dict[str, Turn] = field(default_factory=dict)
open_line: str | None = None
committed: int = 0
finished: int = 0
input_closed: bool = False
@property
def settled(self) -> bool:
return self.input_closed and self.finished >= self.committed
def began(self, item_id: str) -> Turn:
"""Start numbering a turn, whichever event mentioned it first.
Text streams while the buffer is still open, so the first delta usually
arrives before the `input_audio_buffer.committed` that announces the item.
Waiting for the announcement would mean printing nothing until the turn was
already over, which is the opposite of what a streaming session is for.
"""
turn = self.turns.get(item_id)
if turn is None:
turn = self.turns[item_id] = Turn(number=len(self.turns) + 1)
return turn
def delta(self, item_id: str, text: str) -> None:
"""Print deltas as they land, starting a new line when the turn changes.
Deltas are immutable and never revised, so they can go straight to the
terminal -- but two turns can be producing them at once, hence the tag.
"""
turn = self.began(item_id)
turn.streamed += text
if self.open_line != item_id:
self.end_line()
print(f"[{turn.number}] ", end="")
self.open_line = item_id
print(text, end="", flush=True)
def completed(self, item_id: str, transcript: str, seconds: float) -> None:
turn = self.began(item_id)
turn.transcript = transcript
turn.seconds = seconds
self.finished += 1
self.end_line()
# The deltas were only ever a preview; this is the final transcript.
# Usually the same text, so reprint the turn only when it is not -- the
# deltas carry their own spacing, hence the strip rather than ==.
if transcript.strip() != turn.streamed.strip():
print(f"[{turn.number}] {transcript}")
def failed(self, item_id: str, error: dict) -> None:
turn = self.began(item_id)
self.finished += 1
self.end_line()
print(f"[{turn.number}] failed -- {explain(error)}", file=sys.stderr)
def end_line(self) -> None:
if self.open_line is not None:
print()
self.open_line = None
async def receive(connection, session: Session) -> None:
"""Relay server events into `session` until every committed turn has settled."""
async for event in connection:
if event.type == "error":
raise SessionError(explain(event.error.to_dict()))
elif event.type == "input_audio_buffer.committed":
session.began(event.item_id)
elif event.type == f"{TRANSCRIPTION}.delta":
session.delta(event.item_id, event.delta)
elif event.type == f"{TRANSCRIPTION}.completed":
session.completed(
event.item_id, event.transcript, getattr(event.usage, "seconds", 0.0)
)
elif event.type == f"{TRANSCRIPTION}.failed":
session.failed(event.item_id, event.error.to_dict())
if session.settled:
return
async def send(
connection, session: Session, path: Path, *, segment: float, pace: bool
) -> float:
"""Feed the file in, committing a turn every `segment` seconds of audio."""
sent = 0
since_commit = 0
per_segment = int(segment * SAMPLE_RATE * BYTES_PER_SAMPLE)
started = time.perf_counter()
async with aclosing(frames(path)) as audio:
async for frame in audio:
await connection.send(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(frame).decode(),
}
)
sent += len(frame)
since_commit += len(frame)
if per_segment and since_commit >= per_segment:
await connection.send({"type": "input_audio_buffer.commit"})
session.committed += 1
since_commit = 0
if pace:
# Hold the send rate to real time, the way a microphone would.
# Sleeping a flat 20 ms per frame would drift, so aim at where the
# clock ought to be by now.
spent = time.perf_counter() - started
behind = sent / (SAMPLE_RATE * BYTES_PER_SAMPLE) - spent
if behind > 0:
await asyncio.sleep(behind)
if since_commit:
await connection.send({"type": "input_audio_buffer.commit"})
session.committed += 1
session.input_closed = True
return sent / (SAMPLE_RATE * BYTES_PER_SAMPLE)
# ------------------------------------------------------------------------- the run
async def stream(connection, session: Session, args: argparse.Namespace) -> float:
"""Send audio and receive events at once.
They have to overlap: the first deltas of one turn arrive while the next is
still being spoken, and an `error` event has to cut the sender short rather than
be noticed once the whole file has gone out.
"""
sender = asyncio.create_task(
send(connection, session, args.file, segment=args.segment, pace=not args.fast)
)
listener = asyncio.create_task(receive(connection, session))
try:
await asyncio.wait({sender, listener}, return_when=asyncio.FIRST_COMPLETED)
if listener.done():
await listener # an error, or a session that settled before we noticed
audio_seconds = await sender
if not session.settled:
await asyncio.wait_for(asyncio.shield(listener), timeout=args.timeout)
return audio_seconds
except TimeoutError:
raise SessionError(
f"no transcript for the last turn within {args.timeout:g}s"
) from None
except Exception as exc:
reason = closed(exc)
if reason is None:
raise
raise SessionError(reason) from exc
finally:
session.end_line()
for task in (sender, listener):
task.cancel()
await asyncio.gather(sender, listener, return_exceptions=True)
async def main() -> None:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument("file", type=Path, help="audio to stream, in any format ffmpeg reads")
parser.add_argument("--model", default="cadence-stt")
parser.add_argument("--language", help="ISO-639-1 code, e.g. en; omit to auto-detect")
parser.add_argument("--prompt", help="spellings or context to guide the transcript")
parser.add_argument(
"--segment",
type=float,
default=15.0,
help="commit a turn every N seconds of audio (default: 15). "
"0 sends the whole file as a single turn.",
)
parser.add_argument(
"--fast",
action="store_true",
help="send as fast as the socket allows instead of at real speed",
)
parser.add_argument(
"--timeout",
type=float,
default=60.0,
help="seconds to wait for the last turn after the audio ends (default: 60)",
)
parser.add_argument(
"--api-key",
default=os.environ.get("XVOICE_API_KEY"),
help="xVoice API key (default: $XVOICE_API_KEY)",
)
parser.add_argument(
"--base-url",
default=os.environ.get("XVOICE_BASE_URL", DEFAULT_BASE_URL),
help=f"API base URL including /v1 (default: $XVOICE_BASE_URL or {DEFAULT_BASE_URL})",
)
args = parser.parse_args()
if not args.api_key:
sys.exit(
"No API key. Create one in the dashboard (Projects → API keys), then either\n"
" export XVOICE_API_KEY=xv_...\n"
"or pass --api-key xv_..."
)
if not args.file.is_file():
sys.exit(f"No such file: {args.file}")
if shutil.which("ffmpeg") is None:
sys.exit(
"ffmpeg is not installed. It is only needed to turn a file into "
"microphone-shaped audio;\na live client already has 24 kHz mono PCM16 to hand."
)
transcription: dict[str, str] = {"model": args.model}
if args.language:
transcription["language"] = args.language
if args.prompt:
transcription["prompt"] = args.prompt
client = AsyncOpenAI(api_key=args.api_key, base_url=args.base_url)
session = Session()
started = time.perf_counter()
async with client.realtime.connect(
# No `model` query parameter: the model is named in session.update below, so
# one connection could serve several. `intent=transcription` is what selects
# this endpoint's behaviour.
extra_query={"intent": "transcription"},
max_retries=0,
) as connection:
created = await connection.recv()
if created.type != "session.created":
raise SessionError(f"expected session.created, got {created.type}")
await connection.send(
{
"type": "session.update",
"session": {
"type": "transcription",
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": SAMPLE_RATE},
"transcription": transcription,
# Explicitly off. The server refuses any other value
# rather than quietly ignoring it.
"turn_detection": None,
}
},
},
}
)
updated = await connection.recv()
if updated.type == "error":
raise SessionError(explain(updated.error.to_dict()))
if updated.type != "session.updated":
raise SessionError(f"expected session.updated, got {updated.type}")
# The session is ready, and billable from here.
audio_seconds = await stream(connection, session, args)
billed = sum(turn.seconds for turn in session.turns.values())
print(
f"\n[{len(session.turns)} turn(s), {audio_seconds:.1f} s of audio, "
f"{billed:.3f} s billed, {time.perf_counter() - started:.1f} s wall clock]"
)
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
sys.exit(130)
except SessionError as exc:
sys.exit(str(exc))
except APIConnectionError as exc:
sys.exit(refused(exc.__cause__ or exc) or f"Could not reach xVoice: {exc.message}")
except APIError as exc:
# xVoice's {"error": {code, message, request_id}} is unwrapped into `body`.
body = exc.body if isinstance(exc.body, dict) else {}
error = body.get("error", body)
error.setdefault("message", exc.message)
sys.exit(f"HTTP {getattr(exc, 'status_code', '?')} {explain(error)}")
except Exception as exc: # noqa: BLE001 - re-raised unless it is a refused upgrade
reason = refused(exc)
if reason is None:
raise
sys.exit(reason)