feat(engine): add sr-voice LLM inference service for NPC voice pipeline
Standalone Rust crate wrapping llama-cpp-2 for GGUF model inference. Persistent HTTP server architecture — model loaded once, requests processed sequentially, zero CPU contention by construction. Subcommands: serve (load model, listen), generate (single prompt), batch (JSONL), benchmark (5-run average). Makefile targets for build/serve/run/stop workflow. Spike 1 validated: Gemma 2B Q4_K_M at ~16 t/s CPU, 4 cultures tested (Krenn, Ireland, Shek'na, Aranthi), composition-engine oath injection mechanism proven. GO for Spike 2. Refs: D-138, #639 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -2,6 +2,8 @@
|
||||
.cache/
|
||||
.tmp/
|
||||
server/target/
|
||||
server/sr-voice/target/
|
||||
server/models/
|
||||
tooling/content-converter/target/
|
||||
tooling/line-previewer/target/
|
||||
tooling/test-client/target/
|
||||
|
||||
@@ -7,6 +7,7 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null)
|
||||
pre-pr-server pre-pr-client pre-pr-content \
|
||||
fixtures-client fixtures-gauntlet golden-diff golden-update \
|
||||
checklist-validate checklist-generate \
|
||||
build-sr-voice run-sr-voice \
|
||||
perf-baseline debug-schedule \
|
||||
test-ipc-fixtures test-ipc-protocol test-ipc-integration test-ipc-benchmark \
|
||||
screenshot visual-movie test-visual visual-update
|
||||
@@ -65,6 +66,10 @@ help:
|
||||
@echo " make pre-pr-content Content-scoped pre-PR (schema + cross-ref validation)"
|
||||
@echo ""
|
||||
@echo " make setup-hooks Install pre-commit hooks (included in setup)"
|
||||
@echo " make build-sr-voice Build sr-voice LLM inference service"
|
||||
@echo " make serve-sr-voice Start sr-voice server (ARGS='--model <path>')"
|
||||
@echo " make run-sr-voice Submit to sr-voice server (ARGS='generate|batch|benchmark ...')"
|
||||
@echo " make stop-sr-voice Stop sr-voice server"
|
||||
@echo " make debug-schedule Print bevy_ecs schedule graph (diff for PR artifacts)"
|
||||
@echo ""
|
||||
@echo " GODOT_VERSION=4.6 make setup Override Godot version"
|
||||
@@ -344,6 +349,25 @@ test-visual:
|
||||
visual-update:
|
||||
@tests/run-visual --update
|
||||
|
||||
LIBCLANG_PATH ?= /usr/lib64/rocm/llvm/lib
|
||||
BINDGEN_CLANG_ARGS ?= -I/usr/lib64/rocm/llvm/lib/clang/19/include
|
||||
SR_VOICE_ENV = LIBCLANG_PATH=$(LIBCLANG_PATH) BINDGEN_EXTRA_CLANG_ARGS="$(BINDGEN_CLANG_ARGS)"
|
||||
|
||||
SR_VOICE_PORT ?= 8321
|
||||
|
||||
build-sr-voice:
|
||||
cd server/sr-voice && $(SR_VOICE_ENV) cargo build --release
|
||||
|
||||
serve-sr-voice:
|
||||
cd server/sr-voice && $(SR_VOICE_ENV) cargo run --release -- serve $(ARGS)
|
||||
|
||||
run-sr-voice:
|
||||
cd server/sr-voice && $(SR_VOICE_ENV) cargo run --release -- $(ARGS)
|
||||
|
||||
stop-sr-voice:
|
||||
@lsof -ti :$(SR_VOICE_PORT) | xargs -r kill 2>/dev/null || true
|
||||
@echo "Stopped sr-voice on port $(SR_VOICE_PORT)"
|
||||
|
||||
content-ron:
|
||||
cd tooling/content-converter && cargo build --release
|
||||
tooling/content-converter/target/release/content-converter --input content --output content-ron --verbose
|
||||
|
||||
Generated
+964
@@ -0,0 +1,964 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ascii"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.72.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"itertools",
|
||||
"log",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "chunked_transfer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.60"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.60"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.5.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enumflags2"
|
||||
version = "0.7.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"
|
||||
dependencies = [
|
||||
"enumflags2_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "enumflags2_derive"
|
||||
version = "0.7.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "find_cuda_helper"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad"
|
||||
dependencies = [
|
||||
"glob",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.182"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cpp-2"
|
||||
version = "0.1.138"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2947ab625c59d1fdf42e61f538c3fa66f43de2f78316971920873f359483d1d8"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"enumflags2",
|
||||
"llama-cpp-sys-2",
|
||||
"thiserror",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cpp-sys-2"
|
||||
version = "0.1.138"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84a529006bf16af70c7485ba957820dc2bc9467d75697e97970c81d2da73c76f"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"cmake",
|
||||
"find_cuda_helper",
|
||||
"glob",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.17.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||
dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
|
||||
|
||||
[[package]]
|
||||
name = "sr-voice"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"llama-cpp-2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tiny_http",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiny_http"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82"
|
||||
dependencies = [
|
||||
"ascii",
|
||||
"chunked_transfer",
|
||||
"httpdate",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"flate2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"ureq-proto",
|
||||
"utf-8",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "wasip2"
|
||||
version = "1.0.2+wasi-0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
|
||||
dependencies = [
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm",
|
||||
"windows_aarch64_msvc",
|
||||
"windows_i686_gnu",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_msvc",
|
||||
"windows_x86_64_gnu",
|
||||
"windows_x86_64_gnullvm",
|
||||
"windows_x86_64_msvc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.51.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "sr-voice"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
llama-cpp-2 = "0.1"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
thiserror = "2"
|
||||
tiny_http = "0.12"
|
||||
ureq = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
metal = ["llama-cpp-2/metal"]
|
||||
vulkan = ["llama-cpp-2/vulkan"]
|
||||
cuda = ["llama-cpp-2/cuda"]
|
||||
rocm = ["llama-cpp-2/rocm"]
|
||||
@@ -0,0 +1,165 @@
|
||||
use std::num::NonZeroU32;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use llama_cpp_2::context::params::LlamaContextParams;
|
||||
use llama_cpp_2::llama_backend::LlamaBackend;
|
||||
use llama_cpp_2::llama_batch::LlamaBatch;
|
||||
use llama_cpp_2::model::params::LlamaModelParams;
|
||||
use llama_cpp_2::model::{AddBos, LlamaModel, Special};
|
||||
use llama_cpp_2::sampling::LlamaSampler;
|
||||
|
||||
use crate::VoiceError;
|
||||
|
||||
/// Configuration for model loading and inference.
|
||||
pub struct InferenceConfig {
|
||||
pub model_path: String,
|
||||
pub threads: u32,
|
||||
pub ctx_size: u32,
|
||||
pub seed: Option<u32>,
|
||||
}
|
||||
|
||||
/// Result of a single generation call.
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct GenerationResult {
|
||||
pub text: String,
|
||||
pub tokens_generated: u32,
|
||||
pub generation_time_ms: u64,
|
||||
pub tokens_per_sec: f64,
|
||||
pub prefill_time_ms: u64,
|
||||
}
|
||||
|
||||
/// Wraps llama.cpp model and context for text generation.
|
||||
pub struct InferenceEngine {
|
||||
backend: LlamaBackend,
|
||||
model: LlamaModel,
|
||||
ctx_size: u32,
|
||||
threads: u32,
|
||||
}
|
||||
|
||||
impl InferenceEngine {
|
||||
/// Load a GGUF model from disk.
|
||||
pub fn load(config: &InferenceConfig) -> Result<Self, VoiceError> {
|
||||
let backend =
|
||||
LlamaBackend::init().map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?;
|
||||
|
||||
let model_params = LlamaModelParams::default();
|
||||
let model = LlamaModel::load_from_file(
|
||||
&backend,
|
||||
Path::new(&config.model_path),
|
||||
&model_params,
|
||||
)
|
||||
.map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?;
|
||||
|
||||
Ok(Self {
|
||||
backend,
|
||||
model,
|
||||
ctx_size: config.ctx_size,
|
||||
threads: config.threads,
|
||||
})
|
||||
}
|
||||
|
||||
/// Generate text from a prompt.
|
||||
pub fn generate(
|
||||
&self,
|
||||
prompt: &str,
|
||||
max_tokens: u32,
|
||||
temperature: f32,
|
||||
top_p: f32,
|
||||
seed: Option<u32>,
|
||||
) -> Result<GenerationResult, VoiceError> {
|
||||
let ctx_params = LlamaContextParams::default()
|
||||
.with_n_ctx(NonZeroU32::new(self.ctx_size))
|
||||
.with_n_threads(self.threads as i32)
|
||||
.with_n_threads_batch(self.threads as i32);
|
||||
|
||||
let mut ctx = self
|
||||
.model
|
||||
.new_context(&self.backend, ctx_params)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
|
||||
// Tokenize the prompt
|
||||
let tokens = self
|
||||
.model
|
||||
.str_to_token(prompt, AddBos::Always)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
|
||||
if tokens.len() as u32 >= self.ctx_size {
|
||||
return Err(VoiceError::InferenceFailed(format!(
|
||||
"Prompt ({} tokens) exceeds context size ({})",
|
||||
tokens.len(),
|
||||
self.ctx_size
|
||||
)));
|
||||
}
|
||||
|
||||
// Prefill: evaluate the prompt tokens
|
||||
let prefill_start = Instant::now();
|
||||
let mut batch = LlamaBatch::new(self.ctx_size as usize, 1);
|
||||
for (i, &token) in tokens.iter().enumerate() {
|
||||
let is_last = i == tokens.len() - 1;
|
||||
batch
|
||||
.add(token, i as i32, &[0], is_last)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
}
|
||||
ctx.decode(&mut batch)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
let prefill_time_ms = prefill_start.elapsed().as_millis() as u64;
|
||||
|
||||
// Generation loop
|
||||
let gen_start = Instant::now();
|
||||
let mut generated_tokens: u32 = 0;
|
||||
let mut output = String::new();
|
||||
let mut cur_pos = tokens.len() as i32;
|
||||
|
||||
let mut sampler = LlamaSampler::chain_simple([
|
||||
LlamaSampler::temp(temperature),
|
||||
LlamaSampler::top_p(top_p, 1),
|
||||
LlamaSampler::dist(seed.unwrap_or(1234)),
|
||||
]);
|
||||
|
||||
loop {
|
||||
if generated_tokens >= max_tokens {
|
||||
break;
|
||||
}
|
||||
|
||||
let logits_index = batch.n_tokens() - 1;
|
||||
let token = sampler.sample(&ctx, logits_index);
|
||||
|
||||
if self.model.is_eog_token(token) {
|
||||
break;
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
let piece = self
|
||||
.model
|
||||
.token_to_str(token, Special::Tokenize)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
output.push_str(&piece);
|
||||
generated_tokens += 1;
|
||||
|
||||
batch.clear();
|
||||
batch
|
||||
.add(token, cur_pos, &[0], true)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
cur_pos += 1;
|
||||
|
||||
ctx.decode(&mut batch)
|
||||
.map_err(|e| VoiceError::InferenceFailed(e.to_string()))?;
|
||||
}
|
||||
|
||||
let generation_time_ms = gen_start.elapsed().as_millis() as u64;
|
||||
let tokens_per_sec = if generation_time_ms > 0 {
|
||||
(generated_tokens as f64 / generation_time_ms as f64) * 1000.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
Ok(GenerationResult {
|
||||
text: output,
|
||||
tokens_generated: generated_tokens,
|
||||
generation_time_ms,
|
||||
tokens_per_sec,
|
||||
prefill_time_ms,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
mod inference;
|
||||
mod prompt;
|
||||
mod server;
|
||||
|
||||
use std::io::Read;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use inference::{InferenceConfig, InferenceEngine};
|
||||
|
||||
/// Errors for the sr-voice CLI.
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum VoiceError {
|
||||
#[error("model load failed: {0}")]
|
||||
ModelLoadFailed(String),
|
||||
#[error("inference failed: {0}")]
|
||||
InferenceFailed(String),
|
||||
#[error("invalid input: {0}")]
|
||||
InvalidInput(String),
|
||||
}
|
||||
|
||||
/// sr-voice — LLM inference service for The Settled Reach
|
||||
#[derive(Parser)]
|
||||
#[command(name = "sr-voice", version, about)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Start the inference server (loads model, listens for requests)
|
||||
Serve {
|
||||
/// Path to GGUF model file
|
||||
#[arg(long)]
|
||||
model: String,
|
||||
/// Listen port
|
||||
#[arg(long, default_value = "8321")]
|
||||
port: u16,
|
||||
/// CPU threads for inference
|
||||
#[arg(long)]
|
||||
threads: Option<u32>,
|
||||
/// Context window size in tokens
|
||||
#[arg(long, default_value = "512")]
|
||||
ctx_size: u32,
|
||||
},
|
||||
/// Generate text from a single prompt (requires running server)
|
||||
Generate {
|
||||
/// Server port
|
||||
#[arg(long, default_value = "8321")]
|
||||
port: u16,
|
||||
/// RNG seed
|
||||
#[arg(long)]
|
||||
seed: Option<u32>,
|
||||
/// Prompt file (reads from stdin if omitted)
|
||||
prompt_file: Option<String>,
|
||||
},
|
||||
/// Process a JSONL batch of prompts (requires running server)
|
||||
Batch {
|
||||
/// Server port
|
||||
#[arg(long, default_value = "8321")]
|
||||
port: u16,
|
||||
/// Input JSONL file
|
||||
#[arg(long)]
|
||||
input: String,
|
||||
},
|
||||
/// Run 5 inferences and report average tokens/sec (requires running server)
|
||||
Benchmark {
|
||||
/// Server port
|
||||
#[arg(long, default_value = "8321")]
|
||||
port: u16,
|
||||
},
|
||||
}
|
||||
|
||||
fn default_threads() -> u32 {
|
||||
let cores = std::thread::available_parallelism()
|
||||
.map(|n| n.get() as u32)
|
||||
.unwrap_or(4);
|
||||
cores.saturating_sub(1).max(1)
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Command::Serve { model, port, threads, ctx_size } => {
|
||||
let threads = threads.unwrap_or_else(default_threads);
|
||||
let config = InferenceConfig {
|
||||
model_path: model.clone(),
|
||||
threads,
|
||||
ctx_size,
|
||||
seed: None,
|
||||
};
|
||||
|
||||
eprintln!("Loading model: {}", config.model_path);
|
||||
let engine = InferenceEngine::load(&config)?;
|
||||
eprintln!("Model loaded ({} threads, {} ctx)", threads, ctx_size);
|
||||
|
||||
let model_name = std::path::Path::new(&model)
|
||||
.file_name()
|
||||
.map(|f| f.to_string_lossy().to_string())
|
||||
.unwrap_or(model);
|
||||
|
||||
server::run_server(engine, port, &model_name)?;
|
||||
}
|
||||
Command::Generate { port, seed, prompt_file } => {
|
||||
let prompt = read_prompt(prompt_file)?;
|
||||
let req = serde_json::json!({ "prompt": prompt, "seed": seed });
|
||||
let body = post_with_status(port, "/generate", &req.to_string())?;
|
||||
let result: serde_json::Value = serde_json::from_str(&body)?;
|
||||
|
||||
if let Some(err) = result.get("error") {
|
||||
return Err(format!("Server error: {}", err).into());
|
||||
}
|
||||
println!("{}", result["text"].as_str().unwrap_or(""));
|
||||
eprintln!(
|
||||
"[{} tokens in {}ms — {:.1} t/s, prefill {}ms]",
|
||||
result["tokens_generated"],
|
||||
result["generation_time_ms"],
|
||||
result["tokens_per_sec"].as_f64().unwrap_or(0.0),
|
||||
result["prefill_time_ms"],
|
||||
);
|
||||
}
|
||||
Command::Batch { port, input } => {
|
||||
let file = std::fs::File::open(&input)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let payloads = prompt::parse_jsonl(reader)?;
|
||||
|
||||
let body = post_with_status(port, "/batch", &serde_json::to_string(&payloads)?)?;
|
||||
|
||||
for line in body.lines() {
|
||||
if line.is_empty() { continue; }
|
||||
let result: serde_json::Value = serde_json::from_str(line)?;
|
||||
let id = result["id"].as_str().unwrap_or("?");
|
||||
if let Some(err) = result.get("error") {
|
||||
eprintln!("--- {} --- ERROR: {}", id, err);
|
||||
} else {
|
||||
println!("--- {} ---", id);
|
||||
println!("{}", result["text"].as_str().unwrap_or(""));
|
||||
eprintln!(
|
||||
"[{} tokens in {}ms — {:.1} t/s]",
|
||||
result["tokens_generated"],
|
||||
result["generation_time_ms"],
|
||||
result["tokens_per_sec"].as_f64().unwrap_or(0.0),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Benchmark { port } => {
|
||||
let prompt = "Rephrase in terse dialect: The worker tends the crops in the field.";
|
||||
let runs = 5;
|
||||
eprintln!("Benchmark: {} runs", runs);
|
||||
|
||||
let mut total_tps = 0.0;
|
||||
let mut total_prefill = 0u64;
|
||||
let mut total_gen = 0u64;
|
||||
|
||||
for i in 0..runs {
|
||||
let req = serde_json::json!({ "prompt": prompt });
|
||||
let body = post_with_status(port, "/generate", &req.to_string())?;
|
||||
let result: serde_json::Value = serde_json::from_str(&body)?;
|
||||
|
||||
let tps = result["tokens_per_sec"].as_f64().unwrap_or(0.0);
|
||||
let prefill = result["prefill_time_ms"].as_u64().unwrap_or(0);
|
||||
let gen = result["generation_time_ms"].as_u64().unwrap_or(0);
|
||||
let tokens = result["tokens_generated"].as_u64().unwrap_or(0);
|
||||
|
||||
eprintln!(" run {}: {} tokens, {:.1} t/s, prefill {}ms", i + 1, tokens, tps, prefill);
|
||||
total_tps += tps;
|
||||
total_prefill += prefill;
|
||||
total_gen += gen;
|
||||
}
|
||||
|
||||
eprintln!("\n=== Benchmark Results ===");
|
||||
eprintln!(" Avg tokens/sec: {:.1}", total_tps / runs as f64);
|
||||
eprintln!(" Avg prefill: {}ms", total_prefill / runs);
|
||||
eprintln!(" Avg generation: {}ms", total_gen / runs);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_prompt(prompt_file: Option<String>) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let raw = match prompt_file {
|
||||
Some(path) => std::fs::read_to_string(&path)?,
|
||||
None => {
|
||||
let mut buf = String::new();
|
||||
std::io::stdin().read_to_string(&mut buf)?;
|
||||
buf
|
||||
}
|
||||
};
|
||||
let trimmed = raw.trim().to_string();
|
||||
if trimmed.is_empty() {
|
||||
return Err("No prompt provided".into());
|
||||
}
|
||||
Ok(trimmed)
|
||||
}
|
||||
|
||||
/// POST to the server. Prints "Server is processing..." if response takes > 500ms.
|
||||
fn post_with_status(port: u16, path: &str, body: &str) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let base = format!("http://127.0.0.1:{}", port);
|
||||
let agent = ureq::Agent::config_builder()
|
||||
.timeout_global(Some(Duration::from_secs(600)))
|
||||
.timeout_connect(Some(Duration::from_secs(2)))
|
||||
.build()
|
||||
.new_agent();
|
||||
|
||||
// Health check — clear error if server isn't running
|
||||
if agent.get(&format!("{}/health", base)).call().is_err() {
|
||||
return Err(format!(
|
||||
"No sr-voice server on port {}. Start one with: sr-voice serve --model <path>",
|
||||
port
|
||||
).into());
|
||||
}
|
||||
|
||||
let url = format!("{}{}", base, path);
|
||||
let start = Instant::now();
|
||||
let printed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let flag = printed.clone();
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
if !flag.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
eprint!("Server is processing...");
|
||||
flag.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
});
|
||||
|
||||
let result = agent.post(&url)
|
||||
.header("Content-Type", "application/json")
|
||||
.send(body);
|
||||
|
||||
let was_printed = printed.load(std::sync::atomic::Ordering::Relaxed);
|
||||
printed.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
let _ = handle.join();
|
||||
if was_printed {
|
||||
eprintln!(" done ({:.1}s)", start.elapsed().as_secs_f64());
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(response) => {
|
||||
let text = response.into_body().read_to_string()?;
|
||||
Ok(text)
|
||||
}
|
||||
Err(e) => Err(format!("Request failed: {}", e).into()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::BufRead;
|
||||
|
||||
use crate::VoiceError;
|
||||
|
||||
/// Content types for voice generation.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ContentType {
|
||||
Behavior,
|
||||
Dialogue,
|
||||
Tell,
|
||||
}
|
||||
|
||||
/// A single prompt payload, used in batch JSONL mode.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PromptPayload {
|
||||
pub id: String,
|
||||
pub content_type: ContentType,
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub base_text: Option<String>,
|
||||
#[serde(default)]
|
||||
pub semantic_core: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse a JSONL file into a list of prompt payloads.
|
||||
pub fn parse_jsonl(reader: impl BufRead) -> Result<Vec<PromptPayload>, VoiceError> {
|
||||
let mut payloads = Vec::new();
|
||||
for (i, line) in reader.lines().enumerate() {
|
||||
let line = line.map_err(|e| VoiceError::InvalidInput(format!("line {}: {}", i + 1, e)))?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let payload: PromptPayload = serde_json::from_str(trimmed)
|
||||
.map_err(|e| VoiceError::InvalidInput(format!("line {}: {}", i + 1, e)))?;
|
||||
payloads.push(payload);
|
||||
}
|
||||
Ok(payloads)
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use tiny_http::{Header, Method, Response, Server};
|
||||
|
||||
use crate::inference::InferenceEngine;
|
||||
use crate::prompt::PromptPayload;
|
||||
|
||||
const MAX_TOKENS: u32 = 64;
|
||||
const TEMPERATURE: f32 = 0.7;
|
||||
const TOP_P: f32 = 0.9;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct GenerateRequest {
|
||||
prompt: String,
|
||||
seed: Option<u32>,
|
||||
}
|
||||
|
||||
pub fn run_server(
|
||||
engine: InferenceEngine,
|
||||
port: u16,
|
||||
model_name: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let addr = format!("127.0.0.1:{}", port);
|
||||
let server = Server::http(&addr)
|
||||
.map_err(|e| format!("Failed to bind {}: {}", addr, e))?;
|
||||
|
||||
let start = Instant::now();
|
||||
eprintln!("sr-voice server ready on http://{}", addr);
|
||||
eprintln!(" model: {}", model_name);
|
||||
eprintln!(" POST /generate POST /batch GET /health");
|
||||
|
||||
for request in server.incoming_requests() {
|
||||
let path = request.url().to_string();
|
||||
let method = request.method().clone();
|
||||
|
||||
match (method, path.as_str()) {
|
||||
(Method::Get, "/health") => {
|
||||
let body = serde_json::json!({
|
||||
"status": "ready",
|
||||
"model": model_name,
|
||||
"uptime_secs": start.elapsed().as_secs(),
|
||||
});
|
||||
respond(request, 200, &body.to_string());
|
||||
}
|
||||
(Method::Post, "/generate") => handle_generate(&engine, request),
|
||||
(Method::Post, "/batch") => handle_batch(&engine, request),
|
||||
_ => {
|
||||
respond(request, 404, &serde_json::json!({"error": "not found"}).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_generate(engine: &InferenceEngine, mut request: tiny_http::Request) {
|
||||
let mut body = String::new();
|
||||
if std::io::Read::read_to_string(request.as_reader(), &mut body).is_err() {
|
||||
respond(request, 400, r#"{"error":"failed to read body"}"#);
|
||||
return;
|
||||
}
|
||||
|
||||
let req: GenerateRequest = match serde_json::from_str(&body) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
let msg = serde_json::json!({"error": format!("invalid JSON: {}", e)});
|
||||
respond(request, 400, &msg.to_string());
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
eprintln!(" generate: {} chars", req.prompt.len());
|
||||
match engine.generate(&req.prompt, MAX_TOKENS, TEMPERATURE, TOP_P, req.seed) {
|
||||
Ok(result) => {
|
||||
eprintln!(" -> {} tokens, {:.1} t/s", result.tokens_generated, result.tokens_per_sec);
|
||||
respond(request, 200, &serde_json::to_string(&result).unwrap());
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = serde_json::json!({"error": e.to_string()});
|
||||
respond(request, 500, &msg.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_batch(engine: &InferenceEngine, mut request: tiny_http::Request) {
|
||||
let mut body = String::new();
|
||||
if std::io::Read::read_to_string(request.as_reader(), &mut body).is_err() {
|
||||
respond(request, 400, r#"{"error":"failed to read body"}"#);
|
||||
return;
|
||||
}
|
||||
|
||||
let payloads: Vec<PromptPayload> = match serde_json::from_str(&body) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
let msg = serde_json::json!({"error": format!("invalid JSON: {}", e)});
|
||||
respond(request, 400, &msg.to_string());
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
eprintln!(" batch: {} prompts", payloads.len());
|
||||
let mut output = String::new();
|
||||
for payload in &payloads {
|
||||
match engine.generate(&payload.prompt, MAX_TOKENS, TEMPERATURE, TOP_P, None) {
|
||||
Ok(result) => {
|
||||
eprintln!(" -> {}: {} tokens, {:.1} t/s", payload.id, result.tokens_generated, result.tokens_per_sec);
|
||||
#[derive(serde::Serialize)]
|
||||
struct BatchLine<'a> {
|
||||
id: &'a str,
|
||||
#[serde(flatten)]
|
||||
result: &'a crate::inference::GenerationResult,
|
||||
}
|
||||
let line = serde_json::to_string(&BatchLine { id: &payload.id, result: &result }).unwrap();
|
||||
output.push_str(&line);
|
||||
output.push('\n');
|
||||
}
|
||||
Err(e) => {
|
||||
let line = serde_json::json!({"id": payload.id, "error": e.to_string()});
|
||||
output.push_str(&line.to_string());
|
||||
output.push('\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
respond(request, 200, &output);
|
||||
}
|
||||
|
||||
fn respond(request: tiny_http::Request, status: u16, body: &str) {
|
||||
let header = Header::from_bytes("Content-Type", "application/json").unwrap();
|
||||
let response = Response::from_string(body)
|
||||
.with_status_code(status)
|
||||
.with_header(header);
|
||||
let _ = request.respond(response);
|
||||
}
|
||||
Reference in New Issue
Block a user