diff --git a/.gitignore b/.gitignore index ce6b5e78d..bfe8e28e7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ .cache/ .tmp/ server/target/ +server/sr-voice/target/ +server/models/ tooling/content-converter/target/ tooling/line-previewer/target/ tooling/test-client/target/ diff --git a/Makefile b/Makefile index 99471422b..68566bbc6 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ GODOT := $(shell command -v godot4 2>/dev/null || command -v godot 2>/dev/null) pre-pr-server pre-pr-client pre-pr-content \ fixtures-client fixtures-gauntlet golden-diff golden-update \ checklist-validate checklist-generate \ + build-sr-voice run-sr-voice \ perf-baseline debug-schedule \ test-ipc-fixtures test-ipc-protocol test-ipc-integration test-ipc-benchmark \ screenshot visual-movie test-visual visual-update @@ -65,6 +66,10 @@ help: @echo " make pre-pr-content Content-scoped pre-PR (schema + cross-ref validation)" @echo "" @echo " make setup-hooks Install pre-commit hooks (included in setup)" + @echo " make build-sr-voice Build sr-voice LLM inference service" + @echo " make serve-sr-voice Start sr-voice server (ARGS='--model ')" + @echo " make run-sr-voice Submit to sr-voice server (ARGS='generate|batch|benchmark ...')" + @echo " make stop-sr-voice Stop sr-voice server" @echo " make debug-schedule Print bevy_ecs schedule graph (diff for PR artifacts)" @echo "" @echo " GODOT_VERSION=4.6 make setup Override Godot version" @@ -344,6 +349,25 @@ test-visual: visual-update: @tests/run-visual --update +LIBCLANG_PATH ?= /usr/lib64/rocm/llvm/lib +BINDGEN_CLANG_ARGS ?= -I/usr/lib64/rocm/llvm/lib/clang/19/include +SR_VOICE_ENV = LIBCLANG_PATH=$(LIBCLANG_PATH) BINDGEN_EXTRA_CLANG_ARGS="$(BINDGEN_CLANG_ARGS)" + +SR_VOICE_PORT ?= 8321 + +build-sr-voice: + cd server/sr-voice && $(SR_VOICE_ENV) cargo build --release + +serve-sr-voice: + cd server/sr-voice && $(SR_VOICE_ENV) cargo run --release -- serve $(ARGS) + +run-sr-voice: + cd server/sr-voice && $(SR_VOICE_ENV) cargo run --release -- $(ARGS) + +stop-sr-voice: + @lsof -ti :$(SR_VOICE_PORT) | xargs -r kill 2>/dev/null || true + @echo "Stopped sr-voice on port $(SR_VOICE_PORT)" + content-ron: cd tooling/content-converter && cargo build --release tooling/content-converter/target/release/content-converter --input content --output content-ron --verbose diff --git a/server/sr-voice/Cargo.lock b/server/sr-voice/Cargo.lock new file mode 100644 index 000000000..bcc6a5547 --- /dev/null +++ b/server/sr-voice/Cargo.lock @@ -0,0 +1,964 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "find_cuda_helper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad" +dependencies = [ + "glob", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "llama-cpp-2" +version = "0.1.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2947ab625c59d1fdf42e61f538c3fa66f43de2f78316971920873f359483d1d8" +dependencies = [ + "encoding_rs", + "enumflags2", + "llama-cpp-sys-2", + "thiserror", + "tracing", + "tracing-core", +] + +[[package]] +name = "llama-cpp-sys-2" +version = "0.1.138" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a529006bf16af70c7485ba957820dc2bc9467d75697e97970c81d2da73c76f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "find_cuda_helper", + "glob", + "walkdir", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "sr-voice" +version = "0.1.0" +dependencies = [ + "clap", + "llama-cpp-2", + "serde", + "serde_json", + "thiserror", + "tiny_http", + "ureq", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc" +dependencies = [ + "base64", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/server/sr-voice/Cargo.toml b/server/sr-voice/Cargo.toml new file mode 100644 index 000000000..291212862 --- /dev/null +++ b/server/sr-voice/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sr-voice" +version = "0.1.0" +edition = "2021" + +[dependencies] +llama-cpp-2 = "0.1" +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +tiny_http = "0.12" +ureq = "3" + +[features] +default = [] +metal = ["llama-cpp-2/metal"] +vulkan = ["llama-cpp-2/vulkan"] +cuda = ["llama-cpp-2/cuda"] +rocm = ["llama-cpp-2/rocm"] diff --git a/server/sr-voice/src/inference.rs b/server/sr-voice/src/inference.rs new file mode 100644 index 000000000..ee1729575 --- /dev/null +++ b/server/sr-voice/src/inference.rs @@ -0,0 +1,165 @@ +use std::num::NonZeroU32; +use std::path::Path; +use std::time::Instant; + +use llama_cpp_2::context::params::LlamaContextParams; +use llama_cpp_2::llama_backend::LlamaBackend; +use llama_cpp_2::llama_batch::LlamaBatch; +use llama_cpp_2::model::params::LlamaModelParams; +use llama_cpp_2::model::{AddBos, LlamaModel, Special}; +use llama_cpp_2::sampling::LlamaSampler; + +use crate::VoiceError; + +/// Configuration for model loading and inference. +pub struct InferenceConfig { + pub model_path: String, + pub threads: u32, + pub ctx_size: u32, + pub seed: Option, +} + +/// Result of a single generation call. +#[derive(serde::Serialize)] +pub struct GenerationResult { + pub text: String, + pub tokens_generated: u32, + pub generation_time_ms: u64, + pub tokens_per_sec: f64, + pub prefill_time_ms: u64, +} + +/// Wraps llama.cpp model and context for text generation. +pub struct InferenceEngine { + backend: LlamaBackend, + model: LlamaModel, + ctx_size: u32, + threads: u32, +} + +impl InferenceEngine { + /// Load a GGUF model from disk. + pub fn load(config: &InferenceConfig) -> Result { + let backend = + LlamaBackend::init().map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?; + + let model_params = LlamaModelParams::default(); + let model = LlamaModel::load_from_file( + &backend, + Path::new(&config.model_path), + &model_params, + ) + .map_err(|e| VoiceError::ModelLoadFailed(e.to_string()))?; + + Ok(Self { + backend, + model, + ctx_size: config.ctx_size, + threads: config.threads, + }) + } + + /// Generate text from a prompt. + pub fn generate( + &self, + prompt: &str, + max_tokens: u32, + temperature: f32, + top_p: f32, + seed: Option, + ) -> Result { + let ctx_params = LlamaContextParams::default() + .with_n_ctx(NonZeroU32::new(self.ctx_size)) + .with_n_threads(self.threads as i32) + .with_n_threads_batch(self.threads as i32); + + let mut ctx = self + .model + .new_context(&self.backend, ctx_params) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + + // Tokenize the prompt + let tokens = self + .model + .str_to_token(prompt, AddBos::Always) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + + if tokens.len() as u32 >= self.ctx_size { + return Err(VoiceError::InferenceFailed(format!( + "Prompt ({} tokens) exceeds context size ({})", + tokens.len(), + self.ctx_size + ))); + } + + // Prefill: evaluate the prompt tokens + let prefill_start = Instant::now(); + let mut batch = LlamaBatch::new(self.ctx_size as usize, 1); + for (i, &token) in tokens.iter().enumerate() { + let is_last = i == tokens.len() - 1; + batch + .add(token, i as i32, &[0], is_last) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + } + ctx.decode(&mut batch) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + let prefill_time_ms = prefill_start.elapsed().as_millis() as u64; + + // Generation loop + let gen_start = Instant::now(); + let mut generated_tokens: u32 = 0; + let mut output = String::new(); + let mut cur_pos = tokens.len() as i32; + + let mut sampler = LlamaSampler::chain_simple([ + LlamaSampler::temp(temperature), + LlamaSampler::top_p(top_p, 1), + LlamaSampler::dist(seed.unwrap_or(1234)), + ]); + + loop { + if generated_tokens >= max_tokens { + break; + } + + let logits_index = batch.n_tokens() - 1; + let token = sampler.sample(&ctx, logits_index); + + if self.model.is_eog_token(token) { + break; + } + + #[allow(deprecated)] + let piece = self + .model + .token_to_str(token, Special::Tokenize) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + output.push_str(&piece); + generated_tokens += 1; + + batch.clear(); + batch + .add(token, cur_pos, &[0], true) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + cur_pos += 1; + + ctx.decode(&mut batch) + .map_err(|e| VoiceError::InferenceFailed(e.to_string()))?; + } + + let generation_time_ms = gen_start.elapsed().as_millis() as u64; + let tokens_per_sec = if generation_time_ms > 0 { + (generated_tokens as f64 / generation_time_ms as f64) * 1000.0 + } else { + 0.0 + }; + + Ok(GenerationResult { + text: output, + tokens_generated: generated_tokens, + generation_time_ms, + tokens_per_sec, + prefill_time_ms, + }) + } +} diff --git a/server/sr-voice/src/main.rs b/server/sr-voice/src/main.rs new file mode 100644 index 000000000..65ba223e8 --- /dev/null +++ b/server/sr-voice/src/main.rs @@ -0,0 +1,248 @@ +mod inference; +mod prompt; +mod server; + +use std::io::Read; +use std::time::{Duration, Instant}; + +use clap::{Parser, Subcommand}; +use inference::{InferenceConfig, InferenceEngine}; + +/// Errors for the sr-voice CLI. +#[derive(thiserror::Error, Debug)] +pub enum VoiceError { + #[error("model load failed: {0}")] + ModelLoadFailed(String), + #[error("inference failed: {0}")] + InferenceFailed(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +/// sr-voice — LLM inference service for The Settled Reach +#[derive(Parser)] +#[command(name = "sr-voice", version, about)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Start the inference server (loads model, listens for requests) + Serve { + /// Path to GGUF model file + #[arg(long)] + model: String, + /// Listen port + #[arg(long, default_value = "8321")] + port: u16, + /// CPU threads for inference + #[arg(long)] + threads: Option, + /// Context window size in tokens + #[arg(long, default_value = "512")] + ctx_size: u32, + }, + /// Generate text from a single prompt (requires running server) + Generate { + /// Server port + #[arg(long, default_value = "8321")] + port: u16, + /// RNG seed + #[arg(long)] + seed: Option, + /// Prompt file (reads from stdin if omitted) + prompt_file: Option, + }, + /// Process a JSONL batch of prompts (requires running server) + Batch { + /// Server port + #[arg(long, default_value = "8321")] + port: u16, + /// Input JSONL file + #[arg(long)] + input: String, + }, + /// Run 5 inferences and report average tokens/sec (requires running server) + Benchmark { + /// Server port + #[arg(long, default_value = "8321")] + port: u16, + }, +} + +fn default_threads() -> u32 { + let cores = std::thread::available_parallelism() + .map(|n| n.get() as u32) + .unwrap_or(4); + cores.saturating_sub(1).max(1) +} + +fn main() -> Result<(), Box> { + let cli = Cli::parse(); + + match cli.command { + Command::Serve { model, port, threads, ctx_size } => { + let threads = threads.unwrap_or_else(default_threads); + let config = InferenceConfig { + model_path: model.clone(), + threads, + ctx_size, + seed: None, + }; + + eprintln!("Loading model: {}", config.model_path); + let engine = InferenceEngine::load(&config)?; + eprintln!("Model loaded ({} threads, {} ctx)", threads, ctx_size); + + let model_name = std::path::Path::new(&model) + .file_name() + .map(|f| f.to_string_lossy().to_string()) + .unwrap_or(model); + + server::run_server(engine, port, &model_name)?; + } + Command::Generate { port, seed, prompt_file } => { + let prompt = read_prompt(prompt_file)?; + let req = serde_json::json!({ "prompt": prompt, "seed": seed }); + let body = post_with_status(port, "/generate", &req.to_string())?; + let result: serde_json::Value = serde_json::from_str(&body)?; + + if let Some(err) = result.get("error") { + return Err(format!("Server error: {}", err).into()); + } + println!("{}", result["text"].as_str().unwrap_or("")); + eprintln!( + "[{} tokens in {}ms — {:.1} t/s, prefill {}ms]", + result["tokens_generated"], + result["generation_time_ms"], + result["tokens_per_sec"].as_f64().unwrap_or(0.0), + result["prefill_time_ms"], + ); + } + Command::Batch { port, input } => { + let file = std::fs::File::open(&input)?; + let reader = std::io::BufReader::new(file); + let payloads = prompt::parse_jsonl(reader)?; + + let body = post_with_status(port, "/batch", &serde_json::to_string(&payloads)?)?; + + for line in body.lines() { + if line.is_empty() { continue; } + let result: serde_json::Value = serde_json::from_str(line)?; + let id = result["id"].as_str().unwrap_or("?"); + if let Some(err) = result.get("error") { + eprintln!("--- {} --- ERROR: {}", id, err); + } else { + println!("--- {} ---", id); + println!("{}", result["text"].as_str().unwrap_or("")); + eprintln!( + "[{} tokens in {}ms — {:.1} t/s]", + result["tokens_generated"], + result["generation_time_ms"], + result["tokens_per_sec"].as_f64().unwrap_or(0.0), + ); + } + } + } + Command::Benchmark { port } => { + let prompt = "Rephrase in terse dialect: The worker tends the crops in the field."; + let runs = 5; + eprintln!("Benchmark: {} runs", runs); + + let mut total_tps = 0.0; + let mut total_prefill = 0u64; + let mut total_gen = 0u64; + + for i in 0..runs { + let req = serde_json::json!({ "prompt": prompt }); + let body = post_with_status(port, "/generate", &req.to_string())?; + let result: serde_json::Value = serde_json::from_str(&body)?; + + let tps = result["tokens_per_sec"].as_f64().unwrap_or(0.0); + let prefill = result["prefill_time_ms"].as_u64().unwrap_or(0); + let gen = result["generation_time_ms"].as_u64().unwrap_or(0); + let tokens = result["tokens_generated"].as_u64().unwrap_or(0); + + eprintln!(" run {}: {} tokens, {:.1} t/s, prefill {}ms", i + 1, tokens, tps, prefill); + total_tps += tps; + total_prefill += prefill; + total_gen += gen; + } + + eprintln!("\n=== Benchmark Results ==="); + eprintln!(" Avg tokens/sec: {:.1}", total_tps / runs as f64); + eprintln!(" Avg prefill: {}ms", total_prefill / runs); + eprintln!(" Avg generation: {}ms", total_gen / runs); + } + } + + Ok(()) +} + +fn read_prompt(prompt_file: Option) -> Result> { + let raw = match prompt_file { + Some(path) => std::fs::read_to_string(&path)?, + None => { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + buf + } + }; + let trimmed = raw.trim().to_string(); + if trimmed.is_empty() { + return Err("No prompt provided".into()); + } + Ok(trimmed) +} + +/// POST to the server. Prints "Server is processing..." if response takes > 500ms. +fn post_with_status(port: u16, path: &str, body: &str) -> Result> { + let base = format!("http://127.0.0.1:{}", port); + let agent = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(600))) + .timeout_connect(Some(Duration::from_secs(2))) + .build() + .new_agent(); + + // Health check — clear error if server isn't running + if agent.get(&format!("{}/health", base)).call().is_err() { + return Err(format!( + "No sr-voice server on port {}. Start one with: sr-voice serve --model ", + port + ).into()); + } + + let url = format!("{}{}", base, path); + let start = Instant::now(); + let printed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = printed.clone(); + + let handle = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(500)); + if !flag.load(std::sync::atomic::Ordering::Relaxed) { + eprint!("Server is processing..."); + flag.store(true, std::sync::atomic::Ordering::Relaxed); + } + }); + + let result = agent.post(&url) + .header("Content-Type", "application/json") + .send(body); + + let was_printed = printed.load(std::sync::atomic::Ordering::Relaxed); + printed.store(true, std::sync::atomic::Ordering::Relaxed); + let _ = handle.join(); + if was_printed { + eprintln!(" done ({:.1}s)", start.elapsed().as_secs_f64()); + } + + match result { + Ok(response) => { + let text = response.into_body().read_to_string()?; + Ok(text) + } + Err(e) => Err(format!("Request failed: {}", e).into()), + } +} diff --git a/server/sr-voice/src/prompt.rs b/server/sr-voice/src/prompt.rs new file mode 100644 index 000000000..6ab06be33 --- /dev/null +++ b/server/sr-voice/src/prompt.rs @@ -0,0 +1,41 @@ +use serde::{Deserialize, Serialize}; +use std::io::BufRead; + +use crate::VoiceError; + +/// Content types for voice generation. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContentType { + Behavior, + Dialogue, + Tell, +} + +/// A single prompt payload, used in batch JSONL mode. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PromptPayload { + pub id: String, + pub content_type: ContentType, + pub prompt: String, + #[serde(default)] + pub base_text: Option, + #[serde(default)] + pub semantic_core: Option, +} + +/// Parse a JSONL file into a list of prompt payloads. +pub fn parse_jsonl(reader: impl BufRead) -> Result, VoiceError> { + let mut payloads = Vec::new(); + for (i, line) in reader.lines().enumerate() { + let line = line.map_err(|e| VoiceError::InvalidInput(format!("line {}: {}", i + 1, e)))?; + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let payload: PromptPayload = serde_json::from_str(trimmed) + .map_err(|e| VoiceError::InvalidInput(format!("line {}: {}", i + 1, e)))?; + payloads.push(payload); + } + Ok(payloads) +} diff --git a/server/sr-voice/src/server.rs b/server/sr-voice/src/server.rs new file mode 100644 index 000000000..004303386 --- /dev/null +++ b/server/sr-voice/src/server.rs @@ -0,0 +1,134 @@ +use std::time::Instant; + +use tiny_http::{Header, Method, Response, Server}; + +use crate::inference::InferenceEngine; +use crate::prompt::PromptPayload; + +const MAX_TOKENS: u32 = 64; +const TEMPERATURE: f32 = 0.7; +const TOP_P: f32 = 0.9; + +#[derive(serde::Deserialize)] +struct GenerateRequest { + prompt: String, + seed: Option, +} + +pub fn run_server( + engine: InferenceEngine, + port: u16, + model_name: &str, +) -> Result<(), Box> { + let addr = format!("127.0.0.1:{}", port); + let server = Server::http(&addr) + .map_err(|e| format!("Failed to bind {}: {}", addr, e))?; + + let start = Instant::now(); + eprintln!("sr-voice server ready on http://{}", addr); + eprintln!(" model: {}", model_name); + eprintln!(" POST /generate POST /batch GET /health"); + + for request in server.incoming_requests() { + let path = request.url().to_string(); + let method = request.method().clone(); + + match (method, path.as_str()) { + (Method::Get, "/health") => { + let body = serde_json::json!({ + "status": "ready", + "model": model_name, + "uptime_secs": start.elapsed().as_secs(), + }); + respond(request, 200, &body.to_string()); + } + (Method::Post, "/generate") => handle_generate(&engine, request), + (Method::Post, "/batch") => handle_batch(&engine, request), + _ => { + respond(request, 404, &serde_json::json!({"error": "not found"}).to_string()); + } + } + } + + Ok(()) +} + +fn handle_generate(engine: &InferenceEngine, mut request: tiny_http::Request) { + let mut body = String::new(); + if std::io::Read::read_to_string(request.as_reader(), &mut body).is_err() { + respond(request, 400, r#"{"error":"failed to read body"}"#); + return; + } + + let req: GenerateRequest = match serde_json::from_str(&body) { + Ok(r) => r, + Err(e) => { + let msg = serde_json::json!({"error": format!("invalid JSON: {}", e)}); + respond(request, 400, &msg.to_string()); + return; + } + }; + + eprintln!(" generate: {} chars", req.prompt.len()); + match engine.generate(&req.prompt, MAX_TOKENS, TEMPERATURE, TOP_P, req.seed) { + Ok(result) => { + eprintln!(" -> {} tokens, {:.1} t/s", result.tokens_generated, result.tokens_per_sec); + respond(request, 200, &serde_json::to_string(&result).unwrap()); + } + Err(e) => { + let msg = serde_json::json!({"error": e.to_string()}); + respond(request, 500, &msg.to_string()); + } + } +} + +fn handle_batch(engine: &InferenceEngine, mut request: tiny_http::Request) { + let mut body = String::new(); + if std::io::Read::read_to_string(request.as_reader(), &mut body).is_err() { + respond(request, 400, r#"{"error":"failed to read body"}"#); + return; + } + + let payloads: Vec = match serde_json::from_str(&body) { + Ok(p) => p, + Err(e) => { + let msg = serde_json::json!({"error": format!("invalid JSON: {}", e)}); + respond(request, 400, &msg.to_string()); + return; + } + }; + + eprintln!(" batch: {} prompts", payloads.len()); + let mut output = String::new(); + for payload in &payloads { + match engine.generate(&payload.prompt, MAX_TOKENS, TEMPERATURE, TOP_P, None) { + Ok(result) => { + eprintln!(" -> {}: {} tokens, {:.1} t/s", payload.id, result.tokens_generated, result.tokens_per_sec); + #[derive(serde::Serialize)] + struct BatchLine<'a> { + id: &'a str, + #[serde(flatten)] + result: &'a crate::inference::GenerationResult, + } + let line = serde_json::to_string(&BatchLine { id: &payload.id, result: &result }).unwrap(); + output.push_str(&line); + output.push('\n'); + } + Err(e) => { + let line = serde_json::json!({"id": payload.id, "error": e.to_string()}); + output.push_str(&line.to_string()); + output.push('\n'); + } + } + } + + respond(request, 200, &output); +} + +fn respond(request: tiny_http::Request, status: u16, body: &str) { + let header = Header::from_bytes("Content-Type", "application/json").unwrap(); + let response = Response::from_string(body) + .with_status_code(status) + .with_header(header); + let _ = request.respond(response); +}