jpmschweitzerandClaude Opus 4.5
363ab378af
- Rewrite system prompt with negative constraints and step-by-step process - Set temperature to 0.1 for deterministic tool calling - Sort room groups to top of device list (address positional bias) - Add [ROOM GROUP] marker in list_devices output - Update tool docstrings with explicit entity_id= parameter examples - Add optimization findings doc (experiment log: 0% → 100% success) - Add test script for room group detection regression testing 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
142 lines
4.1 KiB
Bash
Executable File
142 lines
4.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Housekeeper Room Group Detection Test Suite
|
|
# Verifies room groups are controlled by checking actual state changes
|
|
|
|
API_URL="http://localhost:8777/v1/chat/completions"
|
|
CORE_API="http://192.168.86.149:8083"
|
|
RESULTS_FILE="/tmp/housekeeper_test_results.txt"
|
|
|
|
GREEN='\033[0;32m'
|
|
RED='\033[0;31m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
get_state() {
|
|
curl -s "$CORE_API/housekeeping/devices/$1" 2>/dev/null | jq -r '.state' 2>/dev/null
|
|
}
|
|
|
|
echo "=========================================="
|
|
echo "Housekeeper Room Group Test Suite"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
> "$RESULTS_FILE"
|
|
|
|
run_toggle_test() {
|
|
local test_num=$1
|
|
local room=$2
|
|
local entity="light.$room"
|
|
local prompt_room="${room//_/ }"
|
|
|
|
printf "Test %2d: Toggle %-12s lights ... " "$test_num" "$prompt_room"
|
|
|
|
local before=$(get_state "$entity")
|
|
if [ -z "$before" ] || [ "$before" = "null" ]; then
|
|
echo -e "${YELLOW}SKIP${NC} (cannot get state)"
|
|
echo "SKIP|$test_num|Toggle $room|error" >> "$RESULTS_FILE"
|
|
return
|
|
fi
|
|
|
|
curl -s -X POST "$API_URL" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Toggle the $prompt_room lights\"}]}" > /dev/null
|
|
|
|
sleep 4
|
|
|
|
local after=$(get_state "$entity")
|
|
|
|
if [ "$before" != "$after" ]; then
|
|
echo -e "${GREEN}PASS${NC} ($before -> $after)"
|
|
echo "PASS|$test_num|Toggle $room|$before->$after" >> "$RESULTS_FILE"
|
|
else
|
|
echo -e "${RED}FAIL${NC} (state unchanged: $before)"
|
|
echo "FAIL|$test_num|Toggle $room|unchanged:$before" >> "$RESULTS_FILE"
|
|
fi
|
|
}
|
|
|
|
run_onoff_test() {
|
|
local test_num=$1
|
|
local room=$2
|
|
local action=$3
|
|
local expected_state=$4
|
|
# Entity uses underscore, prompt uses space
|
|
local entity="light.${room//_/ }"
|
|
entity="light.$room"
|
|
local prompt_room="${room//_/ }"
|
|
|
|
printf "Test %2d: %-8s %-12s lights ... " "$test_num" "$action" "$prompt_room"
|
|
|
|
curl -s -X POST "$API_URL" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"$action the $prompt_room lights\"}]}" > /dev/null
|
|
|
|
sleep 4
|
|
|
|
local after=$(get_state "$entity")
|
|
|
|
if [ "$after" = "$expected_state" ]; then
|
|
echo -e "${GREEN}PASS${NC} ($after)"
|
|
echo "PASS|$test_num|$action $room|$after" >> "$RESULTS_FILE"
|
|
else
|
|
echo -e "${RED}FAIL${NC} (got $after, expected $expected_state)"
|
|
echo "FAIL|$test_num|$action $room|got:$after,expected:$expected_state" >> "$RESULTS_FILE"
|
|
fi
|
|
}
|
|
|
|
echo "Running tests (~4s each)..."
|
|
echo ""
|
|
|
|
# Study tests
|
|
run_onoff_test 1 "study" "Turn off" "off"
|
|
run_onoff_test 2 "study" "Turn on" "on"
|
|
run_toggle_test 3 "study"
|
|
|
|
# Kitchen tests
|
|
run_onoff_test 4 "kitchen" "Turn off" "off"
|
|
run_onoff_test 5 "kitchen" "Turn on" "on"
|
|
run_toggle_test 6 "kitchen"
|
|
|
|
# Bedroom tests
|
|
run_onoff_test 7 "bedroom" "Turn off" "off"
|
|
run_onoff_test 8 "bedroom" "Turn on" "on"
|
|
|
|
# Living room tests (entity is light.living_room)
|
|
run_onoff_test 9 "living_room" "Turn off" "off"
|
|
run_onoff_test 10 "living_room" "Turn on" "on"
|
|
|
|
# Ensure all lights end up ON
|
|
echo ""
|
|
echo "Restoring all lights to ON..."
|
|
for room in "study" "kitchen" "bedroom" "living room"; do
|
|
curl -s -X POST "$API_URL" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Turn on the $room lights\"}]}" > /dev/null
|
|
sleep 3
|
|
done
|
|
echo "Done."
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Results"
|
|
echo "=========================================="
|
|
|
|
PASS=$(grep -c "^PASS" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
|
FAIL=$(grep -c "^FAIL" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
|
SKIP=$(grep -c "^SKIP" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
|
TOTAL=$((PASS + FAIL))
|
|
|
|
echo "Passed: $PASS"
|
|
echo "Failed: $FAIL"
|
|
echo "Skipped: $SKIP"
|
|
|
|
if [ "$TOTAL" -gt 0 ]; then
|
|
echo ""
|
|
echo "Success Rate: $((PASS * 100 / TOTAL))% ($PASS/$TOTAL)"
|
|
fi
|
|
|
|
if [ "$FAIL" -gt 0 ]; then
|
|
echo ""
|
|
echo "Failures:"
|
|
grep "^FAIL" "$RESULTS_FILE"
|
|
fi
|