Refactor into modular app structure
Split monolithic files into focused modules: - app/core: settings, logging, lifecycle - app/signaling: websocket server, ICE parser, message models - app/webrtc: peer session, video receiver, frame source - app/vision: pose landmarker wrapper, model config, pose types - app/exercises/dead_bug: detector, metrics, rules, state machine, types - app/rendering: skeleton renderer, status overlay, window display - app/audio: rep announcer - app/diagnostics: perf timer, crash handler - configs: environment-based settings - tests: unit tests for rules, state machine, ICE parser - run.py: entry point
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class RepAnnouncer:
|
||||
def __init__(self, *, enabled: bool = True, rate: int = 185, volume: float = 1.0) -> None:
|
||||
self.enabled = enabled
|
||||
self.rate = rate
|
||||
self.volume = volume
|
||||
self._queue: queue.Queue[str | None] = queue.Queue()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._engine: Any | None = None
|
||||
self._use_macos_say = False
|
||||
self._current_process: subprocess.Popen | None = None
|
||||
|
||||
if self.enabled:
|
||||
self._start()
|
||||
|
||||
def announce_count(self, count: int) -> None:
|
||||
if not self.enabled or count <= 0:
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
self._queue.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._queue.put(str(count))
|
||||
|
||||
def close(self) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
self._queue.put(None)
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=1.0)
|
||||
if self._current_process is not None and self._current_process.poll() is None:
|
||||
self._current_process.terminate()
|
||||
|
||||
def _start(self) -> None:
|
||||
if sys.platform == "darwin":
|
||||
self._use_macos_say = True
|
||||
logger.info("Rep announcer initialized with macOS say")
|
||||
else:
|
||||
try:
|
||||
import pyttsx3
|
||||
|
||||
self._engine = pyttsx3.init()
|
||||
self._engine.setProperty("rate", self.rate)
|
||||
self._engine.setProperty("volume", self.volume)
|
||||
logger.info("Rep announcer initialized with pyttsx3")
|
||||
except Exception as exc:
|
||||
self.enabled = False
|
||||
logger.warning("Rep announcer disabled, pyttsx3 unavailable: {}", exc)
|
||||
return
|
||||
|
||||
self._thread = threading.Thread(target=self._run, name="RepAnnouncer", daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
text = self._queue.get()
|
||||
if text is None:
|
||||
return
|
||||
|
||||
try:
|
||||
if self._use_macos_say:
|
||||
if self._current_process is not None and self._current_process.poll() is None:
|
||||
self._current_process.terminate()
|
||||
self._current_process = subprocess.Popen(
|
||||
["say", "-r", str(self.rate), text],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
elif self._engine is not None:
|
||||
self._engine.say(text)
|
||||
self._engine.runAndWait()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to announce rep count {}: {}", text, exc)
|
||||
@@ -0,0 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.diagnostics.crash_handler import enable_crash_handler
|
||||
from configs.default import LOG_DIR
|
||||
|
||||
|
||||
def startup() -> None:
|
||||
enable_crash_handler(LOG_DIR)
|
||||
from app.core.logging import setup_logging
|
||||
setup_logging()
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from configs.default import LOG_DIR, LOG_RETENTION, LOG_ROTATION
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.add(
|
||||
LOG_DIR / "posefit-server_{time:YYYY-MM-DD}.log",
|
||||
rotation=LOG_ROTATION,
|
||||
retention=LOG_RETENTION,
|
||||
enqueue=True,
|
||||
backtrace=True,
|
||||
diagnose=True,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import faulthandler
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def enable_crash_handler(log_dir: str | Path) -> None:
|
||||
log_dir = Path(log_dir)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
crash_log = open(log_dir / "posefit-crash.log", "a", buffering=1)
|
||||
faulthandler.enable(file=crash_log, all_threads=True)
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class PerfTimer:
|
||||
def __init__(self, name: str = "") -> None:
|
||||
self.name = name
|
||||
self._start = 0.0
|
||||
self._elapsed = 0.0
|
||||
|
||||
def start(self) -> PerfTimer:
|
||||
self._start = time.perf_counter()
|
||||
return self
|
||||
|
||||
def stop(self) -> float:
|
||||
self._elapsed = time.perf_counter() - self._start
|
||||
return self._elapsed
|
||||
|
||||
@property
|
||||
def elapsed_ms(self) -> float:
|
||||
return self._elapsed * 1000
|
||||
|
||||
|
||||
@contextmanager
|
||||
def measure(name: str = ""):
|
||||
timer = PerfTimer(name).start()
|
||||
yield timer
|
||||
elapsed = timer.stop()
|
||||
if name:
|
||||
logger.debug("{} took {:.1f}ms", name, timer.elapsed_ms)
|
||||
@@ -0,0 +1,172 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
from app.exercises.dead_bug.metrics import calculate_metrics
|
||||
from app.exercises.dead_bug.rules import has_required_visibility
|
||||
from app.exercises.dead_bug.state_machine import DeadBugStateMachine
|
||||
from app.exercises.dead_bug.types import DeadBugMetrics, DeadBugPhase, DeadBugResult, Point
|
||||
from app.rendering.overlay_renderer import draw_status_overlay
|
||||
from app.rendering.skeleton_renderer import draw_landmarks
|
||||
from app.vision.frame_utils import bgr_to_rgba
|
||||
from app.vision.pose_landmarker import PoseLandmarkerWrapper
|
||||
from app.vision.pose_types import (
|
||||
LEFT_ANKLE,
|
||||
LEFT_ELBOW,
|
||||
LEFT_HIP,
|
||||
LEFT_KNEE,
|
||||
LEFT_SHOULDER,
|
||||
LEFT_WRIST,
|
||||
REQUIRED_LANDMARKS,
|
||||
RIGHT_ANKLE,
|
||||
RIGHT_ELBOW,
|
||||
RIGHT_HIP,
|
||||
RIGHT_KNEE,
|
||||
RIGHT_SHOULDER,
|
||||
RIGHT_WRIST,
|
||||
)
|
||||
|
||||
|
||||
class DeadBugDetector:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model_path: str | None = None,
|
||||
visibility_threshold: float = 0.45,
|
||||
extension_confirm_frames: int = 4,
|
||||
reset_confirm_frames: int = 3,
|
||||
prefer_gpu: bool = True,
|
||||
) -> None:
|
||||
self.visibility_threshold = visibility_threshold
|
||||
|
||||
self._latest_result = None
|
||||
self._result_lock = threading.Lock()
|
||||
self._result_event = threading.Event()
|
||||
self._inflight = False
|
||||
self._inflight_started_at = 0.0
|
||||
|
||||
def on_result(pose_result, _image, _timestamp_ms):
|
||||
with self._result_lock:
|
||||
self._latest_result = pose_result
|
||||
self._inflight = False
|
||||
self._inflight_started_at = 0.0
|
||||
self._result_event.set()
|
||||
|
||||
self._landmarker = PoseLandmarkerWrapper(
|
||||
model_path=model_path,
|
||||
prefer_gpu=prefer_gpu,
|
||||
result_callback=on_result,
|
||||
)
|
||||
|
||||
self._state = DeadBugStateMachine(
|
||||
extension_confirm_frames=extension_confirm_frames,
|
||||
reset_confirm_frames=reset_confirm_frames,
|
||||
)
|
||||
|
||||
self._last_timestamp_ms = -1
|
||||
|
||||
def close(self) -> None:
|
||||
self._landmarker.close()
|
||||
|
||||
def process_frame(self, bgr_frame: np.ndarray, timestamp_ms: int) -> tuple[np.ndarray, DeadBugResult]:
|
||||
timestamp_ms = self._normalize_timestamp(timestamp_ms)
|
||||
|
||||
with self._result_lock:
|
||||
if self._inflight and time.monotonic() - self._inflight_started_at > 0.5:
|
||||
logger.warning("MediaPipe detect_async timed out; allowing next frame submission")
|
||||
self._inflight = False
|
||||
self._inflight_started_at = 0.0
|
||||
should_submit = not self._inflight
|
||||
if should_submit:
|
||||
self._inflight = True
|
||||
self._inflight_started_at = time.monotonic()
|
||||
|
||||
if should_submit:
|
||||
rgba_frame = bgr_to_rgba(bgr_frame)
|
||||
mp_image = mp.Image(image_format=mp.ImageFormat.SRGBA, data=rgba_frame)
|
||||
self._result_event.clear()
|
||||
try:
|
||||
self._landmarker.detect_async(mp_image, timestamp_ms)
|
||||
except Exception:
|
||||
with self._result_lock:
|
||||
self._inflight = False
|
||||
self._inflight_started_at = 0.0
|
||||
raise
|
||||
self._result_event.wait(timeout=0.08)
|
||||
|
||||
with self._result_lock:
|
||||
pose_result = self._latest_result
|
||||
|
||||
annotated = bgr_frame.copy()
|
||||
|
||||
if pose_result is None or not pose_result.pose_landmarks:
|
||||
result = DeadBugResult(
|
||||
rep_count=self._state.rep_count,
|
||||
phase=DeadBugPhase.NO_POSE,
|
||||
side=self._state.active_side,
|
||||
is_standard=False,
|
||||
feedback=["No full body detected"],
|
||||
metrics=None,
|
||||
)
|
||||
draw_status_overlay(annotated, result)
|
||||
return annotated, result
|
||||
|
||||
landmarks = [Point(lm.x, lm.y, lm.z, getattr(lm, "visibility", 1.0)) for lm in pose_result.pose_landmarks[0]]
|
||||
draw_landmarks(annotated, landmarks, REQUIRED_LANDMARKS, visibility_threshold=self.visibility_threshold)
|
||||
|
||||
if not has_required_visibility(landmarks, REQUIRED_LANDMARKS, self.visibility_threshold):
|
||||
result = DeadBugResult(
|
||||
rep_count=self._state.rep_count,
|
||||
phase=DeadBugPhase.NO_POSE,
|
||||
side=self._state.active_side,
|
||||
is_standard=False,
|
||||
feedback=["Keep shoulders, elbows, wrists, hips, knees, ankles visible"],
|
||||
metrics=None,
|
||||
)
|
||||
draw_status_overlay(annotated, result)
|
||||
return annotated, result
|
||||
|
||||
raw = calculate_metrics(
|
||||
landmarks,
|
||||
left_shoulder=LEFT_SHOULDER,
|
||||
right_shoulder=RIGHT_SHOULDER,
|
||||
left_elbow=LEFT_ELBOW,
|
||||
right_elbow=RIGHT_ELBOW,
|
||||
left_wrist=LEFT_WRIST,
|
||||
right_wrist=RIGHT_WRIST,
|
||||
left_hip=LEFT_HIP,
|
||||
right_hip=RIGHT_HIP,
|
||||
left_knee=LEFT_KNEE,
|
||||
right_knee=RIGHT_KNEE,
|
||||
left_ankle=LEFT_ANKLE,
|
||||
right_ankle=RIGHT_ANKLE,
|
||||
visibility_threshold=self.visibility_threshold,
|
||||
)
|
||||
|
||||
metrics = DeadBugMetrics(
|
||||
left_arm_extended=raw["left_arm_extended"],
|
||||
right_arm_extended=raw["right_arm_extended"],
|
||||
left_leg_extended=raw["left_leg_extended"],
|
||||
right_leg_extended=raw["right_leg_extended"],
|
||||
left_elbow_angle=raw["left_elbow_angle"],
|
||||
right_elbow_angle=raw["right_elbow_angle"],
|
||||
left_knee_angle=raw["left_knee_angle"],
|
||||
right_knee_angle=raw["right_knee_angle"],
|
||||
feedback=raw["feedback"],
|
||||
)
|
||||
|
||||
result = self._state.update(metrics)
|
||||
draw_status_overlay(annotated, result)
|
||||
return annotated, result
|
||||
|
||||
def _normalize_timestamp(self, timestamp_ms: int) -> int:
|
||||
if timestamp_ms <= self._last_timestamp_ms:
|
||||
timestamp_ms = self._last_timestamp_ms + 1
|
||||
self._last_timestamp_ms = timestamp_ms
|
||||
return timestamp_ms
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from app.exercises.dead_bug.types import Point
|
||||
|
||||
|
||||
def angle(a: Point, b: Point, c: Point) -> float:
|
||||
ba = np.array([a.x - b.x, a.y - b.y], dtype=np.float32)
|
||||
bc = np.array([c.x - b.x, c.y - b.y], dtype=np.float32)
|
||||
denom = float(np.linalg.norm(ba) * np.linalg.norm(bc))
|
||||
if denom == 0:
|
||||
return 0.0
|
||||
cos_value = float(np.dot(ba, bc) / denom)
|
||||
return float(np.degrees(np.arccos(np.clip(cos_value, -1.0, 1.0))))
|
||||
|
||||
|
||||
def distance(a: Point, b: Point) -> float:
|
||||
return float(np.hypot(a.x - b.x, a.y - b.y))
|
||||
|
||||
|
||||
def calculate_metrics(
|
||||
lm: list[Point],
|
||||
*,
|
||||
left_shoulder: int,
|
||||
right_shoulder: int,
|
||||
left_elbow: int,
|
||||
right_elbow: int,
|
||||
left_wrist: int,
|
||||
right_wrist: int,
|
||||
left_hip: int,
|
||||
right_hip: int,
|
||||
left_knee: int,
|
||||
right_knee: int,
|
||||
left_ankle: int,
|
||||
right_ankle: int,
|
||||
visibility_threshold: float = 0.45,
|
||||
) -> dict:
|
||||
left_elbow_angle = angle(lm[left_shoulder], lm[left_elbow], lm[left_wrist])
|
||||
right_elbow_angle = angle(lm[right_shoulder], lm[right_elbow], lm[right_wrist])
|
||||
left_knee_angle = angle(lm[left_hip], lm[left_knee], lm[left_ankle])
|
||||
right_knee_angle = angle(lm[right_hip], lm[right_knee], lm[right_ankle])
|
||||
|
||||
shoulder_width = distance(lm[left_shoulder], lm[right_shoulder])
|
||||
hip_width = distance(lm[left_hip], lm[right_hip])
|
||||
scale = max(shoulder_width, hip_width, 0.08)
|
||||
|
||||
left_arm_extended = (
|
||||
left_elbow_angle >= 145
|
||||
and distance(lm[left_shoulder], lm[left_wrist]) >= scale * 1.15
|
||||
and lm[left_wrist].y <= lm[left_shoulder].y + scale * 0.35
|
||||
)
|
||||
right_arm_extended = (
|
||||
right_elbow_angle >= 145
|
||||
and distance(lm[right_shoulder], lm[right_wrist]) >= scale * 1.15
|
||||
and lm[right_wrist].y <= lm[right_shoulder].y + scale * 0.35
|
||||
)
|
||||
|
||||
left_leg_extended = (
|
||||
left_knee_angle >= 150
|
||||
and distance(lm[left_hip], lm[left_ankle]) >= scale * 1.55
|
||||
and lm[left_ankle].y >= lm[left_knee].y - scale * 0.2
|
||||
)
|
||||
right_leg_extended = (
|
||||
right_knee_angle >= 150
|
||||
and distance(lm[right_hip], lm[right_ankle]) >= scale * 1.55
|
||||
and lm[right_ankle].y >= lm[right_knee].y - scale * 0.2
|
||||
)
|
||||
|
||||
feedback: list[str] = []
|
||||
if left_arm_extended and left_elbow_angle < 160:
|
||||
feedback.append("Straighten left arm")
|
||||
if right_arm_extended and right_elbow_angle < 160:
|
||||
feedback.append("Straighten right arm")
|
||||
if left_leg_extended and left_knee_angle < 165:
|
||||
feedback.append("Straighten left leg")
|
||||
if right_leg_extended and right_knee_angle < 165:
|
||||
feedback.append("Straighten right leg")
|
||||
|
||||
return {
|
||||
"left_arm_extended": left_arm_extended,
|
||||
"right_arm_extended": right_arm_extended,
|
||||
"left_leg_extended": left_leg_extended,
|
||||
"right_leg_extended": right_leg_extended,
|
||||
"left_elbow_angle": left_elbow_angle,
|
||||
"right_elbow_angle": right_elbow_angle,
|
||||
"left_knee_angle": left_knee_angle,
|
||||
"right_knee_angle": right_knee_angle,
|
||||
"scale": scale,
|
||||
"feedback": feedback,
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.exercises.dead_bug.types import DeadBugMetrics, Point
|
||||
|
||||
|
||||
def has_required_visibility(landmarks: list[Point], required_indices: tuple[int, ...], visibility_threshold: float) -> bool:
|
||||
return all(landmarks[i].visibility >= visibility_threshold for i in required_indices)
|
||||
|
||||
|
||||
def detect_diagonal_extension(metrics: DeadBugMetrics) -> str | None:
|
||||
if metrics.left_leg_extended and metrics.right_leg_extended:
|
||||
return None
|
||||
|
||||
if metrics.right_leg_extended and metrics.left_arm_extended:
|
||||
return "left_arm_right_leg"
|
||||
if metrics.left_leg_extended and metrics.right_arm_extended:
|
||||
return "right_arm_left_leg"
|
||||
return None
|
||||
|
||||
|
||||
def is_ready_position(metrics: DeadBugMetrics) -> bool:
|
||||
knees_bent = metrics.left_knee_angle <= 140 and metrics.right_knee_angle <= 140
|
||||
legs_not_extended = not metrics.left_leg_extended and not metrics.right_leg_extended
|
||||
return knees_bent and legs_not_extended and detect_diagonal_extension(metrics) is None
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.exercises.dead_bug.rules import detect_diagonal_extension, is_ready_position
|
||||
from app.exercises.dead_bug.types import DeadBugMetrics, DeadBugPhase, DeadBugResult
|
||||
|
||||
|
||||
class DeadBugStateMachine:
|
||||
def __init__(self, *, extension_confirm_frames: int = 4, reset_confirm_frames: int = 3) -> None:
|
||||
self.extension_confirm_frames = extension_confirm_frames
|
||||
self.reset_confirm_frames = reset_confirm_frames
|
||||
|
||||
self.rep_count = 0
|
||||
self.phase = DeadBugPhase.READY
|
||||
self.active_side: str | None = None
|
||||
self._candidate_side: str | None = None
|
||||
self._candidate_frames = 0
|
||||
self._reset_frames = 0
|
||||
|
||||
def update(self, metrics: DeadBugMetrics) -> DeadBugResult:
|
||||
side = detect_diagonal_extension(metrics)
|
||||
ready = is_ready_position(metrics)
|
||||
|
||||
if side is None:
|
||||
self._candidate_side = None
|
||||
self._candidate_frames = 0
|
||||
elif side == self._candidate_side:
|
||||
self._candidate_frames += 1
|
||||
else:
|
||||
self._candidate_side = side
|
||||
self._candidate_frames = 1
|
||||
|
||||
if self.phase in (DeadBugPhase.READY, DeadBugPhase.NO_POSE):
|
||||
if self._candidate_frames >= self.extension_confirm_frames and side is not None:
|
||||
self.phase = DeadBugPhase.EXTENDING
|
||||
self.active_side = side
|
||||
self._reset_frames = 0
|
||||
elif self.phase == DeadBugPhase.EXTENDING:
|
||||
if side == self.active_side:
|
||||
self.phase = DeadBugPhase.NEED_RESET
|
||||
elif self.phase == DeadBugPhase.NEED_RESET:
|
||||
if ready:
|
||||
self._reset_frames += 1
|
||||
if self._reset_frames >= self.reset_confirm_frames:
|
||||
self.rep_count += 1
|
||||
self.phase = DeadBugPhase.READY
|
||||
self.active_side = None
|
||||
self._candidate_side = None
|
||||
self._candidate_frames = 0
|
||||
self._reset_frames = 0
|
||||
else:
|
||||
self._reset_frames = 0
|
||||
|
||||
feedback = list(metrics.feedback)
|
||||
if side is None and not ready:
|
||||
feedback.append("Extend opposite arm and leg only")
|
||||
if ready:
|
||||
feedback.append("Ready position")
|
||||
elif side == "left_arm_right_leg":
|
||||
feedback.append("Left arm + right leg")
|
||||
elif side == "right_arm_left_leg":
|
||||
feedback.append("Right arm + left leg")
|
||||
|
||||
is_standard = side is not None and not metrics.feedback
|
||||
return DeadBugResult(
|
||||
rep_count=self.rep_count,
|
||||
phase=self.phase,
|
||||
side=side,
|
||||
is_standard=is_standard,
|
||||
feedback=feedback[:3],
|
||||
metrics=metrics,
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class DeadBugPhase(str, Enum):
|
||||
READY = "ready"
|
||||
EXTENDING = "extending"
|
||||
NEED_RESET = "need_reset"
|
||||
NO_POSE = "no_pose"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Point:
|
||||
x: float
|
||||
y: float
|
||||
z: float
|
||||
visibility: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeadBugMetrics:
|
||||
left_arm_extended: bool
|
||||
right_arm_extended: bool
|
||||
left_leg_extended: bool
|
||||
right_leg_extended: bool
|
||||
left_elbow_angle: float
|
||||
right_elbow_angle: float
|
||||
left_knee_angle: float
|
||||
right_knee_angle: float
|
||||
feedback: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeadBugResult:
|
||||
rep_count: int
|
||||
phase: DeadBugPhase
|
||||
side: str | None
|
||||
is_standard: bool
|
||||
feedback: list[str]
|
||||
metrics: DeadBugMetrics | None
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
os.environ["MEDIAPIPE_DISABLE_LOGGING"] = "1"
|
||||
os.environ["GLOG_minloglevel"] = "3"
|
||||
|
||||
import asyncio
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.core.lifecycle import startup
|
||||
from app.signaling.websocket_server import main as serve
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
startup()
|
||||
logger.info("Starting server...")
|
||||
asyncio.run(serve())
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from app.exercises.dead_bug.types import DeadBugResult
|
||||
|
||||
|
||||
def draw_status_overlay(image: np.ndarray, result: DeadBugResult) -> None:
|
||||
color = (60, 220, 90) if result.is_standard else (50, 180, 255)
|
||||
cv2.rectangle(image, (12, 12), (520, 142), (20, 20, 20), -1)
|
||||
cv2.putText(image, f"Dead bug reps: {result.rep_count}", (28, 48), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, 2)
|
||||
cv2.putText(image, f"phase: {result.phase.value}", (28, 82), cv2.FONT_HERSHEY_SIMPLEX, 0.68, (230, 230, 230), 2)
|
||||
status = "standard" if result.is_standard else "adjust"
|
||||
cv2.putText(image, f"status: {status}", (28, 116), cv2.FONT_HERSHEY_SIMPLEX, 0.68, color, 2)
|
||||
|
||||
y = 170
|
||||
for text in result.feedback:
|
||||
cv2.putText(image, text, (28, y), cv2.FONT_HERSHEY_SIMPLEX, 0.68, (255, 255, 255), 2)
|
||||
y += 30
|
||||
@@ -0,0 +1,46 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from app.exercises.dead_bug.types import DeadBugResult, Point
|
||||
from app.vision.pose_types import _POSE_CONNECTIONS
|
||||
|
||||
|
||||
def draw_landmarks(
|
||||
image: np.ndarray,
|
||||
landmarks: list[Point],
|
||||
required_indices: tuple[int, ...],
|
||||
connections: tuple[tuple[int, int], ...] | None = None,
|
||||
visibility_threshold: float = 0.45,
|
||||
line_color: tuple[int, int, int] = (65, 180, 255),
|
||||
point_color: tuple[int, int, int] = (80, 255, 120),
|
||||
line_thickness: int = 2,
|
||||
point_radius: int = 4,
|
||||
) -> None:
|
||||
if connections is None:
|
||||
connections = _POSE_CONNECTIONS
|
||||
|
||||
h, w = image.shape[:2]
|
||||
|
||||
for start, end in connections:
|
||||
if start >= len(landmarks) or end >= len(landmarks):
|
||||
continue
|
||||
p1 = landmarks[start]
|
||||
p2 = landmarks[end]
|
||||
if p1.visibility < visibility_threshold or p2.visibility < visibility_threshold:
|
||||
continue
|
||||
cv2.line(
|
||||
image,
|
||||
(int(p1.x * w), int(p1.y * h)),
|
||||
(int(p2.x * w), int(p2.y * h)),
|
||||
line_color,
|
||||
line_thickness,
|
||||
)
|
||||
|
||||
for idx in required_indices:
|
||||
if idx >= len(landmarks):
|
||||
continue
|
||||
p = landmarks[idx]
|
||||
if p.visibility >= visibility_threshold:
|
||||
cv2.circle(image, (int(p.x * w), int(p.y * h)), point_radius, point_color, -1)
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
|
||||
WINDOW_NAME = "Android Camera (WebRTC)"
|
||||
|
||||
|
||||
def show_frame(image, window_name: str = WINDOW_NAME) -> None:
|
||||
cv2.imshow(window_name, image)
|
||||
|
||||
|
||||
def wait_key(delay_ms: int = 1) -> int:
|
||||
return cv2.waitKey(delay_ms) & 0xFF
|
||||
|
||||
|
||||
def is_esc_pressed() -> bool:
|
||||
return wait_key(1) == 27
|
||||
|
||||
|
||||
def close_window() -> None:
|
||||
cv2.destroyAllWindows()
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from aiortc import RTCIceCandidate
|
||||
|
||||
|
||||
def parse_ice(data: dict[str, Any]) -> RTCIceCandidate | None:
|
||||
match = re.match(
|
||||
r'candidate:(\S+) (\d) (\S+) (\d+) (\S+) (\d+) typ (\S+)(?: raddr (\S+) rport (\d+))?',
|
||||
data["candidate"],
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
g = match.groups()
|
||||
cand = RTCIceCandidate(
|
||||
foundation=g[0],
|
||||
component=int(g[1]),
|
||||
protocol=g[2].lower(),
|
||||
priority=int(g[3]),
|
||||
ip=g[4],
|
||||
port=int(g[5]),
|
||||
type=g[6],
|
||||
relatedAddress=g[7],
|
||||
relatedPort=int(g[8]) if g[8] else None,
|
||||
)
|
||||
cand.sdpMid = data.get("sdpMid")
|
||||
cand.sdpMLineIndex = data.get("sdpMLineIndex", 0)
|
||||
return cand
|
||||
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class SignalingMessage:
|
||||
type: str
|
||||
sdp: str = ""
|
||||
candidate: str = ""
|
||||
sdpMid: str | None = None
|
||||
sdpMLineIndex: int = 0
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import websockets
|
||||
from loguru import logger
|
||||
|
||||
from app.webrtc.peer_session import PeerSession
|
||||
from configs.default import WS_HOST, WS_MAX_SIZE, WS_PORT
|
||||
|
||||
|
||||
async def handle_client(websocket):
|
||||
client = websocket.remote_address
|
||||
logger.info(f"Client connected: {client}")
|
||||
|
||||
session = PeerSession()
|
||||
await session.handle(websocket)
|
||||
|
||||
logger.info(f"Connection closed: {client}")
|
||||
|
||||
|
||||
async def main():
|
||||
logger.info(f"WebRTC signaling server: ws://{WS_HOST}:{WS_PORT}")
|
||||
async with websockets.serve(handle_client, WS_HOST, WS_PORT, max_size=WS_MAX_SIZE):
|
||||
await asyncio.Future()
|
||||
@@ -0,0 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
|
||||
TARGET_WIDTH = 1280
|
||||
TARGET_HEIGHT = 720
|
||||
|
||||
|
||||
def resize_to_target(image: np.ndarray, width: int = TARGET_WIDTH, height: int = TARGET_HEIGHT) -> np.ndarray:
|
||||
h, w = image.shape[:2]
|
||||
if w == width and h == height:
|
||||
return image
|
||||
return cv2.resize(image, (width, height))
|
||||
|
||||
|
||||
def bgr_to_rgba(bgr: np.ndarray) -> np.ndarray:
|
||||
return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGBA)
|
||||
|
||||
|
||||
def bgr_to_rgb(bgr: np.ndarray) -> np.ndarray:
|
||||
return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
import mediapipe as mp
|
||||
from loguru import logger
|
||||
|
||||
from app.vision.pose_models import DEFAULT_MODEL_PATH
|
||||
|
||||
PoseLandmarker = mp.tasks.vision.PoseLandmarker
|
||||
PoseLandmarkerOptions = mp.tasks.vision.PoseLandmarkerOptions
|
||||
VisionRunningMode = mp.tasks.vision.RunningMode
|
||||
BaseOptions = mp.tasks.BaseOptions
|
||||
|
||||
|
||||
class PoseLandmarkerWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model_path: str | None = None,
|
||||
prefer_gpu: bool = True,
|
||||
result_callback: Callable | None = None,
|
||||
) -> None:
|
||||
self.model_path = model_path or DEFAULT_MODEL_PATH
|
||||
|
||||
if prefer_gpu:
|
||||
try:
|
||||
self.delegate = BaseOptions.Delegate.GPU
|
||||
self._landmarker = self._create(PoseLandmarker.Delegate.GPU)
|
||||
logger.info("MediaPipe PoseLandmarker initialized with GPU delegate")
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.warning("MediaPipe GPU delegate unavailable, falling back to CPU: {}", exc)
|
||||
|
||||
self.delegate = BaseOptions.Delegate.CPU
|
||||
self._landmarker = self._create(PoseLandmarker.Delegate.CPU, result_callback)
|
||||
logger.info("MediaPipe PoseLandmarker initialized with CPU delegate")
|
||||
|
||||
def _create(self, delegate, result_callback=None):
|
||||
options = PoseLandmarkerOptions(
|
||||
base_options=BaseOptions(model_asset_path=self.model_path, delegate=delegate),
|
||||
running_mode=VisionRunningMode.LIVE_STREAM,
|
||||
result_callback=result_callback,
|
||||
num_poses=1,
|
||||
min_pose_detection_confidence=0.5,
|
||||
min_pose_presence_confidence=0.5,
|
||||
min_tracking_confidence=0.5,
|
||||
)
|
||||
return PoseLandmarker.create_from_options(options)
|
||||
|
||||
def detect_async(self, mp_image, timestamp_ms: int) -> None:
|
||||
return self._landmarker.detect_async(mp_image, timestamp_ms)
|
||||
|
||||
def close(self) -> None:
|
||||
self._landmarker.close()
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import mediapipe as mp
|
||||
|
||||
BaseOptions = mp.tasks.BaseOptions
|
||||
|
||||
_MODELS_DIR = Path(__file__).resolve().parent.parent.parent / "pose_models"
|
||||
|
||||
POSE_LANDMARKER_FULL = _MODELS_DIR / "pose_landmarker_full.task"
|
||||
POSE_LANDMARKER_LITE = _MODELS_DIR / "pose_landmarker_lite.task"
|
||||
|
||||
DEFAULT_MODEL_PATH = str(POSE_LANDMARKER_FULL) if POSE_LANDMARKER_FULL.exists() else str(POSE_LANDMARKER_LITE)
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
_POSE_CONNECTIONS = (
|
||||
(11, 12),
|
||||
(11, 13),
|
||||
(13, 15),
|
||||
(12, 14),
|
||||
(14, 16),
|
||||
(11, 23),
|
||||
(12, 24),
|
||||
(23, 24),
|
||||
(23, 25),
|
||||
(25, 27),
|
||||
(24, 26),
|
||||
(26, 28),
|
||||
)
|
||||
|
||||
LANDMARK_NAMES: dict[int, str] = {
|
||||
0: "nose",
|
||||
1: "left_eye_inner",
|
||||
2: "left_eye",
|
||||
3: "left_eye_outer",
|
||||
4: "right_eye_inner",
|
||||
5: "right_eye",
|
||||
6: "right_eye_outer",
|
||||
7: "left_ear",
|
||||
8: "right_ear",
|
||||
9: "mouth_left",
|
||||
10: "mouth_right",
|
||||
11: "left_shoulder",
|
||||
12: "right_shoulder",
|
||||
13: "left_elbow",
|
||||
14: "right_elbow",
|
||||
15: "left_wrist",
|
||||
16: "right_wrist",
|
||||
17: "left_pinky",
|
||||
18: "right_pinky",
|
||||
19: "left_index",
|
||||
20: "right_index",
|
||||
21: "left_thumb",
|
||||
22: "right_thumb",
|
||||
23: "left_hip",
|
||||
24: "right_hip",
|
||||
25: "left_knee",
|
||||
26: "right_knee",
|
||||
27: "left_ankle",
|
||||
28: "right_ankle",
|
||||
29: "left_heel",
|
||||
30: "right_heel",
|
||||
31: "left_foot_index",
|
||||
32: "right_foot_index",
|
||||
}
|
||||
|
||||
LEFT_SHOULDER = 11
|
||||
RIGHT_SHOULDER = 12
|
||||
LEFT_ELBOW = 13
|
||||
RIGHT_ELBOW = 14
|
||||
LEFT_WRIST = 15
|
||||
RIGHT_WRIST = 16
|
||||
LEFT_HIP = 23
|
||||
RIGHT_HIP = 24
|
||||
LEFT_KNEE = 25
|
||||
RIGHT_KNEE = 26
|
||||
LEFT_ANKLE = 27
|
||||
RIGHT_ANKLE = 28
|
||||
|
||||
REQUIRED_LANDMARKS = (
|
||||
LEFT_SHOULDER,
|
||||
RIGHT_SHOULDER,
|
||||
LEFT_ELBOW,
|
||||
RIGHT_ELBOW,
|
||||
LEFT_WRIST,
|
||||
RIGHT_WRIST,
|
||||
LEFT_HIP,
|
||||
RIGHT_HIP,
|
||||
LEFT_KNEE,
|
||||
RIGHT_KNEE,
|
||||
LEFT_ANKLE,
|
||||
RIGHT_ANKLE,
|
||||
)
|
||||
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
|
||||
TARGET_WIDTH = 1280
|
||||
TARGET_HEIGHT = 720
|
||||
|
||||
|
||||
def validate_frame_size(image: np.ndarray, width: int = TARGET_WIDTH, height: int = TARGET_HEIGHT) -> None:
|
||||
h, w = image.shape[:2]
|
||||
if w != width or h != height:
|
||||
logger.warning("Unexpected frame size: {}x{}", w, h)
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import websockets
|
||||
from aiortc import RTCPeerConnection, RTCSessionDescription
|
||||
from loguru import logger
|
||||
|
||||
from app.signaling.ice_parser import parse_ice
|
||||
from app.webrtc.video_receiver import VideoReceiver
|
||||
|
||||
|
||||
class PeerSession:
|
||||
def __init__(self) -> None:
|
||||
self._pc = RTCPeerConnection()
|
||||
self._video_task: asyncio.Task | None = None
|
||||
|
||||
async def handle(self, websocket) -> None:
|
||||
self._setup_events()
|
||||
|
||||
try:
|
||||
async for message in websocket:
|
||||
data = json.loads(message)
|
||||
msg_type = data.get("type")
|
||||
|
||||
if msg_type == "offer":
|
||||
offer = RTCSessionDescription(sdp=data["sdp"], type="offer")
|
||||
await self._pc.setRemoteDescription(offer)
|
||||
answer = await self._pc.createAnswer()
|
||||
await self._pc.setLocalDescription(answer)
|
||||
await websocket.send(json.dumps({
|
||||
"type": "answer",
|
||||
"sdp": self._pc.localDescription.sdp,
|
||||
}))
|
||||
|
||||
elif msg_type == "candidate":
|
||||
cand = parse_ice(data)
|
||||
if cand:
|
||||
await self._pc.addIceCandidate(cand)
|
||||
|
||||
except websockets.ConnectionClosed:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.exception(f"Error: {e}")
|
||||
finally:
|
||||
await self._cleanup()
|
||||
|
||||
def _setup_events(self) -> None:
|
||||
@self._pc.on("track")
|
||||
async def on_track(track):
|
||||
logger.info(f"Track received: kind={track.kind}")
|
||||
if track.kind == "video":
|
||||
receiver = VideoReceiver(track)
|
||||
self._video_task = asyncio.ensure_future(receiver.run())
|
||||
|
||||
@self._pc.on("iceconnectionstatechange")
|
||||
async def on_iceconnectionstatechange():
|
||||
logger.info(f"ICE state: {self._pc.iceConnectionState}")
|
||||
if self._pc.iceConnectionState in ("failed", "closed", "disconnected"):
|
||||
await self._pc.close()
|
||||
|
||||
async def _cleanup(self) -> None:
|
||||
if self._video_task:
|
||||
self._video_task.cancel()
|
||||
try:
|
||||
await self._video_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await self._pc.close()
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import cv2
|
||||
from aiortc.mediastreams import MediaStreamError
|
||||
from loguru import logger
|
||||
|
||||
from app.audio.rep_announcer import RepAnnouncer
|
||||
from app.exercises.dead_bug.detector import DeadBugDetector
|
||||
from app.rendering.window_display import close_window, is_esc_pressed, show_frame
|
||||
from configs.default import (
|
||||
EXTENSION_CONFIRM_FRAMES,
|
||||
MODEL_PATH,
|
||||
PREFER_GPU,
|
||||
PROCESS_EVERY_N_FRAMES,
|
||||
REP_ANNOUNCER_ENABLED,
|
||||
REP_ANNOUNCER_RATE,
|
||||
REP_ANNOUNCER_VOLUME,
|
||||
RESET_CONFIRM_FRAMES,
|
||||
VISIBILITY_THRESHOLD,
|
||||
)
|
||||
|
||||
|
||||
def _format_pose_debug(pose_result) -> str:
|
||||
metrics = pose_result.metrics
|
||||
if metrics is None:
|
||||
return "metrics=None"
|
||||
return (
|
||||
f"side={pose_result.side}, standard={pose_result.is_standard}, "
|
||||
f"angles(le={metrics.left_elbow_angle:.1f}, re={metrics.right_elbow_angle:.1f}, "
|
||||
f"lk={metrics.left_knee_angle:.1f}, rk={metrics.right_knee_angle:.1f}), "
|
||||
f"extended(la={metrics.left_arm_extended}, ra={metrics.right_arm_extended}, "
|
||||
f"ll={metrics.left_leg_extended}, rl={metrics.right_leg_extended})"
|
||||
)
|
||||
|
||||
|
||||
class VideoReceiver:
|
||||
def __init__(self, track) -> None:
|
||||
self._track = track
|
||||
|
||||
async def run(self) -> None:
|
||||
logger.info("Start receiving video frames, process_every_n={}", PROCESS_EVERY_N_FRAMES)
|
||||
|
||||
frame_count = 0
|
||||
processed_count = 0
|
||||
detector = DeadBugDetector(
|
||||
model_path=MODEL_PATH,
|
||||
visibility_threshold=VISIBILITY_THRESHOLD,
|
||||
extension_confirm_frames=EXTENSION_CONFIRM_FRAMES,
|
||||
reset_confirm_frames=RESET_CONFIRM_FRAMES,
|
||||
prefer_gpu=PREFER_GPU,
|
||||
)
|
||||
announcer = RepAnnouncer(
|
||||
enabled=REP_ANNOUNCER_ENABLED,
|
||||
rate=REP_ANNOUNCER_RATE,
|
||||
volume=REP_ANNOUNCER_VOLUME,
|
||||
)
|
||||
last_announced_rep = 0
|
||||
last_pose_result = None
|
||||
last_annotated = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
frame = await self._track.recv()
|
||||
frame_count += 1
|
||||
raw_img = frame.to_ndarray(format="bgr24")
|
||||
timestamp_ms = int(frame.time * 1000) if frame.time is not None else frame_count * 33
|
||||
|
||||
if frame_count % PROCESS_EVERY_N_FRAMES == 0 or last_pose_result is None:
|
||||
processed_count += 1
|
||||
last_annotated, last_pose_result = detector.process_frame(raw_img, timestamp_ms)
|
||||
if last_pose_result.rep_count > last_announced_rep:
|
||||
last_announced_rep = last_pose_result.rep_count
|
||||
announcer.announce_count(last_announced_rep)
|
||||
|
||||
display_img = last_annotated if last_annotated is not None else raw_img
|
||||
show_frame(display_img)
|
||||
|
||||
if frame_count % 100 == 0:
|
||||
logger.info(
|
||||
"Received {} frames, processed={}, raw_shape={}, reps={}, phase={}, feedback={}, {}",
|
||||
frame_count,
|
||||
processed_count,
|
||||
raw_img.shape,
|
||||
last_pose_result.rep_count if last_pose_result is not None else 0,
|
||||
last_pose_result.phase.value if last_pose_result is not None else "none",
|
||||
" | ".join(last_pose_result.feedback) if last_pose_result is not None else "",
|
||||
_format_pose_debug(last_pose_result) if last_pose_result is not None else "metrics=None",
|
||||
)
|
||||
|
||||
if is_esc_pressed():
|
||||
logger.info("ESC pressed, closing display")
|
||||
break
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Video receive task cancelled")
|
||||
except MediaStreamError:
|
||||
logger.info("Video track ended")
|
||||
except Exception as e:
|
||||
logger.exception(f"Video receive error: {e!r}")
|
||||
finally:
|
||||
announcer.close()
|
||||
detector.close()
|
||||
close_window()
|
||||
Reference in New Issue
Block a user