1. Обнаружение landmarks руки

def detect_hand_skeleton(frame, hand_detector):
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
results = hand_detector.process(rgb_frame)

if results.multi_hand_landmarks and results.multi_handedness:
hand_landmarks = results.multi_hand_landmarks[0]
handedness = results.multi_handedness[0].classification[0].label
return hand_landmarks, handedness
return None, None 3.

2. Определение жеста на основе landmarks

def get_hand_gesture(hand_landmarks, frame_width, frame_height):
landmarks = []
for lm in hand_landmarks.landmark:
x = int(lm.x * frame_width)
y = int(lm.y * frame_height)
landmarks.append((x, y))

# Определение состояния пальцев (согнуты/разогнуты)
fingers_up = []
# Логика для большого пальца
thumb_tip = landmarks[4]
thumb_mcp = landmarks[2]
if thumb_tip[0] > thumb_mcp[0]: # Для правой руки
fingers_up.append(1)
else:
fingers_up.append(0)

# Остальные пальцы
finger_tips = [8, 12, 16, 20] # Указательный, средний, безымянный, мизинец
finger_pips = [6, 10, 14, 18] # Суставы

for tip, pip in zip(finger_tips, finger_pips):
if landmarks[tip][1] < landmarks[pip][1]: # Кончик выше сустава
fingers_up.append(1)
else:
fingers_up.append(0)

total_fingers = sum(fingers_up)

if total_fingers == 0:
return "fist", fingers_up
elif total_fingers == 1 and fingers_up[1] == 1:
return "one_finger", fingers_up
elif total_fingers >= 2:
return "open_hand", fingers_up
else:
return "unknown", fingers_up

3.Определение направления указательного пальца

def get_finger_direction(hand_landmarks, frame_width, frame_height):
landmarks = []
for lm in hand_landmarks.landmark:
x = int(lm.x * frame_width)
y = int(lm.y * frame_height)
landmarks.append((x, y))

index_mcp = landmarks[5] # Сустав указательного пальца
index_tip = landmarks[8] # Кончик указательного пальца

dx = index_tip[0] - index_mcp[0]
dy = index_tip[1] - index_mcp[1]

if dx == 0:
angle = 90
else:
angle = np.degrees(np.arctan2(dy, dx))

x_center = index_tip[0] / frame_width

if x_center < 0.4:
return "left", angle
elif x_center > 0.6:
return "right", angle
else:
return "center", angle 5.

4.Визуализация скелета
руки

def draw_hand_skeleton(frame, hand_landmarks, gesture, direction):
mp_drawing.draw_landmarks(
frame,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style()
)

cv2.putText(frame, f"Жест: {gesture}", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)
cv2.putText(frame, f"Направление: {direction}", (10, 60),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)
return frame Пример использования из кода: if hand_landmarks:
# Определение жеста
gesture, fingers_up = get_hand_gesture(hand_landmarks, frame.shape[1], frame.shape[0])

# Определение направления
direction, angle = get_finger_direction(hand_landmarks, frame.shape[1], frame.shape[0])

# Визуализация
frame = draw_hand_skeleton(frame, hand_landmarks, gesture, direction)

# Управление роботом на основе жестов
if gesture == "fist":
send_command_to_esp("stop")
elif gesture == "one_finger" and direction == "center":
send_command_to_esp("forward")
elif gesture == "one_finger" and direction == "left":
send_command_to_esp("left")
elif gesture == "one_finger" and direction == "right":
send_command_to_esp("right")