Sign Language recognition using LSTM

Deployment
, ,
1

As said in my previous post, I have removed the cv.VideoCapture(0) and used webrtc_streamer. but now my model is not working properly. can anyone help me how to solve this error?

talking about the current situation, even if camera is opening, I am not getting proper response. also it gets stuck after a while.


import os
import cv2
import pickle
import pyttsx3 
import numpy as np
import mediapipe as mp
from matplotlib import pyplot as plt
t2s = pyttsx3.init()

import streamlit as st
from streamlit_webrtc import webrtc_streamer
import av

from tensorflow.keras.models import load_model
model = load_model('action.h5')


mp_holistic = mp.solutions.holistic
mp_drawing = mp.solutions.drawing_utils

actions = np.array(['Hello','I am','Affan','Thanks', 'i love you','Fever','See you', 'God'])

def mediapipe_detection(image, model):
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image.flags.writeable = False
    results = model.process(image)
    image.flags.writeable = True
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
    return image, results

def extract_keypoints(results):
    lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)
    rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
    return np.concatenate([lh, rh])

def draw_styled_landmarks(image, results):
    mp_drawing.draw_landmarks(image, results.left_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
                             mp_drawing.DrawingSpec(color=(100, 20, 70), thickness=2, circle_radius=4),
                             mp_drawing.DrawingSpec(color=(100, 50, 20), thickness=2, circle_radius=2)
                             )
    mp_drawing.draw_landmarks(image, results.right_hand_landmarks, mp_holistic.HAND_CONNECTIONS,
                             mp_drawing.DrawingSpec(color=(200,100,100), thickness=2, circle_radius=4),
                             mp_drawing.DrawingSpec(color=(200,50,50), thickness=2, circle_radius=2)
                             )

sequence = []
sentence = []
predictions = []
threshold = 0.5


st.title("LSTM Sign Language ")


def callback(frame):

    global sequence, sentence, predictions, threshold

    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
        img = frame.to_ndarray(format="bgr24")

    
        image, results = mediapipe_detection(img, holistic)
        print(results)

        # Draw landmarks
        draw_styled_landmarks(image, results)

        # Prediction logic
        keypoints = extract_keypoints(results)
        sequence.append(keypoints)
        sequence = sequence[-30:]

        if len(sequence) == 30:
            res = model.predict(np.expand_dims(sequence, axis=0))[0]
            print(actions[np.argmax(res)])
            predictions.append(np.argmax(res))

            # Viz logic
            if np.unique(predictions[-10:])[0] == np.argmax(res):
                if res[np.argmax(res)] > threshold:
                    if len(sentence) > 0:
                        if actions[np.argmax(res)] != sentence[-1]:
                            sentence.append(actions[np.argmax(res)])
                            new_word = actions[np.argmax(res)]
                            t2s.say(new_word)
                            t2s.runAndWait()
                            predictions = []  # Clear predictions list
                    else:
                        sentence.append(actions[np.argmax(res)])
                        new_word = actions[np.argmax(res)]
                        t2s.say(new_word)
                        t2s.runAndWait()
                        predictions = []  # Clear predictions list

            if len(sentence) > 5:
                sentence = sentence[-5:]

            cv2.rectangle(image, (0, 0), (640, 40), (115, 115, 115), -1)
            cv2.putText(image, ' '.join(sentence), (3, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)


    return av.VideoFrame.from_ndarray(img, format="bgr24")



webrtc_streamer(key="example", video_frame_callback=callback)

link for files:
https://drive.google.com/drive/folders/13GJR19XXVWctyvto4zrYErlBoB12sOBT?usp=drive_link

2

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.