HomeAssistant MCP

Verified
Home Automation & IoT
Apache 2.0
Overview InspectNew Schema Related Servers Reviews Score
import os
import json
import queue
import threading
import numpy as np
import sounddevice as sd
from openwakeword import Model
from datetime import datetime
import wave
from faster_whisper import WhisperModel
import requests
import logging
import time

# Set up logging
logging.basicConfig(
    level=logging.DEBUG,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Configuration
SAMPLE_RATE = 16000
CHANNELS = 1
CHUNK_SIZE = 1024
BUFFER_DURATION = 10  # seconds to keep in buffer
DETECTION_THRESHOLD = 0.5
CONTINUOUS_TRANSCRIPTION_INTERVAL = 3  # seconds between transcriptions
MAX_MODEL_LOAD_RETRIES = 3
MODEL_LOAD_RETRY_DELAY = 5  # seconds
MODEL_DOWNLOAD_TIMEOUT = 600  # 10 minutes timeout for model download

# ALSA device configuration
AUDIO_DEVICE = 'hw:0,0'  # Use ALSA hardware device directly

# Audio processing parameters
NOISE_THRESHOLD = 0.08  # Increased threshold for better noise filtering
MIN_SPEECH_DURATION = 2.0  # Longer minimum duration to avoid fragments
SILENCE_DURATION = 1.0  # Longer silence duration
MAX_REPETITIONS = 1  # More aggressive repetition filtering
ECHO_THRESHOLD = 0.75  # More sensitive echo detection
MIN_SEGMENT_DURATION = 1.0  # Longer minimum segment duration
FEEDBACK_WINDOW = 5  # Window size for feedback detection in seconds

# Feature flags from environment
WAKE_WORD_ENABLED = os.environ.get('ENABLE_WAKE_WORD', 'false').lower() == 'true'
SPEECH_ENABLED = os.environ.get('ENABLE_SPEECH_FEATURES', 'true').lower() == 'true'

# Wake word models to use (only if wake word is enabled)
WAKE_WORDS = ["hey_jarvis"]  # Using hey_jarvis as it's more similar to "hey gaja"
WAKE_WORD_ALIAS = "gaja"  # What we print when wake word is detected

# Home Assistant Configuration
HASS_HOST = os.environ.get('HASS_HOST', 'http://homeassistant.local:8123')
HASS_TOKEN = os.environ.get('HASS_TOKEN')

def initialize_asr_model():
    """Initialize the ASR model with retries and timeout"""
    model_path = os.environ.get('WHISPER_MODEL_PATH', '/models')
    model_name = os.environ.get('WHISPER_MODEL_TYPE', 'base')
    
    start_time = time.time()
    for attempt in range(MAX_MODEL_LOAD_RETRIES):
        try:
            if time.time() - start_time > MODEL_DOWNLOAD_TIMEOUT:
                logger.error("Model download timeout exceeded")
                raise TimeoutError("Model download took too long")
                
            logger.info(f"Loading ASR model (attempt {attempt + 1}/{MAX_MODEL_LOAD_RETRIES})")
            model = WhisperModel(
                model_size_or_path=model_name,
                device="cpu",
                compute_type="int8",
                download_root=model_path,
                num_workers=1  # Reduce concurrent downloads
            )
            logger.info("ASR model loaded successfully")
            return model
        except Exception as e:
            logger.error(f"Failed to load ASR model (attempt {attempt + 1}): {e}")
            if attempt < MAX_MODEL_LOAD_RETRIES - 1:
                logger.info(f"Retrying in {MODEL_LOAD_RETRY_DELAY} seconds...")
                time.sleep(MODEL_LOAD_RETRY_DELAY)
            else:
                logger.error("Failed to load ASR model after all retries")
                raise

# Initialize the ASR model with retries
try:
    asr_model = initialize_asr_model()
except Exception as e:
    logger.error(f"Critical error initializing ASR model: {e}")
    raise

def send_command_to_hass(domain, service, entity_id):
    """Send command to Home Assistant"""
    if not HASS_TOKEN:
        logger.error("Error: HASS_TOKEN not set")
        return False

    headers = {
        "Authorization": f"Bearer {HASS_TOKEN}",
        "Content-Type": "application/json",
    }

    url = f"{HASS_HOST}/api/services/{domain}/{service}"
    data = {"entity_id": entity_id}

    try:
        response = requests.post(url, headers=headers, json=data)
        response.raise_for_status()
        logger.info(f"Command sent: {domain}.{service} for {entity_id}")
        return True
    except Exception as e:
        logger.error(f"Error sending command to Home Assistant: {e}")
        return False

def is_speech(audio_data, threshold=NOISE_THRESHOLD):
    """Detect if audio segment contains speech based on amplitude and frequency content"""
    # Calculate RMS amplitude
    rms = np.sqrt(np.mean(np.square(audio_data)))
    
    # Calculate signal energy in speech frequency range (100-4000 Hz)
    fft = np.fft.fft(audio_data)
    freqs = np.fft.fftfreq(len(audio_data), 1/SAMPLE_RATE)
    speech_mask = (np.abs(freqs) >= 100) & (np.abs(freqs) <= 4000)
    speech_energy = np.sum(np.abs(fft[speech_mask])) / len(audio_data)
    
    # Enhanced echo detection
    # 1. Check for periodic patterns in the signal
    autocorr = np.correlate(audio_data, audio_data, mode='full')
    autocorr = autocorr[len(autocorr)//2:]  # Use only positive lags
    peaks = np.where(autocorr > ECHO_THRESHOLD * np.max(autocorr))[0]
    peak_spacing = np.diff(peaks)
    has_periodic_echo = len(peak_spacing) > 2 and np.std(peak_spacing) < 0.1 * np.mean(peak_spacing)
    
    # 2. Check for sudden amplitude changes
    amplitude_envelope = np.abs(audio_data)
    amplitude_changes = np.diff(amplitude_envelope)
    has_feedback_spikes = np.any(np.abs(amplitude_changes) > threshold * 2)
    
    # 3. Check frequency distribution
    freq_magnitudes = np.abs(fft)[:len(fft)//2]
    peak_freqs = freqs[:len(fft)//2][np.argsort(freq_magnitudes)[-3:]]
    has_feedback_freqs = np.any((peak_freqs > 2000) & (peak_freqs < 4000))
    
    # Combine all criteria
    is_valid_speech = (
        rms > threshold and
        speech_energy > threshold and
        not has_periodic_echo and
        not has_feedback_spikes and
        not has_feedback_freqs
    )
    
    return is_valid_speech

def process_command(text):
    """Process the transcribed command and execute appropriate action"""
    text = text.lower().strip()
    
    # Skip if text is too short or contains numbers (likely noise)
    if len(text) < 5 or any(char.isdigit() for char in text):
        logger.debug("Text too short or contains numbers, skipping")
        return
    
    # Enhanced noise pattern detection
    noise_patterns = ["lei", "los", "und", "aber", "nicht mehr", "das das", "und und"]
    for pattern in noise_patterns:
        if text.count(pattern) > 1:  # More aggressive pattern filtering
            logger.debug(f"Detected noise pattern '{pattern}', skipping")
            return
    
    # More aggressive repetition detection
    words = text.split()
    if len(words) >= 2:
        # Check for immediate word repetitions
        for i in range(len(words)-1):
            if words[i] == words[i+1]:
                logger.debug(f"Detected immediate word repetition: '{words[i]}', skipping")
                return
        
        # Check for phrase repetitions
        phrases = [' '.join(words[i:i+2]) for i in range(len(words)-1)]
        phrase_counts = {}
        for phrase in phrases:
            phrase_counts[phrase] = phrase_counts.get(phrase, 0) + 1
            if phrase_counts[phrase] > MAX_REPETITIONS:
                logger.debug(f"Skipping due to excessive repetition: '{phrase}'")
                return
    
    # German command mappings
    commands = {
        "ausschalten": "turn_off",
        "einschalten": "turn_on",
        "an": "turn_on",
        "aus": "turn_off"
    }
    
    rooms = {
        "wohnzimmer": "living_room",
        "küche": "kitchen",
        "schlafzimmer": "bedroom",
        "bad": "bathroom"
    }
    
    # Detect room
    detected_room = None
    for german_room, english_room in rooms.items():
        if german_room in text:
            detected_room = english_room
            break
    
    # Detect command
    detected_command = None
    for german_cmd, english_cmd in commands.items():
        if german_cmd in text:
            detected_command = english_cmd
            break
    
    if detected_room and detected_command:
        # Construct entity ID (assuming light)
        entity_id = f"light.{detected_room}"
        
        # Send command to Home Assistant
        if send_command_to_hass("light", detected_command, entity_id):
            logger.info(f"Executed: {detected_command} for {entity_id}")
        else:
            logger.error("Failed to execute command")
    else:
        logger.debug(f"No command found in text: '{text}'")

class AudioProcessor:
    def __init__(self):
        logger.info("Initializing AudioProcessor...")
        self.audio_buffer = queue.Queue()
        self.recording = False
        self.buffer = np.zeros(SAMPLE_RATE * BUFFER_DURATION)
        self.buffer_lock = threading.Lock()
        self.last_transcription_time = 0
        
        try:
            logger.info(f"Opening audio device: {AUDIO_DEVICE}")
            self.stream = sd.InputStream(
                device=AUDIO_DEVICE,
                samplerate=SAMPLE_RATE,
                channels=CHANNELS,
                dtype=np.int16,
                blocksize=CHUNK_SIZE,
                callback=self._audio_callback
            )
            logger.info("Audio stream initialized successfully")
        except Exception as e:
            logger.error(f"Failed to initialize audio stream: {e}")
            raise

        self.speech_detected = False
        self.silence_frames = 0
        self.speech_frames = 0

        # Initialize wake word detection only if enabled
        if WAKE_WORD_ENABLED:
            try:
                logger.info("Initializing wake word model...")
                self.wake_word_model = Model(vad_threshold=0.5)
                self.last_prediction = None
                logger.info("Wake word model initialized successfully")
            except Exception as e:
                logger.error(f"Failed to initialize wake word model: {e}")
                raise
        else:
            self.wake_word_model = None
            self.last_prediction = None
            logger.info("Wake word detection disabled")

    def should_transcribe(self):
        """Determine if we should transcribe based on mode and timing"""
        current_time = datetime.now().timestamp()
        if not WAKE_WORD_ENABLED:
            # Check if enough time has passed since last transcription
            time_since_last = current_time - self.last_transcription_time
            if time_since_last >= CONTINUOUS_TRANSCRIPTION_INTERVAL:
                # Only transcribe if we detect speech
                frames_per_chunk = CHUNK_SIZE
                min_speech_frames = int(MIN_SPEECH_DURATION * SAMPLE_RATE / frames_per_chunk)
                
                if self.speech_frames >= min_speech_frames:
                    self.last_transcription_time = current_time
                    self.speech_frames = 0  # Reset counter
                    return True
        return False

    def _audio_callback(self, indata, frames, time, status):
        """Callback for audio input"""
        if status:
            logger.warning(f"Audio callback status: {status}")
        
        # Convert to mono if necessary
        if CHANNELS > 1:
            audio_data = np.mean(indata, axis=1)
        else:
            audio_data = indata.flatten()

        # Check for speech
        if is_speech(audio_data):
            self.speech_frames += 1
            self.silence_frames = 0
        else:
            self.silence_frames += 1
            frames_per_chunk = CHUNK_SIZE
            silence_frames_threshold = int(SILENCE_DURATION * SAMPLE_RATE / frames_per_chunk)
            
            if self.silence_frames >= silence_frames_threshold:
                self.speech_frames = 0

        # Update circular buffer
        with self.buffer_lock:
            self.buffer = np.roll(self.buffer, -len(audio_data))
            self.buffer[-len(audio_data):] = audio_data

        if WAKE_WORD_ENABLED:
            # Process for wake word detection
            self.last_prediction = self.wake_word_model.predict(audio_data)
            
            # Check if wake word detected
            for wake_word in WAKE_WORDS:
                confidence = self.last_prediction[wake_word]
                if confidence > DETECTION_THRESHOLD:
                    logger.info(
                        f"Wake word: {WAKE_WORD_ALIAS} (confidence: {confidence:.2f})"
                    )
                    self.process_audio()
                    break
        else:
            # Continuous transcription mode
            if self.should_transcribe():
                self.process_audio()

    def process_audio(self):
        """Process the current audio buffer (save and transcribe)"""
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"/audio/audio_segment_{timestamp}.wav"
        
        # Save the audio buffer to a WAV file
        with wave.open(filename, 'wb') as wf:
            wf.setnchannels(CHANNELS)
            wf.setsampwidth(2)  # 16-bit audio
            wf.setframerate(SAMPLE_RATE)
            
            # Convert float32 to int16
            audio_data = (self.buffer * 32767).astype(np.int16)
            wf.writeframes(audio_data.tobytes())
        
        logger.info(f"Saved audio segment to {filename}")

        # Transcribe the audio with German language preference
        try:
            segments, info = asr_model.transcribe(
                filename,
                language="de",  # Set German as preferred language
                beam_size=5,
                temperature=0
            )
            
            # Get the full transcribed text
            transcribed_text = " ".join(segment.text for segment in segments)
            logger.info(f"Transcribed text: {transcribed_text}")
            
            # Process the command
            process_command(transcribed_text)
                
        except Exception as e:
            logger.error(f"Error during transcription or processing: {e}")

    def start(self):
        """Start audio processing"""
        try:
            logger.info("Starting audio processor...")
            
            # Log configuration
            logger.debug(f"Sample Rate: {SAMPLE_RATE}")
            logger.debug(f"Channels: {CHANNELS}")
            logger.debug(f"Chunk Size: {CHUNK_SIZE}")
            logger.debug(f"Buffer Duration: {BUFFER_DURATION}")
            logger.debug(f"Wake Word Enabled: {WAKE_WORD_ENABLED}")
            logger.debug(f"Speech Enabled: {SPEECH_ENABLED}")
            logger.debug(f"ASR Model: {os.environ.get('ASR_MODEL')}")
            
            if WAKE_WORD_ENABLED:
                logger.info("Initializing wake word detection...")
                logger.info(f"Loaded wake words: {', '.join(WAKE_WORDS)}")
            else:
                logger.info("Starting continuous transcription mode...")
                interval = CONTINUOUS_TRANSCRIPTION_INTERVAL
                logger.info(f"Will transcribe every {interval} seconds")
            
            try:
                logger.debug("Setting up audio input stream...")
                with sd.InputStream(
                    channels=CHANNELS,
                    samplerate=SAMPLE_RATE,
                    blocksize=CHUNK_SIZE,
                    callback=self._audio_callback
                ):
                    logger.info("Audio input stream started successfully")
                    logger.info("Listening for audio input...")
                    logger.info("Press Ctrl+C to stop")
                    
                    while True:
                        sd.sleep(1000)  # Sleep for 1 second
                        
            except sd.PortAudioError as e:
                logger.error(f"Error setting up audio stream: {e}")
                logger.error("Check if microphone is connected and accessible")
                raise
            except Exception as e:
                logger.error(f"Unexpected error in audio stream: {e}")
                raise
                    
        except KeyboardInterrupt:
            logger.info("\nStopping audio processing...")
        except Exception as e:
            logger.error("Critical error in audio processing", exc_info=True)
            raise

if __name__ == "__main__":
    try:
        logger.info("Initializing AudioProcessor...")
        processor = AudioProcessor()
        processor.start()
    except Exception as e:
        logger.error("Failed to start AudioProcessor", exc_info=True)
        raise 