src/tts_utils.py

import io

import librosa
import numpy as np
import pydub

from src.utils import have_pyrubberband


# Keep non-native package imports out of global space


def get_wave_header(frame_input=b"", channels=1, sample_width=2, sample_rate=24000):
    # This will create a wave header then append the frame input
    # It should be first on a streaming wav file
    # Other frames better should not have it (else you will hear some artifacts each chunk start)
    import wave
    wav_buf = io.BytesIO()
    with wave.open(wav_buf, "wb") as vfout:
        vfout.setnchannels(channels)
        vfout.setsampwidth(sample_width)
        vfout.setframerate(sample_rate)
        vfout.writeframes(frame_input)

    wav_buf.seek(0)
    return wav_buf.read()


def prepare_speech(sr=24000):
    # Must set autoplay to True first
    return get_wave_header(sample_rate=sr)


def get_no_audio(return_as_byte=True, return_nonbyte_as_file=False, sr=None):
    if return_as_byte:
        return b""
    else:
        if return_nonbyte_as_file:
            return None
        else:
            assert sr is not None
            return sr, np.array([]).astype(np.int16)


def combine_audios(audios, audio=None, channels=1, sample_width=2, sr=24000, expect_bytes=True):
    no_audio = get_no_audio(sr=sr)
    have_audio = any(x not in [no_audio, None, ''] for x in audios) or audio not in [no_audio, None, '']
    if not have_audio:
        return no_audio

    if audio or audios:
        is_bytes = expect_bytes  # force default as bytes no matter input if know should have been bytes
        if audios:
            is_bytes |= isinstance(audios[0], (bytes, bytearray))
        if audio:
            is_bytes |= isinstance(audio, (bytes, bytearray))
        assert audio is None or isinstance(audio, (bytes, bytearray))
        from pydub import AudioSegment
        combined_wav = AudioSegment.empty()
        for x in audios:
            if x is not None:
                s = io.BytesIO(x) if is_bytes else x
                combined_wav += AudioSegment.from_raw(s, sample_width=sample_width, frame_rate=sr, channels=channels)
        if audio is not None:
            s = io.BytesIO(audio) if is_bytes else audio
            combined_wav += AudioSegment.from_raw(s, sample_width=sample_width, frame_rate=sr, channels=channels)
        if is_bytes:
            combined_wav = combined_wav.export(format='raw').read()
        return combined_wav
    # audio just empty stream, but not None, else would nuke audio
    return audio


def chunk_speed_change(chunk, sr, tts_speed=1.0):
    if tts_speed == 1.0:
        return chunk

    if have_pyrubberband:
        import pyrubberband as pyrb
        chunk = pyrb.time_stretch(chunk, sr, tts_speed)
        chunk = (chunk * 32767).astype(np.int16)
        return chunk

    if tts_speed < 1.0:
        # chunk = chunk.astype(np.float32)
        # chunk = 0.5 * chunk / np.max(chunk)
        # chunk = librosa.effects.time_stretch(chunk, rate=tts_speed)
        return chunk

    # speed-up
    from pydub import AudioSegment
    from pydub.effects import speedup

    s = io.BytesIO(chunk)
    channels = 1
    sample_width = 2
    audio = AudioSegment.from_raw(s, sample_width=sample_width, frame_rate=sr, channels=channels)
    # chunk = speedup(audio, tts_speed, 150).export(format='raw').read()
    chunk = pydub_to_np(speedup(audio, tts_speed, 150))
    # audio = audio._spawn(audio.raw_data, overrides={
    #    "frame_rate": int(audio.frame_rate * tts_speed)
    # })
    # chunk = np.array(audio.get_array_of_samples())

    return chunk


def pydub_to_np(audio: pydub.AudioSegment) -> (np.ndarray, int):
    """
    Converts pydub audio segment into np.int16 of shape [duration_in_seconds*sample_rate, channels],
    """
    return np.array(audio.get_array_of_samples(), dtype=np.int16).reshape((-1, audio.channels))