344 lines
11 KiB
Python
344 lines
11 KiB
Python
|
from typing import Dict
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import torch
|
||
|
|
from torch import nn
|
||
|
|
|
||
|
|
|
||
|
|
def numpy_to_torch(np_array, dtype, cuda=False, device="cpu"):
|
||
|
|
if cuda:
|
||
|
|
device = "cuda"
|
||
|
|
if np_array is None:
|
||
|
|
return None
|
||
|
|
tensor = torch.as_tensor(np_array, dtype=dtype, device=device)
|
||
|
|
return tensor
|
||
|
|
|
||
|
|
|
||
|
|
def compute_style_mel(style_wav, ap, cuda=False, device="cpu"):
|
||
|
|
if cuda:
|
||
|
|
device = "cuda"
|
||
|
|
style_mel = torch.FloatTensor(
|
||
|
|
ap.melspectrogram(ap.load_wav(style_wav, sr=ap.sample_rate)),
|
||
|
|
device=device,
|
||
|
|
).unsqueeze(0)
|
||
|
|
return style_mel
|
||
|
|
|
||
|
|
|
||
|
|
def run_model_torch(
|
||
|
|
model: nn.Module,
|
||
|
|
inputs: torch.Tensor,
|
||
|
|
speaker_id: int = None,
|
||
|
|
style_mel: torch.Tensor = None,
|
||
|
|
style_text: str = None,
|
||
|
|
d_vector: torch.Tensor = None,
|
||
|
|
language_id: torch.Tensor = None,
|
||
|
|
) -> Dict:
|
||
|
|
"""Run a torch model for inference. It does not support batch inference.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
model (nn.Module): The model to run inference.
|
||
|
|
inputs (torch.Tensor): Input tensor with character ids.
|
||
|
|
speaker_id (int, optional): Input speaker ids for multi-speaker models. Defaults to None.
|
||
|
|
style_mel (torch.Tensor, optional): Spectrograms used for voice styling . Defaults to None.
|
||
|
|
d_vector (torch.Tensor, optional): d-vector for multi-speaker models . Defaults to None.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Dict: model outputs.
|
||
|
|
"""
|
||
|
|
input_lengths = torch.tensor(inputs.shape[1:2]).to(inputs.device)
|
||
|
|
if hasattr(model, "module"):
|
||
|
|
_func = model.module.inference
|
||
|
|
else:
|
||
|
|
_func = model.inference
|
||
|
|
outputs = _func(
|
||
|
|
inputs,
|
||
|
|
aux_input={
|
||
|
|
"x_lengths": input_lengths,
|
||
|
|
"speaker_ids": speaker_id,
|
||
|
|
"d_vectors": d_vector,
|
||
|
|
"style_mel": style_mel,
|
||
|
|
"style_text": style_text,
|
||
|
|
"language_ids": language_id,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
return outputs
|
||
|
|
|
||
|
|
|
||
|
|
def trim_silence(wav, ap):
|
||
|
|
return wav[: ap.find_endpoint(wav)]
|
||
|
|
|
||
|
|
|
||
|
|
def inv_spectrogram(postnet_output, ap, CONFIG):
|
||
|
|
if CONFIG.model.lower() in ["tacotron"]:
|
||
|
|
wav = ap.inv_spectrogram(postnet_output.T)
|
||
|
|
else:
|
||
|
|
wav = ap.inv_melspectrogram(postnet_output.T)
|
||
|
|
return wav
|
||
|
|
|
||
|
|
|
||
|
|
def id_to_torch(aux_id, cuda=False, device="cpu"):
|
||
|
|
if cuda:
|
||
|
|
device = "cuda"
|
||
|
|
if aux_id is not None:
|
||
|
|
aux_id = np.asarray(aux_id)
|
||
|
|
aux_id = torch.from_numpy(aux_id).to(device)
|
||
|
|
return aux_id
|
||
|
|
|
||
|
|
|
||
|
|
def embedding_to_torch(d_vector, cuda=False, device="cpu"):
|
||
|
|
if cuda:
|
||
|
|
device = "cuda"
|
||
|
|
if d_vector is not None:
|
||
|
|
d_vector = np.asarray(d_vector)
|
||
|
|
d_vector = torch.from_numpy(d_vector).type(torch.FloatTensor)
|
||
|
|
d_vector = d_vector.squeeze().unsqueeze(0).to(device)
|
||
|
|
return d_vector
|
||
|
|
|
||
|
|
|
||
|
|
# TODO: perform GL with pytorch for batching
|
||
|
|
def apply_griffin_lim(inputs, input_lens, CONFIG, ap):
|
||
|
|
"""Apply griffin-lim to each sample iterating throught the first dimension.
|
||
|
|
Args:
|
||
|
|
inputs (Tensor or np.Array): Features to be converted by GL. First dimension is the batch size.
|
||
|
|
input_lens (Tensor or np.Array): 1D array of sample lengths.
|
||
|
|
CONFIG (Dict): TTS config.
|
||
|
|
ap (AudioProcessor): TTS audio processor.
|
||
|
|
"""
|
||
|
|
wavs = []
|
||
|
|
for idx, spec in enumerate(inputs):
|
||
|
|
wav_len = (input_lens[idx] * ap.hop_length) - ap.hop_length # inverse librosa padding
|
||
|
|
wav = inv_spectrogram(spec, ap, CONFIG)
|
||
|
|
# assert len(wav) == wav_len, f" [!] wav lenght: {len(wav)} vs expected: {wav_len}"
|
||
|
|
wavs.append(wav[:wav_len])
|
||
|
|
return wavs
|
||
|
|
|
||
|
|
|
||
|
|
def synthesis(
|
||
|
|
model,
|
||
|
|
text,
|
||
|
|
CONFIG,
|
||
|
|
use_cuda,
|
||
|
|
speaker_id=None,
|
||
|
|
style_wav=None,
|
||
|
|
style_text=None,
|
||
|
|
use_griffin_lim=False,
|
||
|
|
do_trim_silence=False,
|
||
|
|
d_vector=None,
|
||
|
|
language_id=None,
|
||
|
|
):
|
||
|
|
"""Synthesize voice for the given text using Griffin-Lim vocoder or just compute output features to be passed to
|
||
|
|
the vocoder model.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
model (TTS.tts.models):
|
||
|
|
The TTS model to synthesize audio with.
|
||
|
|
|
||
|
|
text (str):
|
||
|
|
The input text to convert to speech.
|
||
|
|
|
||
|
|
CONFIG (Coqpit):
|
||
|
|
Model configuration.
|
||
|
|
|
||
|
|
use_cuda (bool):
|
||
|
|
Enable/disable CUDA.
|
||
|
|
|
||
|
|
speaker_id (int):
|
||
|
|
Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None.
|
||
|
|
|
||
|
|
style_wav (str | Dict[str, float]):
|
||
|
|
Path or tensor to/of a waveform used for computing the style embedding based on GST or Capacitron.
|
||
|
|
Defaults to None, meaning that Capacitron models will sample from the prior distribution to
|
||
|
|
generate random but realistic prosody.
|
||
|
|
|
||
|
|
style_text (str):
|
||
|
|
Transcription of style_wav for Capacitron models. Defaults to None.
|
||
|
|
|
||
|
|
enable_eos_bos_chars (bool):
|
||
|
|
enable special chars for end of sentence and start of sentence. Defaults to False.
|
||
|
|
|
||
|
|
do_trim_silence (bool):
|
||
|
|
trim silence after synthesis. Defaults to False.
|
||
|
|
|
||
|
|
d_vector (torch.Tensor):
|
||
|
|
d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None.
|
||
|
|
|
||
|
|
language_id (int):
|
||
|
|
Language ID passed to the language embedding layer in multi-langual model. Defaults to None.
|
||
|
|
"""
|
||
|
|
# device
|
||
|
|
device = next(model.parameters()).device
|
||
|
|
if use_cuda:
|
||
|
|
device = "cuda"
|
||
|
|
|
||
|
|
# GST or Capacitron processing
|
||
|
|
# TODO: need to handle the case of setting both gst and capacitron to true somewhere
|
||
|
|
style_mel = None
|
||
|
|
if CONFIG.has("gst") and CONFIG.gst and style_wav is not None:
|
||
|
|
if isinstance(style_wav, dict):
|
||
|
|
style_mel = style_wav
|
||
|
|
else:
|
||
|
|
style_mel = compute_style_mel(style_wav, model.ap, device=device)
|
||
|
|
|
||
|
|
if CONFIG.has("capacitron_vae") and CONFIG.use_capacitron_vae and style_wav is not None:
|
||
|
|
style_mel = compute_style_mel(style_wav, model.ap, device=device)
|
||
|
|
style_mel = style_mel.transpose(1, 2) # [1, time, depth]
|
||
|
|
|
||
|
|
language_name = None
|
||
|
|
if language_id is not None:
|
||
|
|
language = [k for k, v in model.language_manager.name_to_id.items() if v == language_id]
|
||
|
|
assert len(language) == 1, "language_id must be a valid language"
|
||
|
|
language_name = language[0]
|
||
|
|
|
||
|
|
# convert text to sequence of token IDs
|
||
|
|
text_inputs = np.asarray(
|
||
|
|
model.tokenizer.text_to_ids(text, language=language_name),
|
||
|
|
dtype=np.int32,
|
||
|
|
)
|
||
|
|
# pass tensors to backend
|
||
|
|
if speaker_id is not None:
|
||
|
|
speaker_id = id_to_torch(speaker_id, device=device)
|
||
|
|
|
||
|
|
if d_vector is not None:
|
||
|
|
d_vector = embedding_to_torch(d_vector, device=device)
|
||
|
|
|
||
|
|
if language_id is not None:
|
||
|
|
language_id = id_to_torch(language_id, device=device)
|
||
|
|
|
||
|
|
if not isinstance(style_mel, dict):
|
||
|
|
# GST or Capacitron style mel
|
||
|
|
style_mel = numpy_to_torch(style_mel, torch.float, device=device)
|
||
|
|
if style_text is not None:
|
||
|
|
style_text = np.asarray(
|
||
|
|
model.tokenizer.text_to_ids(style_text, language=language_id),
|
||
|
|
dtype=np.int32,
|
||
|
|
)
|
||
|
|
style_text = numpy_to_torch(style_text, torch.long, device=device)
|
||
|
|
style_text = style_text.unsqueeze(0)
|
||
|
|
|
||
|
|
text_inputs = numpy_to_torch(text_inputs, torch.long, device=device)
|
||
|
|
text_inputs = text_inputs.unsqueeze(0)
|
||
|
|
# synthesize voice
|
||
|
|
outputs = run_model_torch(
|
||
|
|
model,
|
||
|
|
text_inputs,
|
||
|
|
speaker_id,
|
||
|
|
style_mel,
|
||
|
|
style_text,
|
||
|
|
d_vector=d_vector,
|
||
|
|
language_id=language_id,
|
||
|
|
)
|
||
|
|
model_outputs = outputs["model_outputs"]
|
||
|
|
model_outputs = model_outputs[0].data.cpu().numpy()
|
||
|
|
alignments = outputs["alignments"]
|
||
|
|
|
||
|
|
# convert outputs to numpy
|
||
|
|
# plot results
|
||
|
|
wav = None
|
||
|
|
model_outputs = model_outputs.squeeze()
|
||
|
|
if model_outputs.ndim == 2: # [T, C_spec]
|
||
|
|
if use_griffin_lim:
|
||
|
|
wav = inv_spectrogram(model_outputs, model.ap, CONFIG)
|
||
|
|
# trim silence
|
||
|
|
if do_trim_silence:
|
||
|
|
wav = trim_silence(wav, model.ap)
|
||
|
|
else: # [T,]
|
||
|
|
wav = model_outputs
|
||
|
|
return_dict = {
|
||
|
|
"wav": wav,
|
||
|
|
"alignments": alignments,
|
||
|
|
"text_inputs": text_inputs,
|
||
|
|
"outputs": outputs,
|
||
|
|
}
|
||
|
|
return return_dict
|
||
|
|
|
||
|
|
|
||
|
|
def transfer_voice(
|
||
|
|
model,
|
||
|
|
CONFIG,
|
||
|
|
use_cuda,
|
||
|
|
reference_wav,
|
||
|
|
speaker_id=None,
|
||
|
|
d_vector=None,
|
||
|
|
reference_speaker_id=None,
|
||
|
|
reference_d_vector=None,
|
||
|
|
do_trim_silence=False,
|
||
|
|
use_griffin_lim=False,
|
||
|
|
):
|
||
|
|
"""Synthesize voice for the given text using Griffin-Lim vocoder or just compute output features to be passed to
|
||
|
|
the vocoder model.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
model (TTS.tts.models):
|
||
|
|
The TTS model to synthesize audio with.
|
||
|
|
|
||
|
|
CONFIG (Coqpit):
|
||
|
|
Model configuration.
|
||
|
|
|
||
|
|
use_cuda (bool):
|
||
|
|
Enable/disable CUDA.
|
||
|
|
|
||
|
|
reference_wav (str):
|
||
|
|
Path of reference_wav to be used to voice conversion.
|
||
|
|
|
||
|
|
speaker_id (int):
|
||
|
|
Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None.
|
||
|
|
|
||
|
|
d_vector (torch.Tensor):
|
||
|
|
d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None.
|
||
|
|
|
||
|
|
reference_speaker_id (int):
|
||
|
|
Reference Speaker ID passed to the speaker embedding layer in multi-speaker model. Defaults to None.
|
||
|
|
|
||
|
|
reference_d_vector (torch.Tensor):
|
||
|
|
Reference d-vector for multi-speaker models in share :math:`[1, D]`. Defaults to None.
|
||
|
|
|
||
|
|
enable_eos_bos_chars (bool):
|
||
|
|
enable special chars for end of sentence and start of sentence. Defaults to False.
|
||
|
|
|
||
|
|
do_trim_silence (bool):
|
||
|
|
trim silence after synthesis. Defaults to False.
|
||
|
|
"""
|
||
|
|
# device
|
||
|
|
device = next(model.parameters()).device
|
||
|
|
if use_cuda:
|
||
|
|
device = "cuda"
|
||
|
|
|
||
|
|
# pass tensors to backend
|
||
|
|
if speaker_id is not None:
|
||
|
|
speaker_id = id_to_torch(speaker_id, device=device)
|
||
|
|
|
||
|
|
if d_vector is not None:
|
||
|
|
d_vector = embedding_to_torch(d_vector, device=device)
|
||
|
|
|
||
|
|
if reference_d_vector is not None:
|
||
|
|
reference_d_vector = embedding_to_torch(reference_d_vector, device=device)
|
||
|
|
|
||
|
|
# load reference_wav audio
|
||
|
|
reference_wav = embedding_to_torch(
|
||
|
|
model.ap.load_wav(
|
||
|
|
reference_wav, sr=model.args.encoder_sample_rate if model.args.encoder_sample_rate else model.ap.sample_rate
|
||
|
|
),
|
||
|
|
device=device,
|
||
|
|
)
|
||
|
|
|
||
|
|
if hasattr(model, "module"):
|
||
|
|
_func = model.module.inference_voice_conversion
|
||
|
|
else:
|
||
|
|
_func = model.inference_voice_conversion
|
||
|
|
model_outputs = _func(reference_wav, speaker_id, d_vector, reference_speaker_id, reference_d_vector)
|
||
|
|
|
||
|
|
# convert outputs to numpy
|
||
|
|
# plot results
|
||
|
|
wav = None
|
||
|
|
model_outputs = model_outputs.squeeze()
|
||
|
|
if model_outputs.ndim == 2: # [T, C_spec]
|
||
|
|
if use_griffin_lim:
|
||
|
|
wav = inv_spectrogram(model_outputs, model.ap, CONFIG)
|
||
|
|
# trim silence
|
||
|
|
if do_trim_silence:
|
||
|
|
wav = trim_silence(wav, model.ap)
|
||
|
|
else: # [T,]
|
||
|
|
wav = model_outputs
|
||
|
|
|
||
|
|
return wav
|