Initial commit
This commit is contained in:
王新升
@@ -0,0 +1,100 @@
|
||||
import librosa
|
||||
import numpy as np
|
||||
import wave
|
||||
import soundfile as sf
|
||||
|
||||
|
||||
def librosa_pad_lr(x, fsize, fshift, pad_sides=1):
|
||||
'''compute right padding (final frame) or both sides padding (first and final frames)
|
||||
'''
|
||||
assert pad_sides in (1, 2)
|
||||
# return int(fsize // 2)
|
||||
pad = (x.shape[0] // fshift + 1) * fshift - x.shape[0]
|
||||
if pad_sides == 1:
|
||||
return 0, pad
|
||||
else:
|
||||
return pad // 2, pad // 2 + pad % 2
|
||||
|
||||
|
||||
def amp_to_db(x):
|
||||
return 20 * np.log10(np.maximum(1e-5, x))
|
||||
|
||||
|
||||
def db_to_amp(x):
|
||||
return 10.0 ** (x * 0.05)
|
||||
|
||||
|
||||
def normalize(S, min_level_db):
|
||||
return (S - min_level_db) / -min_level_db
|
||||
|
||||
|
||||
def denormalize(D, min_level_db):
|
||||
return (D * -min_level_db) + min_level_db
|
||||
|
||||
|
||||
def librosa_wav2spec(wav_path,
|
||||
fft_size=1024,
|
||||
hop_size=256,
|
||||
win_length=1024,
|
||||
window="hann",
|
||||
num_mels=80,
|
||||
fmin=80,
|
||||
fmax=-1,
|
||||
eps=1e-6,
|
||||
sample_rate=22050,
|
||||
loud_norm=False,
|
||||
trim_long_sil=False):
|
||||
import pyloudnorm as pyln
|
||||
if isinstance(wav_path, str):
|
||||
if trim_long_sil:
|
||||
from .vad import trim_long_silences
|
||||
wav, _, _ = trim_long_silences(wav_path, sample_rate)
|
||||
else:
|
||||
wav, _ = librosa.core.load(wav_path, sr=sample_rate)
|
||||
else:
|
||||
wav = wav_path
|
||||
wav_orig = np.copy(wav)
|
||||
|
||||
if loud_norm:
|
||||
meter = pyln.Meter(sample_rate) # create BS.1770 meter
|
||||
loudness = meter.integrated_loudness(wav)
|
||||
wav = pyln.normalize.loudness(wav, loudness, -22.0)
|
||||
if np.abs(wav).max() > 1:
|
||||
wav = wav / np.abs(wav).max()
|
||||
|
||||
# get amplitude spectrogram
|
||||
x_stft = librosa.stft(wav, n_fft=fft_size, hop_length=hop_size,
|
||||
win_length=win_length, window=window, pad_mode="constant")
|
||||
linear_spc = np.abs(x_stft) # (n_bins, T)
|
||||
|
||||
# get mel basis
|
||||
fmin = 0 if fmin == -1 else fmin
|
||||
fmax = sample_rate / 2 if fmax == -1 else fmax
|
||||
mel_basis = librosa.filters.mel(sr=sample_rate, n_fft=fft_size, n_mels=num_mels, fmin=fmin, fmax=fmax)
|
||||
|
||||
# calculate mel spec
|
||||
mel = mel_basis @ linear_spc
|
||||
mel = np.log10(np.maximum(eps, mel)) # (n_mel_bins, T)
|
||||
l_pad, r_pad = librosa_pad_lr(wav, fft_size, hop_size, 1)
|
||||
wav = np.pad(wav, (l_pad, r_pad), mode='constant', constant_values=0.0)
|
||||
wav = wav[:mel.shape[1] * hop_size]
|
||||
|
||||
# log linear spec
|
||||
linear_spc = np.log10(np.maximum(eps, linear_spc))
|
||||
return {'wav': wav, 'mel': mel.T, 'linear': linear_spc.T, 'mel_basis': mel_basis, 'wav_orig': wav_orig}
|
||||
|
||||
def get_wav_num_frames(path, sr=None):
|
||||
try:
|
||||
with wave.open(path, 'rb') as f:
|
||||
sr_ = f.getframerate()
|
||||
if sr is None:
|
||||
sr = sr_
|
||||
return int(f.getnframes() / (sr_ / sr))
|
||||
except wave.Error:
|
||||
wav_file, sr_ = sf.read(path, dtype='float32')
|
||||
if sr is None:
|
||||
sr = sr_
|
||||
return int(len(wav_file) / (sr_ / sr))
|
||||
except:
|
||||
wav_file, sr_ = librosa.core.load(path, sr=sr)
|
||||
return len(wav_file)
|
||||
@@ -0,0 +1,90 @@
|
||||
import re
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
from ..text.text_encoder import is_sil_phoneme
|
||||
|
||||
|
||||
def get_mel2ph(tg_fn, ph, mel, hop_size, audio_sample_rate, min_sil_duration=0):
|
||||
from textgrid import TextGrid
|
||||
ph_list = ph.split(" ")
|
||||
itvs = TextGrid.fromFile(tg_fn)[1]
|
||||
itvs_ = []
|
||||
for i in range(len(itvs)):
|
||||
if itvs[i].maxTime - itvs[i].minTime < min_sil_duration and i > 0 and is_sil_phoneme(itvs[i].mark):
|
||||
itvs_[-1].maxTime = itvs[i].maxTime
|
||||
else:
|
||||
itvs_.append(itvs[i])
|
||||
itvs.intervals = itvs_
|
||||
itv_marks = [itv.mark for itv in itvs]
|
||||
tg_len = len([x for x in itvs if not is_sil_phoneme(x.mark)])
|
||||
ph_len = len([x for x in ph_list if not is_sil_phoneme(x)])
|
||||
assert tg_len == ph_len, (tg_len, ph_len, itv_marks, ph_list, tg_fn)
|
||||
mel2ph = np.zeros([mel.shape[0]], int)
|
||||
i_itv = 0
|
||||
i_ph = 0
|
||||
while i_itv < len(itvs):
|
||||
itv = itvs[i_itv]
|
||||
ph = ph_list[i_ph]
|
||||
itv_ph = itv.mark
|
||||
start_frame = int(itv.minTime * audio_sample_rate / hop_size + 0.5)
|
||||
end_frame = int(itv.maxTime * audio_sample_rate / hop_size + 0.5)
|
||||
if is_sil_phoneme(itv_ph) and not is_sil_phoneme(ph):
|
||||
mel2ph[start_frame:end_frame] = i_ph
|
||||
i_itv += 1
|
||||
elif not is_sil_phoneme(itv_ph) and is_sil_phoneme(ph):
|
||||
i_ph += 1
|
||||
else:
|
||||
if not ((is_sil_phoneme(itv_ph) and is_sil_phoneme(ph)) \
|
||||
or re.sub(r'\d+', '', itv_ph.lower()) == re.sub(r'\d+', '', ph.lower())):
|
||||
print(f"| WARN: {tg_fn} phs are not same: ", itv_ph, ph, itv_marks, ph_list)
|
||||
mel2ph[start_frame:end_frame] = i_ph + 1
|
||||
i_ph += 1
|
||||
i_itv += 1
|
||||
mel2ph[-1] = mel2ph[-2]
|
||||
assert not np.any(mel2ph == 0)
|
||||
T_t = len(ph_list)
|
||||
dur = mel2token_to_dur(mel2ph, T_t)
|
||||
return mel2ph.tolist(), dur.tolist()
|
||||
|
||||
|
||||
def split_audio_by_mel2ph(audio, mel2ph, hop_size, audio_num_mel_bins):
|
||||
if isinstance(audio, torch.Tensor):
|
||||
audio = audio.numpy()
|
||||
if isinstance(mel2ph, torch.Tensor):
|
||||
mel2ph = mel2ph.numpy()
|
||||
assert len(audio.shape) == 1, len(mel2ph.shape) == 1
|
||||
split_locs = []
|
||||
for i in range(1, len(mel2ph)):
|
||||
if mel2ph[i] != mel2ph[i - 1]:
|
||||
split_loc = i * hop_size
|
||||
split_locs.append(split_loc)
|
||||
|
||||
new_audio = []
|
||||
for i in range(len(split_locs) - 1):
|
||||
new_audio.append(audio[split_locs[i]:split_locs[i + 1]])
|
||||
new_audio.append(np.zeros([0.5 * audio_num_mel_bins]))
|
||||
return np.concatenate(new_audio)
|
||||
|
||||
|
||||
def mel2token_to_dur(mel2token, T_txt=None, max_dur=None):
|
||||
is_torch = isinstance(mel2token, torch.Tensor)
|
||||
has_batch_dim = True
|
||||
if not is_torch:
|
||||
mel2token = torch.LongTensor(mel2token)
|
||||
if T_txt is None:
|
||||
T_txt = mel2token.max()
|
||||
if len(mel2token.shape) == 1:
|
||||
mel2token = mel2token[None, ...]
|
||||
has_batch_dim = False
|
||||
B, _ = mel2token.shape
|
||||
dur = mel2token.new_zeros(B, T_txt + 1).scatter_add(1, mel2token, torch.ones_like(mel2token))
|
||||
dur = dur[:, 1:]
|
||||
if max_dur is not None:
|
||||
dur = dur.clamp(max=max_dur)
|
||||
if not is_torch:
|
||||
dur = dur.numpy()
|
||||
if not has_batch_dim:
|
||||
dur = dur[0]
|
||||
return dur
|
||||
@@ -0,0 +1,22 @@
|
||||
import subprocess
|
||||
|
||||
import numpy as np
|
||||
from scipy.io import wavfile
|
||||
|
||||
|
||||
def save_wav(wav, path, sr, norm=False):
|
||||
if norm:
|
||||
wav = wav / np.abs(wav).max()
|
||||
wav = wav * 32767
|
||||
wavfile.write(path[:-4] + '.wav', sr, wav.astype(np.int16))
|
||||
if path[-4:] == '.mp3':
|
||||
to_mp3(path[:-4])
|
||||
|
||||
|
||||
def to_mp3(out_path):
|
||||
if out_path[-4:] == '.wav':
|
||||
out_path = out_path[:-4]
|
||||
subprocess.check_call(
|
||||
f'ffmpeg -threads 1 -loglevel error -i "{out_path}.wav" -vn -b:a 192k -y -hide_banner -async 1 "{out_path}.mp3"',
|
||||
shell=True, stdin=subprocess.PIPE)
|
||||
subprocess.check_call(f'rm -f "{out_path}.wav"', shell=True)
|
||||
@@ -0,0 +1,139 @@
|
||||
import math
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.utils.data
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
from scipy.io.wavfile import read
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
MAX_WAV_VALUE = 32768.0
|
||||
|
||||
|
||||
def load_wav(full_path):
|
||||
sampling_rate, data = read(full_path)
|
||||
return data, sampling_rate
|
||||
|
||||
|
||||
def dynamic_range_compression(x, C=1, clip_val=1e-5):
|
||||
return np.log10(np.clip(x, a_min=clip_val, a_max=None) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression(x, C=1):
|
||||
return np.exp(x) / C
|
||||
|
||||
|
||||
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
|
||||
return torch.log10(torch.clamp(x, min=clip_val) * C)
|
||||
|
||||
|
||||
def dynamic_range_decompression_torch(x, C=1):
|
||||
return torch.exp(x) / C
|
||||
|
||||
|
||||
def spectral_normalize_torch(magnitudes):
|
||||
output = dynamic_range_compression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
def spectral_de_normalize_torch(magnitudes):
|
||||
output = dynamic_range_decompression_torch(magnitudes)
|
||||
return output
|
||||
|
||||
|
||||
class MelNet(nn.Module):
|
||||
def __init__(self, hparams, device='cpu') -> None:
|
||||
super().__init__()
|
||||
self.n_fft = hparams['fft_size']
|
||||
self.num_mels = hparams['audio_num_mel_bins']
|
||||
self.sampling_rate = hparams['audio_sample_rate']
|
||||
self.hop_size = hparams['hop_size']
|
||||
self.win_size = hparams['win_size']
|
||||
self.fmin = hparams['fmin']
|
||||
self.fmax = hparams['fmax']
|
||||
self.device = device
|
||||
|
||||
mel = librosa_mel_fn(sr=self.sampling_rate, n_fft=self.n_fft, n_mels=self.num_mels, fmin=self.fmin,
|
||||
fmax=self.fmax)
|
||||
self.mel_basis = torch.from_numpy(mel).float().to(self.device)
|
||||
self.hann_window = torch.hann_window(self.win_size).to(self.device)
|
||||
|
||||
def to(self, device, **kwagrs):
|
||||
super().to(device=device, **kwagrs)
|
||||
self.mel_basis = self.mel_basis.to(device)
|
||||
self.hann_window = self.hann_window.to(device)
|
||||
self.device = device
|
||||
|
||||
def forward(self, y, center=False, complex=False):
|
||||
if isinstance(y, np.ndarray):
|
||||
y = torch.FloatTensor(y)
|
||||
if len(y.shape) == 1:
|
||||
y = y.unsqueeze(0)
|
||||
y = y.clamp(min=-1., max=1.).to(self.device)
|
||||
|
||||
pad_length = math.ceil(y.shape[1] / self.hop_size) * self.hop_size - y.shape[1]
|
||||
y = torch.nn.functional.pad(y.unsqueeze(1),
|
||||
[int((self.n_fft - self.hop_size) / 2),
|
||||
int((self.n_fft - self.hop_size) / 2 + pad_length)],
|
||||
mode='reflect')
|
||||
y = y.squeeze(1)
|
||||
|
||||
spec = torch.stft(y, self.n_fft, hop_length=self.hop_size, win_length=self.win_size, window=self.hann_window,
|
||||
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=True)
|
||||
if not complex:
|
||||
spec = torch.view_as_real(spec)
|
||||
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) # [B, n_fft, T]
|
||||
spec = torch.matmul(self.mel_basis, spec)
|
||||
spec = spectral_normalize_torch(spec)
|
||||
spec = spec.transpose(1, 2) # [B, T, n_fft]
|
||||
else:
|
||||
B, C, T, _ = spec.shape
|
||||
spec = spec.transpose(1, 2) # [B, T, n_fft, 2]
|
||||
return spec
|
||||
|
||||
|
||||
## below can be used in one gpu, but not ddp
|
||||
mel_basis = {}
|
||||
hann_window = {}
|
||||
|
||||
|
||||
def mel_spectrogram(y, hparams, center=False, complex=False): # y should be a tensor with shape (b,wav_len)
|
||||
# hop_size: 512 # For 22050Hz, 275 ~= 12.5 ms (0.0125 * sample_rate)
|
||||
# win_size: 2048 # For 22050Hz, 1100 ~= 50 ms (If None, win_size: fft_size) (0.05 * sample_rate)
|
||||
# fmin: 55 # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
|
||||
# fmax: 10000 # To be increased/reduced depending on data.
|
||||
# fft_size: 2048 # Extra window size is filled with 0 paddings to match this parameter
|
||||
# n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax,
|
||||
n_fft = hparams['fft_size']
|
||||
num_mels = hparams['audio_num_mel_bins']
|
||||
sampling_rate = hparams['audio_sample_rate']
|
||||
hop_size = hparams['hop_size']
|
||||
win_size = hparams['win_size']
|
||||
fmin = hparams['fmin']
|
||||
fmax = hparams['fmax']
|
||||
if isinstance(y, np.ndarray):
|
||||
y = torch.FloatTensor(y)
|
||||
if len(y.shape) == 1:
|
||||
y = y.unsqueeze(0)
|
||||
y = y.clamp(min=-1., max=1.)
|
||||
global mel_basis, hann_window
|
||||
if fmax not in mel_basis:
|
||||
mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
|
||||
mel_basis[str(fmax) + '_' + str(y.device)] = torch.from_numpy(mel).float().to(y.device)
|
||||
hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device)
|
||||
|
||||
y = torch.nn.functional.pad(y.unsqueeze(1), [int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)],
|
||||
mode='reflect')
|
||||
y = y.squeeze(1)
|
||||
|
||||
spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[str(y.device)],
|
||||
center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=complex)
|
||||
|
||||
if not complex:
|
||||
spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
|
||||
spec = torch.matmul(mel_basis[str(fmax) + '_' + str(y.device)], spec)
|
||||
spec = spectral_normalize_torch(spec)
|
||||
else:
|
||||
B, C, T, _ = spec.shape
|
||||
spec = spec.transpose(1, 2) # [B, T, n_fft, 2]
|
||||
return spec
|
||||
@@ -0,0 +1,60 @@
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
PITCH_EXTRACTOR = {}
|
||||
|
||||
|
||||
def register_pitch_extractor(name):
|
||||
def register_pitch_extractor_(cls):
|
||||
PITCH_EXTRACTOR[name] = cls
|
||||
return cls
|
||||
|
||||
return register_pitch_extractor_
|
||||
|
||||
|
||||
def get_pitch_extractor(name):
|
||||
return PITCH_EXTRACTOR[name]
|
||||
|
||||
|
||||
def extract_pitch_simple(wav):
|
||||
from ..commons.hparams import hparams
|
||||
return extract_pitch(hparams['pitch_extractor'], wav,
|
||||
hparams['hop_size'], hparams['audio_sample_rate'],
|
||||
f0_min=hparams['f0_min'], f0_max=hparams['f0_max'])
|
||||
|
||||
|
||||
def extract_pitch(extractor_name, wav_data, hop_size, audio_sample_rate, f0_min=75, f0_max=800, **kwargs):
|
||||
return get_pitch_extractor(extractor_name)(wav_data, hop_size, audio_sample_rate, f0_min, f0_max, **kwargs)
|
||||
|
||||
|
||||
@register_pitch_extractor('parselmouth')
|
||||
def parselmouth_pitch(wav_data, hop_size, audio_sample_rate, f0_min, f0_max,
|
||||
voicing_threshold=0.6, *args, **kwargs):
|
||||
import parselmouth
|
||||
time_step = hop_size / audio_sample_rate * 1000
|
||||
n_mel_frames = int(len(wav_data) // hop_size)
|
||||
f0_pm = parselmouth.Sound(wav_data, audio_sample_rate).to_pitch_ac(
|
||||
time_step=time_step / 1000, voicing_threshold=voicing_threshold,
|
||||
pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
|
||||
pad_size = (n_mel_frames - len(f0_pm) + 1) // 2
|
||||
f0 = np.pad(f0_pm, [[pad_size, n_mel_frames - len(f0_pm) - pad_size]], mode='constant')
|
||||
return f0
|
||||
|
||||
@register_pitch_extractor('pyworld')
|
||||
def pyworld_pitch(wav_data, hop_size, audio_sample_rate, f0_min, f0_max,
|
||||
voicing_threshold=0.6, *args, **kwargs):
|
||||
import pyworld as pw
|
||||
# f0, _ = pw.harvest(wav_data.astype(np.double), audio_sample_rate, f0_floor=f0_min, f0_ceil=f0_max,
|
||||
# frame_period=hop_size * 1000 / audio_sample_rate)
|
||||
f0, _ = pw.dio(wav_data.astype(np.double), audio_sample_rate, f0_floor=f0_min, f0_ceil=f0_max, frame_period=hop_size * 1000 / audio_sample_rate)
|
||||
f0[f0 < f0_min] = 0.0
|
||||
f0[f0 > f0_max] = 0.0
|
||||
n_mel_frames = math.ceil(len(wav_data) / hop_size)
|
||||
if n_mel_frames > len(f0):
|
||||
pad_size = (n_mel_frames - len(f0) + 1) // 2
|
||||
f0 = np.pad(f0, [[pad_size, n_mel_frames - len(f0) - pad_size]], mode='constant')
|
||||
elif n_mel_frames < len(f0):
|
||||
left_del = (len(f0) - n_mel_frames + 1) // 2
|
||||
right_del = len(f0) - n_mel_frames - left_del
|
||||
f0 = f0[left_del: (-right_del if right_del > 0 else len(f0))]
|
||||
return f0
|
||||
@@ -0,0 +1,303 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import pretty_midi
|
||||
|
||||
def to_lf0(f0):
|
||||
f0[f0 < 1.0e-5] = 1.0e-6
|
||||
lf0 = f0.log() if isinstance(f0, torch.Tensor) else np.log(f0)
|
||||
lf0[f0 < 1.0e-5] = - 1.0E+10
|
||||
return lf0
|
||||
|
||||
|
||||
def to_f0(lf0):
|
||||
f0 = np.where(lf0 <= 0, 0.0, np.exp(lf0))
|
||||
return f0.flatten()
|
||||
|
||||
|
||||
def f0_to_coarse(f0, f0_bin=256, f0_max=900.0, f0_min=50.0):
|
||||
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
||||
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
|
||||
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (f0_mel_max - f0_mel_min) + 1
|
||||
|
||||
f0_mel[f0_mel <= 1] = 1
|
||||
f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
|
||||
f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(int)
|
||||
assert f0_coarse.max() <= f0_bin-1 and f0_coarse.min() >= 1, (f0_coarse.max(), f0_coarse.min(), f0.min(), f0.max())
|
||||
return f0_coarse
|
||||
|
||||
|
||||
def coarse_to_f0(f0_coarse, f0_bin=256, f0_max=900.0, f0_min=50.0):
|
||||
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
||||
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
||||
uv = f0_coarse == 1
|
||||
f0 = f0_mel_min + (f0_coarse - 1) * (f0_mel_max - f0_mel_min) / (f0_bin - 2)
|
||||
f0 = ((f0 / 1127).exp() - 1) * 700
|
||||
f0[uv] = 0
|
||||
return f0
|
||||
|
||||
|
||||
def norm_f0(f0, uv, pitch_norm='log', f0_mean=400, f0_std=100):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if pitch_norm == 'standard':
|
||||
f0 = (f0 - f0_mean) / f0_std
|
||||
if pitch_norm == 'log':
|
||||
f0 = torch.log2(f0 + 1e-8) if is_torch else np.log2(f0 + 1e-8)
|
||||
if uv is not None:
|
||||
f0[uv > 0] = 0
|
||||
return f0
|
||||
|
||||
|
||||
def norm_interp_f0(f0, pitch_norm='log', f0_mean=None, f0_std=None):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if is_torch:
|
||||
device = f0.device
|
||||
f0 = f0.data.cpu().numpy()
|
||||
uv = f0 == 0
|
||||
f0 = norm_f0(f0, uv, pitch_norm, f0_mean, f0_std)
|
||||
if sum(uv) == len(f0):
|
||||
f0[uv] = 0
|
||||
elif sum(uv) > 0:
|
||||
f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
|
||||
if is_torch:
|
||||
uv = torch.FloatTensor(uv)
|
||||
f0 = torch.FloatTensor(f0)
|
||||
f0 = f0.to(device)
|
||||
uv = uv.to(device)
|
||||
return f0, uv
|
||||
|
||||
|
||||
def denorm_f0(f0, uv, pitch_norm='log', f0_mean=400, f0_std=100, pitch_padding=None, min=50, max=900):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if pitch_norm == 'standard':
|
||||
f0 = f0 * f0_std + f0_mean
|
||||
if pitch_norm == 'log':
|
||||
f0 = 2 ** f0
|
||||
f0 = f0.clamp(min=min, max=max) if is_torch else np.clip(f0, a_min=min, a_max=max)
|
||||
if uv is not None:
|
||||
f0[uv > 0] = 0
|
||||
if pitch_padding is not None:
|
||||
f0[pitch_padding] = 0
|
||||
return f0
|
||||
|
||||
def interp_f0(f0, uv=None):
|
||||
if uv is None:
|
||||
uv = f0 == 0
|
||||
f0 = norm_f0(f0, uv)
|
||||
if uv.any() and not uv.all():
|
||||
f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
|
||||
return denorm_f0(f0, uv=None), uv
|
||||
|
||||
def resample_align_curve(points: np.ndarray, original_timestep: float, target_timestep: float, align_length=-1):
|
||||
t_max = (len(points) - 1) * original_timestep
|
||||
curve_interp = np.interp(
|
||||
np.arange(0, t_max, target_timestep),
|
||||
original_timestep * np.arange(len(points)),
|
||||
points
|
||||
).astype(points.dtype)
|
||||
if align_length > 0:
|
||||
delta_l = align_length - len(curve_interp)
|
||||
if delta_l < 0:
|
||||
curve_interp = curve_interp[:align_length]
|
||||
elif delta_l > 0:
|
||||
curve_interp = np.concatenate((curve_interp, np.full(delta_l, fill_value=curve_interp[-1])), axis=0)
|
||||
return curve_interp
|
||||
|
||||
def midi_to_hz(midi):
|
||||
if type(midi) == np.ndarray:
|
||||
non_mask = midi == 0
|
||||
freq_hz = 440.0 * 2.0 ** ((midi - 69.0) / 12.0)
|
||||
freq_hz[non_mask] = 0
|
||||
else:
|
||||
freq_hz = 440.0 * 2.0 ** ((midi - 69.0) / 12.0)
|
||||
return freq_hz
|
||||
|
||||
def hz_to_midi(hz):
|
||||
if type(hz) == torch.Tensor:
|
||||
non_mask = hz == 0
|
||||
midi = 69.0 + 12.0 * (torch.log2(hz) - torch.log2(torch.Tensor(440.0)))
|
||||
midi[non_mask] = 0
|
||||
elif type(hz) == np.ndarray:
|
||||
non_mask = hz == 0
|
||||
midi = 69.0 + 12.0 * (np.log2(hz) - np.log2(440.0))
|
||||
midi[non_mask] = 0
|
||||
else:
|
||||
midi = 69.0 + 12.0 * (np.log2(hz) - np.log2(440.0))
|
||||
if hz == 0:
|
||||
midi = 0
|
||||
return midi
|
||||
|
||||
def boundary2Interval(bd):
|
||||
# bd has a shape of [T] with T frames
|
||||
is_torch = isinstance(bd, torch.Tensor)
|
||||
if is_torch:
|
||||
device = bd.device
|
||||
bd = bd.data.cpu().numpy()
|
||||
assert len(bd.shape) == 1
|
||||
# force valid begin and end
|
||||
# bd[0] = 0 # took care of in regulate_boundary()
|
||||
# bd[-1] = 0
|
||||
ret = np.zeros(shape=(bd.sum() + 1, 2), dtype=int)
|
||||
ret_idx = 0
|
||||
ret[0, 0] = 0
|
||||
for i, u in enumerate(bd):
|
||||
if i == 0:
|
||||
continue
|
||||
if u == 1:
|
||||
ret[ret_idx, 1] = i
|
||||
ret[ret_idx+1, 0] = i
|
||||
ret_idx += 1
|
||||
ret[-1, 1] = bd.shape[0] - 1
|
||||
if is_torch:
|
||||
ret = torch.LongTensor(ret).to(device)
|
||||
return ret
|
||||
|
||||
def validate_pitch_and_itv(notes, note_itv):
|
||||
# notes [T]
|
||||
# note_itv [T, 2]
|
||||
assert notes.shape[0] == note_itv.shape[0]
|
||||
res_notes = []
|
||||
res_note_itv = []
|
||||
for idx in range(notes.shape[0]):
|
||||
pitch, itv = notes[idx], note_itv[idx]
|
||||
if itv[0] >= itv[1]:
|
||||
raise RuntimeError("The note duration should be positive")
|
||||
if pitch == 0:
|
||||
continue
|
||||
res_notes.append(pitch)
|
||||
res_note_itv.append([itv[0], itv[1]])
|
||||
res_notes = np.array(res_notes)
|
||||
res_note_itv = np.array(res_note_itv)
|
||||
return res_notes, res_note_itv
|
||||
|
||||
def save_midi(notes, note_itv, midi_path):
|
||||
# notes [T]
|
||||
# note_itv [T, 2]
|
||||
notes, note_itv = validate_pitch_and_itv(notes, note_itv)
|
||||
if notes.shape == (0,):
|
||||
return None
|
||||
assert notes.shape[0] == note_itv.shape[0]
|
||||
piano_chord = pretty_midi.PrettyMIDI()
|
||||
piano_program = pretty_midi.instrument_name_to_program('Acoustic Grand Piano')
|
||||
piano = pretty_midi.Instrument(program=piano_program)
|
||||
for idx in range(notes.shape[0]):
|
||||
pitch, itv = notes[idx], note_itv[idx]
|
||||
note = pretty_midi.Note(velocity=120, pitch=pitch, start=itv[0], end=itv[1])
|
||||
piano.notes.append(note)
|
||||
piano_chord.remove_invalid_notes()
|
||||
piano_chord.instruments.append(piano)
|
||||
piano_chord.write(midi_path)
|
||||
return piano_chord
|
||||
|
||||
def midi2NoteInterval(mid):
|
||||
assert type(mid) == pretty_midi.PrettyMIDI
|
||||
if len(mid.instruments) == 0 or len(mid.instruments[0].notes) == 0:
|
||||
return None
|
||||
ret = np.zeros(shape=(len(mid.instruments[0].notes), 2))
|
||||
for i, note in enumerate(mid.instruments[0].notes):
|
||||
ret[i, 0] = note.start
|
||||
ret[i, 1] = note.end
|
||||
return ret
|
||||
|
||||
def midi2NotePitch(mid):
|
||||
assert type(mid) == pretty_midi.PrettyMIDI
|
||||
if len(mid.instruments) == 0 or len(mid.instruments[0].notes) == 0:
|
||||
return None
|
||||
ret = np.zeros(shape=len(mid.instruments[0].notes))
|
||||
for i, note in enumerate(mid.instruments[0].notes):
|
||||
ret[i] = note.pitch
|
||||
return ret
|
||||
|
||||
def midi_onset_eval(mid_gt, mid_pred):
|
||||
import mir_eval
|
||||
interval_true = midi2NoteInterval(mid_gt)
|
||||
if interval_true is None:
|
||||
raise RuntimeError('Midi ground truth is None')
|
||||
interval_pred = midi2NoteInterval(mid_pred)
|
||||
if interval_pred is None:
|
||||
return 0, 0, 0
|
||||
onset_p, onset_r, onset_f = mir_eval.transcription.onset_precision_recall_f1(
|
||||
interval_true, interval_pred, onset_tolerance=0.05, strict=False, beta=1.0)
|
||||
return onset_p, onset_r, onset_f
|
||||
|
||||
def midi_offset_eval(mid_gt, mid_pred):
|
||||
import mir_eval
|
||||
interval_true = midi2NoteInterval(mid_gt)
|
||||
if interval_true is None:
|
||||
raise RuntimeError('Midi ground truth is None')
|
||||
interval_pred = midi2NoteInterval(mid_pred)
|
||||
if interval_pred is None:
|
||||
return 0, 0, 0
|
||||
offset_p, offset_r, offset_f = mir_eval.transcription.offset_precision_recall_f1(
|
||||
interval_true, interval_pred, offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, beta=1.0)
|
||||
return offset_p, offset_r, offset_f
|
||||
|
||||
def midi_pitch_eval(mid_gt, mid_pred, offset_ratio=0.2):
|
||||
import mir_eval
|
||||
interval_true = midi2NoteInterval(mid_gt)
|
||||
pitch_true = midi_to_hz(midi2NotePitch(mid_gt))
|
||||
if interval_true is None or pitch_true is None:
|
||||
raise RuntimeError('Midi ground truth is None')
|
||||
interval_pred = midi2NoteInterval(mid_pred)
|
||||
pitch_pred = midi2NotePitch(mid_pred)
|
||||
if interval_pred is None:
|
||||
return 0, 0, 0, 0
|
||||
if pitch_pred is None:
|
||||
pitch_pred = np.zeros(interval_pred.shape[0])
|
||||
pitch_pred = midi_to_hz(pitch_pred)
|
||||
overlap_p, overlap_r, overlap_f, avg_overlap_ratio = mir_eval.transcription.precision_recall_f1_overlap(
|
||||
interval_true, pitch_true, interval_pred, pitch_pred, onset_tolerance=0.05, pitch_tolerance=50.0,
|
||||
offset_ratio=offset_ratio, offset_min_tolerance=0.05, strict=False, beta=1.0)
|
||||
return overlap_p, overlap_r, overlap_f, avg_overlap_ratio
|
||||
|
||||
def midi_COn_eval(mid_gt, mid_pred):
|
||||
return midi_onset_eval(mid_gt, mid_pred)
|
||||
|
||||
def midi_COnP_eval(mid_gt, mid_pred):
|
||||
return midi_pitch_eval(mid_gt, mid_pred, offset_ratio=None)
|
||||
|
||||
def midi_COnPOff_eval(mid_gt, mid_pred):
|
||||
return midi_pitch_eval(mid_gt, mid_pred)
|
||||
|
||||
def midi_melody_eval(mid_gt, mid_pred, hop_size=256, sample_rate=48000):
|
||||
interval_true = midi2NoteInterval(mid_gt)
|
||||
pitch_true = midi_to_hz(midi2NotePitch(mid_gt))
|
||||
if interval_true is None or pitch_true is None:
|
||||
raise RuntimeError('Midi ground truth is None')
|
||||
interval_pred = midi2NoteInterval(mid_pred)
|
||||
pitch_pred = midi2NotePitch(mid_pred)
|
||||
if interval_pred is None:
|
||||
return 0, 0, 0, 0
|
||||
if pitch_pred is None:
|
||||
pitch_pred = np.zeros(interval_pred.shape[0])
|
||||
pitch_pred = midi_to_hz(pitch_pred)
|
||||
|
||||
vr, vfa, rpa, rca, oa = melody_eval_pitch_and_itv(
|
||||
pitch_true, interval_true, pitch_pred, interval_pred, hop_size, sample_rate)
|
||||
|
||||
return vr, vfa, rpa, rca, oa
|
||||
|
||||
def melody_eval_pitch_and_itv(pitch_true, interval_true, pitch_pred, interval_pred, hop_size=256, sample_rate=48000):
|
||||
import mir_eval
|
||||
t_gt = np.arange(0, interval_true[-1][1], hop_size / sample_rate)
|
||||
freq_gt = np.zeros_like(t_gt)
|
||||
for idx in range(len(pitch_true)):
|
||||
freq_gt[min(len(freq_gt) - 1, round(interval_true[idx][0] * sample_rate / hop_size)): round(
|
||||
interval_true[idx][1] * sample_rate / hop_size)] = pitch_true[idx]
|
||||
|
||||
t_pred = np.arange(0, interval_pred[-1][1], hop_size / sample_rate)
|
||||
freq_pred = np.zeros_like(t_pred)
|
||||
for idx in range(len(pitch_pred)):
|
||||
freq_pred[min(len(freq_pred) - 1, round(interval_pred[idx][0] * sample_rate / hop_size)): round(
|
||||
interval_pred[idx][1] * sample_rate / hop_size)] = pitch_pred[idx]
|
||||
|
||||
ref_voicing, ref_cent, est_voicing, est_cent = mir_eval.melody.to_cent_voicing(t_gt, freq_gt,
|
||||
t_pred, freq_pred)
|
||||
vr, vfa = mir_eval.melody.voicing_measures(ref_voicing,
|
||||
est_voicing) # voicing recall, voicing false alarm
|
||||
rpa = mir_eval.melody.raw_pitch_accuracy(ref_voicing, ref_cent, est_voicing, est_cent)
|
||||
rca = mir_eval.melody.raw_chroma_accuracy(ref_voicing, ref_cent, est_voicing, est_cent)
|
||||
oa = mir_eval.melody.overall_accuracy(ref_voicing, ref_cent, est_voicing, est_cent)
|
||||
|
||||
return vr, vfa, rpa, rca, oa
|
||||
@@ -0,0 +1,78 @@
|
||||
from skimage.transform import resize
|
||||
import struct
|
||||
import webrtcvad
|
||||
from scipy.ndimage.morphology import binary_dilation
|
||||
import librosa
|
||||
import numpy as np
|
||||
import pyloudnorm as pyln
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore", message="Possible clipped samples in output")
|
||||
|
||||
int16_max = (2 ** 15) - 1
|
||||
|
||||
|
||||
def trim_long_silences(path, sr=None, return_raw_wav=False, norm=True, vad_max_silence_length=12):
|
||||
"""
|
||||
Ensures that segments without voice in the waveform remain no longer than a
|
||||
threshold determined by the VAD parameters in params.py.
|
||||
:param wav: the raw waveform as a numpy array of floats
|
||||
:param vad_max_silence_length: Maximum number of consecutive silent frames a segment can have.
|
||||
:return: the same waveform with silences trimmed away (length <= original wav length)
|
||||
"""
|
||||
|
||||
## Voice Activation Detection
|
||||
# Window size of the VAD. Must be either 10, 20 or 30 milliseconds.
|
||||
# This sets the granularity of the VAD. Should not need to be changed.
|
||||
sampling_rate = 16000
|
||||
wav_raw, sr = librosa.core.load(path, sr=sr)
|
||||
|
||||
if norm:
|
||||
meter = pyln.Meter(sr) # create BS.1770 meter
|
||||
loudness = meter.integrated_loudness(wav_raw)
|
||||
wav_raw = pyln.normalize.loudness(wav_raw, loudness, -20.0)
|
||||
if np.abs(wav_raw).max() > 1.0:
|
||||
wav_raw = wav_raw / np.abs(wav_raw).max()
|
||||
|
||||
wav = librosa.resample(wav_raw, sr, sampling_rate, res_type='kaiser_best')
|
||||
|
||||
vad_window_length = 30 # In milliseconds
|
||||
# Number of frames to average together when performing the moving average smoothing.
|
||||
# The larger this value, the larger the VAD variations must be to not get smoothed out.
|
||||
vad_moving_average_width = 8
|
||||
|
||||
# Compute the voice detection window size
|
||||
samples_per_window = (vad_window_length * sampling_rate) // 1000
|
||||
|
||||
# Trim the end of the audio to have a multiple of the window size
|
||||
wav = wav[:len(wav) - (len(wav) % samples_per_window)]
|
||||
|
||||
# Convert the float waveform to 16-bit mono PCM
|
||||
pcm_wave = struct.pack("%dh" % len(wav), *(np.round(wav * int16_max)).astype(np.int16))
|
||||
|
||||
# Perform voice activation detection
|
||||
voice_flags = []
|
||||
vad = webrtcvad.Vad(mode=3)
|
||||
for window_start in range(0, len(wav), samples_per_window):
|
||||
window_end = window_start + samples_per_window
|
||||
voice_flags.append(vad.is_speech(pcm_wave[window_start * 2:window_end * 2],
|
||||
sample_rate=sampling_rate))
|
||||
voice_flags = np.array(voice_flags)
|
||||
|
||||
# Smooth the voice detection with a moving average
|
||||
def moving_average(array, width):
|
||||
array_padded = np.concatenate((np.zeros((width - 1) // 2), array, np.zeros(width // 2)))
|
||||
ret = np.cumsum(array_padded, dtype=float)
|
||||
ret[width:] = ret[width:] - ret[:-width]
|
||||
return ret[width - 1:] / width
|
||||
|
||||
audio_mask = moving_average(voice_flags, vad_moving_average_width)
|
||||
audio_mask = np.round(audio_mask).astype(np.bool)
|
||||
|
||||
# Dilate the voiced regions
|
||||
audio_mask = binary_dilation(audio_mask, np.ones(vad_max_silence_length + 1))
|
||||
audio_mask = np.repeat(audio_mask, samples_per_window)
|
||||
audio_mask = resize(audio_mask, (len(wav_raw),)) > 0
|
||||
if return_raw_wav:
|
||||
return wav_raw, audio_mask, sr
|
||||
return wav_raw[audio_mask], audio_mask, sr
|
||||
Reference in New Issue
Block a user