Initial commit
This commit is contained in:
王新升
@@ -0,0 +1,25 @@
|
||||
import torch
|
||||
import torchaudio
|
||||
|
||||
|
||||
def load_wav(wav_path: str, sample_rate: int):
|
||||
"""Load wav file and resample to target sample rate.
|
||||
|
||||
Args:
|
||||
wav_path (str): Path to wav file.
|
||||
sample_rate (int): Target sample rate.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Waveform tensor with shape (1, T).
|
||||
"""
|
||||
waveform, sr = torchaudio.load(wav_path)
|
||||
|
||||
if sr != sample_rate:
|
||||
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
|
||||
|
||||
if len(waveform.shape) > 1 and waveform.shape[0] > 1:
|
||||
waveform = torch.mean(waveform, dim=0, keepdim=True)
|
||||
|
||||
return waveform
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
import torchaudio
|
||||
from typing import List
|
||||
|
||||
from soulxsinger.utils.audio_utils import load_wav
|
||||
|
||||
|
||||
class DataProcessor:
|
||||
"""Data processor for SoulX-Singer
|
||||
"""
|
||||
def __init__(
|
||||
self,
|
||||
hop_size: int,
|
||||
sample_rate: int,
|
||||
phoneset_path: str = 'soulxsinger/utils/phoneme/phone_set.json',
|
||||
device: str = 'cuda',
|
||||
prompt_append_duration: float = 0.5):
|
||||
"""Initialize data processor.
|
||||
|
||||
Args:
|
||||
hop_size (int): Hop size in samples.
|
||||
sample_rate (int): Sample rate in Hz.
|
||||
phoneset_path (str): Path to phoneme set JSON file.
|
||||
device (str): Device to use for tensor operations.
|
||||
prompt_append_duration (float): Duration to append to prompt in seconds.
|
||||
"""
|
||||
self.hop_size = hop_size
|
||||
self.sample_rate = sample_rate
|
||||
self.device = device
|
||||
self.prompt_append_duration = prompt_append_duration
|
||||
self.prompt_append_length = int(prompt_append_duration * sample_rate / hop_size)
|
||||
self.load_phoneme_id_map(phoneset_path)
|
||||
|
||||
def load_phoneme_id_map(self, phoneset_path: str):
|
||||
with open(phoneset_path, "r", encoding='utf-8') as f:
|
||||
phoneset = json.load(f)
|
||||
self.phone2idx = {ph: idx for idx, ph in enumerate(phoneset)}
|
||||
|
||||
def merge_phoneme(self, meta):
|
||||
merged_items = []
|
||||
|
||||
duration = [float(x) for x in meta["duration"].split()]
|
||||
phoneme = [str(x).replace("<AP>", "<SP>") for i, x in enumerate(meta["phoneme"].split())]
|
||||
note_pitch = [int(x) for x in meta["note_pitch"].split()]
|
||||
note_type = [int(x) if phoneme[i] != "<SP>" else 1 for i, x in enumerate(meta["note_type"].split())]
|
||||
|
||||
for i in range(len(phoneme)):
|
||||
if i > 0 and phoneme[i] == phoneme[i - 1] == "<SP>" and note_type[i] == note_type[i - 1] and note_pitch[i] == note_pitch[i - 1]:
|
||||
merged_items[-1][1] += duration[i]
|
||||
else:
|
||||
merged_items.append([phoneme[i], duration[i], note_pitch[i], note_type[i]])
|
||||
|
||||
single_frame_duration = self.hop_size / self.sample_rate
|
||||
meta['phoneme'] = [x[0] for x in merged_items]
|
||||
meta['duration'] = [x[1] for x in merged_items]
|
||||
meta['note_pitch'] = [x[2] for x in merged_items]
|
||||
meta['note_type'] = [x[3] for x in merged_items]
|
||||
|
||||
return meta
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
note_duration: List[float],
|
||||
phonemes: List[str],
|
||||
note_pitch: List[int],
|
||||
note_type: List[int],
|
||||
):
|
||||
"""
|
||||
Insert <BOW> and <EOW> for each note.
|
||||
Get aligned indices for each frame.
|
||||
|
||||
Args:
|
||||
note_duration: Duration of each note in seconds
|
||||
phonemes: Phoneme sequence for each note
|
||||
note_pitch: Pitch value for each note
|
||||
note_type: Type value for each note
|
||||
|
||||
"""
|
||||
sample_rate = self.sample_rate
|
||||
hop_size = self.hop_size
|
||||
duration = sum(note_duration) * sample_rate / hop_size
|
||||
mel2note = torch.zeros(int(duration), dtype=torch.long)
|
||||
|
||||
ph_locations = [] # idx at mel scale and length
|
||||
new_phonemes = []
|
||||
dur_sum = 0
|
||||
|
||||
note2origin = []
|
||||
|
||||
for ph_idx in range(len(phonemes)):
|
||||
dur = int(np.round(dur_sum * sample_rate / hop_size))
|
||||
dur = min(dur, len(mel2note) - 1)
|
||||
new_phonemes.append("<BOW>")
|
||||
note2origin.append(ph_idx)
|
||||
if phonemes[ph_idx][:3] == "en_":
|
||||
en_phs = ['en_' + x for x in phonemes[ph_idx][3:].split('-')] + ['<SEP>'] # <sep> between en words in one note
|
||||
ph_locations.append([dur, max(1, len(en_phs))])
|
||||
new_phonemes.extend(en_phs)
|
||||
note2origin.extend([ph_idx] * len(en_phs))
|
||||
else:
|
||||
ph_locations.append([dur, 1])
|
||||
new_phonemes.append(phonemes[ph_idx])
|
||||
note2origin.append(ph_idx)
|
||||
new_phonemes.append("<EOW>")
|
||||
note2origin.append(ph_idx)
|
||||
dur_sum += note_duration[ph_idx]
|
||||
|
||||
ph_idx = 1
|
||||
for idx, (i, j) in enumerate(ph_locations):
|
||||
next_phoneme_start = ph_locations[idx + 1][0] if idx < len(ph_locations) - 1 else len(mel2note)
|
||||
if i >= len(mel2note) or i + j > len(mel2note):
|
||||
break
|
||||
if i < len(mel2note) and mel2note[i] > 0:
|
||||
# print(f"warning: overlap of {idx}: {mel2note[i]}")
|
||||
while i < len(mel2note) and mel2note[i] > 0:
|
||||
i += 1
|
||||
mel2note[i] = ph_idx
|
||||
k = i + 1
|
||||
while k + j < next_phoneme_start:
|
||||
mel2note[k : k + j] = torch.arange(ph_idx, ph_idx + j) + 1
|
||||
k += j
|
||||
mel2note[next_phoneme_start - 1] = ph_idx + j + 1
|
||||
ph_idx += j + 2 # <BOW> + ph repeats + <EOW>
|
||||
|
||||
new_phonemes = ["<PAD>"] + new_phonemes
|
||||
new_note_pitch = [0] + [note_pitch[k] for k in note2origin]
|
||||
new_note_type = [1] + [note_type[k] for k in note2origin]
|
||||
|
||||
return {
|
||||
"phoneme": torch.tensor([self.phone2idx[x] for x in new_phonemes], device=self.device).unsqueeze(0),
|
||||
"note_pitch": torch.tensor(new_note_pitch, device=self.device).unsqueeze(0),
|
||||
"note_type": torch.tensor(new_note_type, device=self.device).unsqueeze(0),
|
||||
"mel2note": mel2note.clone().detach().to(self.device).unsqueeze(0),
|
||||
}
|
||||
|
||||
def process(
|
||||
self,
|
||||
meta: dict,
|
||||
wav_path: str = None
|
||||
):
|
||||
|
||||
meta = self.merge_phoneme(meta)
|
||||
|
||||
item = self.preprocess(
|
||||
meta["duration"],
|
||||
meta["phoneme"],
|
||||
meta["note_pitch"],
|
||||
meta["note_type"],
|
||||
)
|
||||
|
||||
f0 = torch.tensor([float(x) for x in meta["f0"].split()])
|
||||
min_frame = min(item["mel2note"].shape[1], f0.shape[0])
|
||||
item['f0'] = f0[:min_frame].unsqueeze(0).float().to(self.device)
|
||||
item["mel2note"] = item["mel2note"][:, :min_frame]
|
||||
|
||||
if wav_path is not None:
|
||||
waveform = load_wav(wav_path, self.sample_rate)
|
||||
item["waveform"] = waveform.to(self.device)[:, :min_frame * self.hop_size]
|
||||
|
||||
return item
|
||||
|
||||
|
||||
# test
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
with open("example/metadata/zh_prompt.json", "r", encoding="utf-8") as f:
|
||||
meta = json.load(f)
|
||||
if isinstance(meta, list):
|
||||
meta = meta[0]
|
||||
processor = DataProcessor(hop_size=480, sample_rate=24000)
|
||||
item = processor.process(meta, "example/audio/zh_prompt.wav")
|
||||
print(item.keys())
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
"""
|
||||
Description:
|
||||
This script contains a collection of functions designed to handle various
|
||||
file reading and writing operations. It provides utilities to read from files,
|
||||
write data to files, and perform file manipulation tasks.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
|
||||
from tqdm import tqdm
|
||||
from typing import List, Dict
|
||||
from pathlib import Path
|
||||
from omegaconf import OmegaConf, DictConfig
|
||||
|
||||
|
||||
def write_jsonl(metadata: List[dict], file_path: Path):
|
||||
"""Writes a list of dictionaries to a JSONL file.
|
||||
|
||||
Args:
|
||||
metadata : List[dict]
|
||||
A list of dictionaries, each representing a piece of meta.
|
||||
file_path : Path
|
||||
The file path to save the JSONL file
|
||||
|
||||
This function writes each dictionary in the list to a new line in the specified file.
|
||||
"""
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
for meta in tqdm(metadata, desc="writing jsonl"):
|
||||
# Convert dictionary to JSON string and write it to the file with a newline
|
||||
json_str = json.dumps(meta, ensure_ascii=False) + "\n"
|
||||
f.write(json_str)
|
||||
print(f"jsonl saved to {file_path}")
|
||||
|
||||
|
||||
def read_jsonl(file_path: Path) -> List[dict]:
|
||||
"""
|
||||
Reads a JSONL file and returns a list of dictionaries.
|
||||
|
||||
Args:
|
||||
file_path : Path
|
||||
The path to the JSONL file to be read.
|
||||
|
||||
Returns:
|
||||
List[dict]
|
||||
A list of dictionaries parsed from each line of the JSONL file.
|
||||
"""
|
||||
metadata = []
|
||||
# Open the file for reading
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
# Split the file into lines
|
||||
lines = f.read().splitlines()
|
||||
# Process each line
|
||||
for line in lines:
|
||||
# Convert JSON string back to dictionary and append to list
|
||||
meta = json.loads(line)
|
||||
metadata.append(meta)
|
||||
# Return the list of metadata
|
||||
return metadata
|
||||
|
||||
|
||||
def load_config(config_path: Path) -> DictConfig:
|
||||
"""Loads a configuration file and optionally merges it with a base configuration.
|
||||
|
||||
Args:
|
||||
config_path (Path): Path to the configuration file.
|
||||
"""
|
||||
# Load the initial configuration from the given path
|
||||
config = OmegaConf.load(config_path)
|
||||
|
||||
# Check if there is a base configuration specified and merge if necessary
|
||||
if config.get("base_config", None) is not None:
|
||||
base_config = OmegaConf.load(config["base_config"])
|
||||
config = OmegaConf.merge(base_config, config)
|
||||
|
||||
return config
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,142 @@
|
||||
# https://github.com/gwx314/TechSinger/blob/main/utils/audio/pitch/utils.py
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
def to_lf0(f0):
|
||||
f0[f0 < 1.0e-5] = 1.0e-6
|
||||
lf0 = f0.log() if isinstance(f0, torch.Tensor) else np.log(f0)
|
||||
lf0[f0 < 1.0e-5] = - 1.0E+10
|
||||
return lf0
|
||||
|
||||
|
||||
def to_f0(lf0):
|
||||
f0 = np.where(lf0 <= 0, 0.0, np.exp(lf0))
|
||||
return f0.flatten()
|
||||
|
||||
|
||||
def f0_to_coarse_mel(f0, f0_bin=256, f0_max=900.0, f0_min=50.0, f0_shift=0):
|
||||
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
||||
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
|
||||
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (f0_mel_max - f0_mel_min) + 1
|
||||
|
||||
f0_mel[f0_mel <= 1] = 1
|
||||
f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
|
||||
f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(int)
|
||||
|
||||
if f0_shift != 0:
|
||||
if f0_shift > 0:
|
||||
f0_shift = min(f0_shift, f0_bin - 1 - f0_coarse[f0_coarse > 1].max().item())
|
||||
else:
|
||||
f0_shift = max(f0_shift, 1 - f0_coarse[f0_coarse > 1].min().item())
|
||||
|
||||
f0_coarse[f0_coarse > 1] = f0_coarse[f0_coarse > 1] + f0_shift
|
||||
|
||||
assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (f0_coarse.max(), f0_coarse.min(), f0.min(), f0.max())
|
||||
return f0_coarse
|
||||
|
||||
|
||||
def coarse_to_f0_mel(f0_coarse, f0_bin=256, f0_max=900.0, f0_min=50.0):
|
||||
f0_mel_min = 1127 * np.log(1 + f0_min / 700)
|
||||
f0_mel_max = 1127 * np.log(1 + f0_max / 700)
|
||||
uv = f0_coarse == 1
|
||||
f0 = f0_mel_min + (f0_coarse - 1) * (f0_mel_max - f0_mel_min) / (f0_bin - 2)
|
||||
f0 = ((f0 / 1127).exp() - 1) * 700
|
||||
f0[uv] = 0
|
||||
return f0
|
||||
|
||||
CONST_C1_FREQ = 32.7031956625 # C1 frequency in Hz
|
||||
CONST_B6_FREQ = 1975.53320502 # B6 frequency in Hz
|
||||
|
||||
def f0_to_coarse_midi(f0, f0_bin=361, f0_max=CONST_B6_FREQ, f0_min=CONST_C1_FREQ, f0_shift=0):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
uv_mask = f0 <= 0
|
||||
|
||||
if is_torch:
|
||||
f0_safe = torch.maximum(f0, torch.tensor(f0_min))
|
||||
f0_cents = 1200 * torch.log2(f0_safe / f0_min)
|
||||
else:
|
||||
f0_safe = np.maximum(f0, f0_min)
|
||||
f0_cents = 1200 * np.log2(f0_safe / f0_min)
|
||||
|
||||
f0_coarse = (f0_cents / 20) + 1
|
||||
|
||||
if is_torch:
|
||||
f0_coarse = torch.round(f0_coarse).long()
|
||||
f0_coarse = torch.clamp(f0_coarse, min=1, max=f0_bin - 1)
|
||||
else:
|
||||
f0_coarse = np.rint(f0_coarse).astype(int)
|
||||
f0_coarse = np.clip(f0_coarse, 1, f0_bin - 1)
|
||||
|
||||
f0_coarse[uv_mask] = 0
|
||||
|
||||
if f0_shift != 0:
|
||||
if is_torch:
|
||||
voiced = f0_coarse > 0
|
||||
if voiced.any():
|
||||
shifted = f0_coarse[voiced] + f0_shift
|
||||
f0_coarse[voiced] = torch.clamp(shifted, 1, f0_bin - 1)
|
||||
else:
|
||||
voiced = f0_coarse > 0
|
||||
if np.any(voiced):
|
||||
shifted = f0_coarse[voiced] + f0_shift
|
||||
f0_coarse[voiced] = np.clip(shifted, 1, f0_bin - 1)
|
||||
|
||||
return f0_coarse
|
||||
|
||||
|
||||
def coarse_to_f0_midi(f0_coarse, f0_bin=361, f0_max=CONST_B6_FREQ, f0_min=CONST_C1_FREQ):
|
||||
|
||||
uv_mask = f0_coarse == 0
|
||||
cents = (f0_coarse - 1) * 20
|
||||
f0 = f0_min * (2 ** (cents / 1200))
|
||||
f0[uv_mask] = 0
|
||||
|
||||
return f0
|
||||
|
||||
|
||||
def norm_f0(f0, uv, pitch_norm='log', f0_mean=400, f0_std=100):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if pitch_norm == 'standard':
|
||||
f0 = (f0 - f0_mean) / f0_std
|
||||
if pitch_norm == 'log':
|
||||
f0 = torch.log2(f0 + 1e-8) if is_torch else np.log2(f0 + 1e-8)
|
||||
if uv is not None:
|
||||
f0[uv > 0] = 0
|
||||
return f0
|
||||
|
||||
|
||||
def norm_interp_f0(f0, pitch_norm='log', f0_mean=None, f0_std=None):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if is_torch:
|
||||
device = f0.device
|
||||
f0 = f0.data.cpu().numpy()
|
||||
uv = f0 == 0
|
||||
f0 = norm_f0(f0, uv, pitch_norm, f0_mean, f0_std)
|
||||
if sum(uv) == len(f0):
|
||||
f0[uv] = 0
|
||||
elif sum(uv) > 0:
|
||||
f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
|
||||
if is_torch:
|
||||
uv = torch.FloatTensor(uv)
|
||||
f0 = torch.FloatTensor(f0)
|
||||
f0 = f0.to(device)
|
||||
uv = uv.to(device)
|
||||
return f0, uv
|
||||
|
||||
|
||||
def denorm_f0(f0, uv, pitch_norm='log', f0_mean=400, f0_std=100, pitch_padding=None, min=50, max=900):
|
||||
is_torch = isinstance(f0, torch.Tensor)
|
||||
if pitch_norm == 'standard':
|
||||
f0 = f0 * f0_std + f0_mean
|
||||
if pitch_norm == 'log':
|
||||
f0 = 2 ** f0
|
||||
f0 = f0.clamp(min=min, max=max) if is_torch else np.clip(f0, a_min=min, a_max=max)
|
||||
if uv is not None:
|
||||
f0[uv > 0] = 0
|
||||
if pitch_padding is not None:
|
||||
f0[pitch_padding] = 0
|
||||
return f0
|
||||
Reference in New Issue
Block a user