mei/SoulX-Singer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add svc inference code

Browse Source
This commit is contained in:
jlqian98
2026-03-03 09:35:06 +08:00
parent f8fce96f9c
commit edf4e11bf9
7 changed files with 986 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
soulxsinger/models/modules/whisper_encoder.py
+74
View File
@@ -0,0 +1,74 @@
"""Frozen Whisper encoder wrapper (wav -> encoder embeddings)."""
from __future__ import annotations
from typing import Optional
import torch
import torch.nn as nn
import torchaudio
from transformers import WhisperFeatureExtractor, WhisperModel
WHISPER_MEL_FRAMES = 3000 # 3000 frames at 16000 Hz
class WhisperEncoder():
def __init__(
self,
device: Optional[str] = None,
) -> None:
self.fe = WhisperFeatureExtractor.from_pretrained("openai/whisper-base")
self.model = WhisperModel.from_pretrained("openai/whisper-base")
self.model = self.model.to(device or ("cuda" if torch.cuda.is_available() else "cpu"))
def encode(
self,
wav: torch.Tensor,
sr: int,
) -> torch.Tensor:
wav = torchaudio.functional.resample(wav, orig_freq=sr, new_freq=self.fe.sampling_rate) if sr != self.fe.sampling_rate else wav
wav_np = wav.cpu().detach().numpy().astype("float32", copy=False)
inputs = self.fe(
wav_np,
sampling_rate=self.fe.sampling_rate,
return_tensors="pt",
padding=False,
truncation=False,
return_attention_mask=True,
)
input_features = inputs.input_features
num_frames = input_features.shape[-1]
if num_frames < WHISPER_MEL_FRAMES:
pad = WHISPER_MEL_FRAMES - num_frames
input_features = torch.nn.functional.pad(input_features, (0, pad))
else:
input_features = input_features[..., :WHISPER_MEL_FRAMES]
input_features = input_features.to(wav.device)
if self.model.device != wav.device:
self.model = self.model.to(wav.device)
attention_mask = inputs.attention_mask.to(wav.device) if inputs.attention_mask is not None else None
encoder_out = self.model.encoder(input_features).last_hidden_state
if attention_mask is not None:
valid_mel_frames = attention_mask.sum(dim=1)
valid_enc_frames = (valid_mel_frames + 1) // 2
max_valid_enc_frames = min(int(valid_enc_frames.max().item()), encoder_out.shape[1])
encoder_out = encoder_out[:, :max_valid_enc_frames, :]
valid_len = min(int(valid_enc_frames[0].item()), max_valid_enc_frames)
if valid_len < max_valid_enc_frames:
encoder_out[0, valid_len:, :] = 0
return encoder_out
if __name__ == "__main__":
torch.manual_seed(0)
audio = torch.randn(1, 24000 * 25).float().to("cuda")
encoder = WhisperEncoder()
whisper_encoder_out = encoder.encode(audio, sr=24000)
print(whisper_encoder_out.shape)
soulxsinger/models/soulxsinger_svc.py
+303
View File
@@ -0,0 +1,303 @@
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from tqdm import tqdm
from typing import Optional, Dict, Any, List, Tuple
from soulxsinger.models.modules.vocoder import Vocoder
from soulxsinger.models.modules.decoder import CFMDecoder
from soulxsinger.models.modules.mel_transform import MelSpectrogramEncoder
from soulxsinger.models.modules.whisper_encoder import WhisperEncoder
class SoulXSingerSVC(nn.Module):
"""
SoulXSinger SVC model.
"""
def __init__(self, config: Dict):
super(SoulXSingerSVC, self).__init__()
self.audio_cfg = config.audio
enc_cfg = config.model.encoder
cfm_cfg = config.model.flow_matching
self.whisper_encoder = WhisperEncoder()
self.f0_encoder = nn.Embedding(enc_cfg["f0_bin"], enc_cfg["f0_dim"])
self.cfm_decoder = CFMDecoder(cfm_cfg)
self.mel = MelSpectrogramEncoder(self.audio_cfg)
self.vocoder = Vocoder()
@staticmethod
def f0_to_coarse(f0, f0_bin=361, f0_min=32.7031956625, f0_shift=0):
"""
Convert continuous F0 values to discrete F0 bins (SIL and C1 - B6, 361 bins).
args:
f0: continuous F0 values
f0_bin: number of F0 bins
f0_min: minimum F0 value
f0_shift: shift value for F0 bins
returns:
f0_coarse: discrete F0 bins
"""
is_torch = isinstance(f0, torch.Tensor)
uv_mask = f0 <= 0
if is_torch:
f0_safe = torch.maximum(f0, torch.tensor(f0_min))
f0_cents = 1200 * torch.log2(f0_safe / f0_min)
else:
f0_safe = np.maximum(f0, f0_min)
f0_cents = 1200 * np.log2(f0_safe / f0_min)
f0_coarse = (f0_cents / 20) + 1
if is_torch:
f0_coarse = torch.round(f0_coarse).long()
f0_coarse = torch.clamp(f0_coarse, min=1, max=f0_bin - 1)
else:
f0_coarse = np.rint(f0_coarse).astype(int)
f0_coarse = np.clip(f0_coarse, 1, f0_bin - 1)
f0_coarse[uv_mask] = 0
if f0_shift != 0:
if is_torch:
voiced = f0_coarse > 0
if voiced.any():
shifted = f0_coarse[voiced] + f0_shift
f0_coarse[voiced] = torch.clamp(shifted, 1, f0_bin - 1)
else:
voiced = f0_coarse > 0
if np.any(voiced):
shifted = f0_coarse[voiced] + f0_shift
f0_coarse[voiced] = np.clip(shifted, 1, f0_bin - 1)
return f0_coarse
@staticmethod
def build_vocal_segments(
f0,
f0_rate: int = 50,
uv_frames_th: int = 5,
min_duration_sec: float = 5.0,
max_duration_sec: float = 30.0,
num_overlaps: int = 1,
ignore_silent_segments: bool = True,
) -> Tuple[List[Tuple[float, float]], List[Tuple[float, float]]]:
"""Build vocal segments based on F0 contour. First split by long silent runs, then merge into segments based on min and max duration constraints.
args:
f0: F0 contour of the audio, 1D array or tensor with shape (T,)
f0_rate: F0 sampling rate in Hz (e.g., 50 for 20ms hop size)
uv_frames_th: number of consecutive zero F0 frames to consider as a split point
min_duration_sec: minimum duration of each segment in seconds
max_duration_sec: maximum duration of each segment in seconds
num_overlaps: number of overlapping segments to create for each non-overlapping segment (for smooth inference)
ignore_silent_segments: whether to ignore segments that are mostly silent (e.g., > 95% zero F0)
returns:
overlap_segments: list of (overlap_start_sec, overlap_end_sec) for each segment, which may overlap with adjacent segments for smooth inference
segments: list of (seg_start_sec, seg_end_sec) for each segment, which are non-overlapping and used for final merging
"""
if isinstance(f0, torch.Tensor):
f0_np = f0.detach().float().cpu().numpy()
else:
f0_np = np.asarray(f0, dtype=np.float32)
f0_np = np.squeeze(f0_np)
total_frames = int(f0_np.shape[0])
if total_frames == 0:
return [], []
min_frames = max(1, int(round(min_duration_sec * f0_rate)))
max_frames = max(1, int(round(max_duration_sec * f0_rate)))
split_points = [0] # silence split points in frame indices, starting with 0 and ending with total_frames
def append_split_point(point: int):
# Ensure split points are within valid range and respect max_frames constraint
point = int(max(0, min(point, total_frames)))
while point - split_points[-1] > max_frames:
split_points.append(split_points[-1] + max_frames)
if point > split_points[-1]:
split_points.append(point)
idx = 0
while idx < total_frames:
if f0_np[idx] == 0:
run_start = idx
while idx < total_frames and f0_np[idx] == 0:
idx += 1
run_end = idx
if (run_end - run_start) >= uv_frames_th:
split_point = max(run_end - 5, (run_start + run_end) // 2)
append_split_point(split_point)
else:
idx += 1
append_split_point(total_frames)
# print(f"Initial split points (in seconds): {[round(p / f0_rate, 2) for p in split_points]}")
segments: List[Tuple[int, int]] = []
overlap_segments: List[Tuple[int, int]] = []
def append_segment(start_idx: int, end_idx: int, num_overlaps: int = num_overlaps):
segments.append((split_points[start_idx] / f0_rate, split_points[end_idx] / f0_rate))
overlap_start_idx = start_idx
if start_idx > 0 and (split_points[end_idx] - split_points[start_idx - num_overlaps]) <= max_frames:
overlap_start_idx = start_idx - num_overlaps
overlap_segments.append((split_points[overlap_start_idx] / f0_rate, split_points[end_idx] / f0_rate))
segment_start, segment_end = 0, 1
while segment_start < len(split_points) - 1:
while segment_end < len(split_points) and (split_points[segment_end] - split_points[segment_start]) < min_frames:
segment_end += 1
if segment_end >= len(split_points):
append_segment(segment_start, len(split_points) - 1, num_overlaps=num_overlaps)
break
append_segment(segment_start, segment_end, num_overlaps=num_overlaps)
segment_start = segment_end
segment_end = segment_start + 1
# print(f"Final segments (overlap_start, overlap_end, seg_start_time, seg_end_time) in seconds: {overlap_segments}")
if ignore_silent_segments:
filtered_idx = []
for i, seg in enumerate(overlap_segments):
start_frame = int(seg[0] * f0_rate)
end_frame = int(seg[1] * f0_rate)
total_frames = end_frame - start_frame
voice_frames = np.sum(f0_np[start_frame:end_frame] > 0)
if voice_frames / total_frames > 0.05 and voice_frames >= 10: # at least 10 voiced frames and >5% voiced frames
filtered_idx.append(i)
overlap_segments = [overlap_segments[i] for i in filtered_idx]
segments = [segments[i] for i in filtered_idx]
# print(f"Filtered segments with mostly silence removed: {overlap_segments}")
return overlap_segments, segments
def infer(
self,
pt_wav: str|torch.Tensor,
gt_wav: str|torch.Tensor,
pt_f0: str|torch.Tensor,
gt_f0: str|torch.Tensor,
auto_shift=False,
pitch_shift=0,
n_steps=32,
cfg=3,
):
"""
SVC inference pipeline. First build vocal segments based on F0 contour, then run inference for each segment and merge results.
args:
pt_wav: prompt waveform path or tensor
gt_wav: target waveform path or tensor
pt_f0: prompt F0 path or tensor
gt_f0: target F0 path or tensor
auto_shift: whether to automatically calculate pitch shift based on median F0 of prompt and target
pitch_shift: manual pitch shift in semitones (overrides auto_shift if > 0)
n_steps: number of diffusion steps for inference
cfg: classifier-free guidance scale for inference
"""
# calculate auto pitch shift
if auto_shift and pitch_shift == 0:
if gt_f0 is not None and pt_f0 is not None:
gt_f0_median = torch.median(gt_f0[gt_f0 > 0])
pt_f0_median = torch.median(pt_f0[pt_f0 > 0])
pitch_shift = torch.round(torch.log2(pt_f0_median / gt_f0_median) * 1200 / 100).int().item()
else:
print("Warning: pitch_shift is True but note_pitch or f0 is None. Set f0_shift to 0.")
pitch_shift = 0
else:
pitch_shift = pitch_shift
generated_audio = []
f0_rate = self.audio_cfg.sample_rate // self.audio_cfg.hop_size
overlap_segments, segments = self.build_vocal_segments(
gt_f0,
f0_rate=f0_rate,
uv_frames_th=10,
min_duration_sec=15.0,
max_duration_sec=30.0,
)
if len(segments) == 0:
segments = [(0.0, gt_wav.shape[-1] / self.audio_cfg.sample_rate)]
overlap_segments = [(0.0, gt_wav.shape[-1] / self.audio_cfg.sample_rate)]
generated_audio = torch.zeros_like(gt_wav)
for idx in tqdm(range(len(segments)), total=len(segments), desc="Inferring segments (SVC)", dynamic_ncols=True):
overlap_start_sec, overlap_end_sec = overlap_segments[idx]
seg_start_sec, seg_end_sec = segments[idx]
wav_start = int(round(overlap_start_sec * self.audio_cfg.sample_rate))
wav_end = int(round(overlap_end_sec * self.audio_cfg.sample_rate))
f0_start = int(round(overlap_start_sec * f0_rate))
f0_end = int(round(overlap_end_sec * f0_rate))
wav_start = max(0, min(wav_start, gt_wav.shape[-1]))
wav_end = max(wav_start, min(wav_end, gt_wav.shape[-1]))
f0_start = max(0, min(f0_start, gt_f0.shape[-1]))
f0_end = max(f0_start, min(f0_end, gt_f0.shape[-1]))
segment_gt_wav = gt_wav[:, wav_start:wav_end]
segment_gt_f0 = gt_f0[:, f0_start:f0_end]
segment_generated_audio = self.infer_segment(
pt_wav=pt_wav,
gt_wav=segment_gt_wav,
pt_f0=pt_f0,
gt_f0=segment_gt_f0,
pitch_shift=pitch_shift,
n_steps=n_steps,
cfg=cfg,
)
segment_start = int(round(seg_start_sec * self.audio_cfg.sample_rate))
segment_end = int(round(seg_end_sec * self.audio_cfg.sample_rate))
segment_generated_audio = segment_generated_audio[segment_start - wav_start: segment_end - wav_start]
generated_audio[:, segment_start:segment_end] = segment_generated_audio
return generated_audio, pitch_shift
def infer_segment(self, pt_wav, gt_wav, pt_f0, gt_f0, pitch_shift=0, n_steps=32, cfg=3):
pt_mel = self.mel(pt_wav)
len_prompt_mel = pt_f0.shape[1]
f0_course_pt = self.f0_to_coarse(pt_f0)
f0_course_gt = self.f0_to_coarse(gt_f0, f0_shift=pitch_shift * 5)
f0_course = torch.cat([f0_course_pt, f0_course_gt], 1)
pt_content_feat = self.whisper_encoder.encode(pt_wav, sr=self.audio_cfg.sample_rate)
gt_content_feat = self.whisper_encoder.encode(gt_wav, sr=self.audio_cfg.sample_rate)
content_feat = torch.cat([pt_content_feat, gt_content_feat], 1)
f0_feat = self.f0_encoder(f0_course)
min_len = min(content_feat.shape[1], f0_feat.shape[1])
content_feat = content_feat[:, :min_len, :]
f0_feat = f0_feat[:, :min_len, :]
features = content_feat + f0_feat
gt_decoder_inp = features[:, len_prompt_mel:, :]
pt_decoder_inp = features[:, :len_prompt_mel, :]
generated_mel = self.cfm_decoder.reverse_diffusion(
pt_mel,
pt_decoder_inp,
gt_decoder_inp,
n_timesteps=n_steps,
cfg=cfg
)
generated_audio = self.vocoder(generated_mel.transpose(1, 2)[0:1, ...])
generated_audio = generated_audio.squeeze()
# cut or pad to match gt_wav length
if generated_audio.shape[-1] > gt_wav.shape[-1]:
generated_audio = generated_audio[:gt_wav.shape[-1]]
elif generated_audio.shape[-1] < gt_wav.shape[-1]:
generated_audio = F.pad(generated_audio, (0, gt_wav.shape[-1] - generated_audio.shape[-1]))
return generated_audio