mei/SoulX-Singer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge feat/svc into main: Integration of SoulX-Singer-SVC

Browse Source
This commit is contained in:
jlqian98
2026-03-16 15:04:19 +08:00
committed by GitHub
parent 4030d846fa 0edafb353b
commit dfb95683e1
19 changed files with 1204 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
preprocess/pipeline.py
+35 -20
View File
@@ -16,12 +16,13 @@ from preprocess.tools import (
class PreprocessPipeline:
def __init__(self, device: str, language: str, save_dir: str, vocal_sep: bool = True, max_merge_duration: int = 60000):
def __init__(self, device: str, language: str, save_dir: str, vocal_sep: bool = True, max_merge_duration: int = 60000, midi_transcribe: bool = True):
self.device = device
self.language = language
self.save_dir = save_dir
self.vocal_sep = vocal_sep
self.max_merge_duration = max_merge_duration
self.midi_transcribe = midi_transcribe
if vocal_sep:
self.vocal_separator = VocalSeparator(
@@ -37,26 +38,31 @@ class PreprocessPipeline:
model_path="pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt",
device=device,
)
self.vocal_detector = VocalDetector(
cut_wavs_output_dir= f"{save_dir}/cut_wavs",
)
self.lyric_transcriber = LyricTranscriber(
zh_model_path="pretrained_models/SoulX-Singer-Preprocess/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
en_model_path="pretrained_models/SoulX-Singer-Preprocess/parakeet-tdt-0.6b-v2/parakeet-tdt-0.6b-v2.nemo",
device=device
)
self.note_transcriber = NoteTranscriber(
rosvot_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rosvot/model.pt",
rwbd_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rwbd/model.pt",
device=device
)
if self.midi_transcribe:
self.vocal_detector = VocalDetector(
cut_wavs_output_dir= f"{save_dir}/cut_wavs",
)
self.lyric_transcriber = LyricTranscriber(
zh_model_path="pretrained_models/SoulX-Singer-Preprocess/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
en_model_path="pretrained_models/SoulX-Singer-Preprocess/parakeet-tdt-0.6b-v2/parakeet-tdt-0.6b-v2.nemo",
device=device
)
self.note_transcriber = NoteTranscriber(
rosvot_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rosvot/model.pt",
rwbd_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rwbd/model.pt",
device=device
)
else:
self.vocal_detector = None
self.lyric_transcriber = None
self.note_transcriber = None
def run(
self,
audio_path: str,
vocal_sep: bool = True,
max_merge_duration: int = 60000,
language: str = "Mandarin"
vocal_sep: bool = None,
max_merge_duration: int = None,
language: str = None,
) -> None:
vocal_sep = self.vocal_sep if vocal_sep is None else vocal_sep
max_merge_duration = self.max_merge_duration if max_merge_duration is None else max_merge_duration
@@ -81,7 +87,11 @@ class PreprocessPipeline:
vocal_path = output_dir / "vocal.wav"
sf.write(vocal_path, vocal, sample_rate)
vocal_f0 = self.f0_extractor.process(str(vocal_path))
vocal_f0 = self.f0_extractor.process(str(vocal_path), f0_path=str(vocal_path).replace(".wav", "_f0.npy"))
if not self.midi_transcribe or self.vocal_detector is None or self.lyric_transcriber is None or self.note_transcriber is None:
return
segments = self.vocal_detector.process(str(vocal_path), f0=vocal_f0)
metadata = []
@@ -124,10 +134,11 @@ def main(args):
save_dir=args.save_dir,
vocal_sep=args.vocal_sep,
max_merge_duration=args.max_merge_duration,
midi_transcribe=args.midi_transcribe,
)
pipeline.run(
audio_path=args.audio_path,
language=args.language
language=args.language,
)
@@ -139,8 +150,12 @@ if __name__ == "__main__":
parser.add_argument("--save_dir", type=str, required=True, help="Directory to save the output files")
parser.add_argument("--language", type=str, default="Mandarin", help="Language of the audio")
parser.add_argument("--device", type=str, default="cuda:0", help="Device to run the models on")
parser.add_argument("--vocal_sep", type=bool, default=True, help="Whether to perform vocal separation")
parser.add_argument("--vocal_sep", type=str, default="True", help="Whether to perform vocal separation")
parser.add_argument("--max_merge_duration", type=int, default=60000, help="Maximum merged segment duration in milliseconds")
parser.add_argument("--midi_transcribe", type=str, default="True", help="Whether to do MIDI transcription")
args = parser.parse_args()
args.vocal_sep = args.vocal_sep.lower() == "true"
args.midi_transcribe = args.midi_transcribe.lower() == "true"
main(args)
preprocess/tools/vocal_separation/model.py
+25 -13
View File
@@ -54,7 +54,7 @@ def build_model(args):
return model, config
def build_models(dict_args):
def build_models(dict_args, use_der: bool = False):
args = parse_args_inference(dict_args)
########## load model ##########
@@ -65,25 +65,26 @@ def build_models(dict_args):
sep_model, sep_config = build_model(args)
args.config_path = args.der_config_path
args.start_check_point = args.der_start_check_point
dereverb_model, dereverb_config = build_model(args)
sep_model = sep_model
dereverb_model = dereverb_model
if use_der:
args.config_path = args.der_config_path
args.start_check_point = args.der_start_check_point
dereverb_model, dereverb_config = build_model(args)
else:
dereverb_model, dereverb_config = None, None
return sep_model, sep_config, dereverb_model, dereverb_config, args
def main(args, sep_model=None, sep_config=None, dereverb_model=None, dereverb_config=None, device=None):
######## process data ##########
sample_rate = getattr(sep_config.audio, 'sample_rate', 44100)
path = args.input_path
mix, _ = librosa.load(path, sr=sample_rate, mono=False)
vocals = process(mix, sep_model, args, sep_config, device)
dereverbed_vocals = process(vocals.mean(0), dereverb_model, args, dereverb_config, device)
if dereverb_model is not None and dereverb_config is not None:
dereverbed_vocals = process(vocals.mean(0), dereverb_model, args, dereverb_config, device)
else:
dereverbed_vocals = vocals
accompaniment = mix - dereverbed_vocals
return mix, vocals, dereverbed_vocals, accompaniment, sample_rate
@@ -113,6 +114,8 @@ class VocalSeparator:
der_model_path: str,
der_config_path: str,
*,
chunk_length_sec: int = 5,
use_der: bool = False,
model_type: str = "mel_band_roformer",
disable_detailed_pbar: bool = True,
device: str = "cuda",
@@ -122,11 +125,14 @@ class VocalSeparator:
Args:
device: Torch device string, e.g. ``"cuda:0"``.
use_der: If True, load and run dereverb model; if False, skip dereverb (default False).
model_type: Separation model type key.
sep_config_path: Config path for separation model.
sep_start_check_point: Checkpoint path for separation model.
der_config_path: Config path for dereverb model.
der_start_check_point: Checkpoint path for dereverb model.
chunk_length_sec: Chunk length in seconds. Set lower if you want to reduce gpu memory usage.
use_der: If True, load and run dereverb model; if False, skip dereverb (default False). Set to False if you want to reduce gpu memory usage.
disable_detailed_pbar: Disable detailed progress bars in underlying utils.
verbose: Whether to print verbose logs.
"""
@@ -144,10 +150,15 @@ class VocalSeparator:
if verbose:
print("[vocal extraction] init: start")
sep_model, sep_config, dereverb_model, dereverb_config, args = build_models(args_dict)
sep_model, sep_config, dereverb_model, dereverb_config, args = build_models(args_dict, use_der=use_der)
sep_model = sep_model.half()
sep_model = sep_model.to(device)
dereverb_model = dereverb_model.to(device)
sep_config.inference.chunk_size = int(chunk_length_sec * sep_config.audio.sample_rate)
if dereverb_model is not None:
dereverb_config.inference.chunk_size = int(chunk_length_sec * dereverb_config.audio.sample_rate)
dereverb_model = dereverb_model.half()
dereverb_model = dereverb_model.to(device)
self.sep_model = sep_model
self.sep_config = sep_config
@@ -158,8 +169,9 @@ class VocalSeparator:
self.verbose = verbose
if verbose:
der_status = "loaded" if dereverb_model is not None else "skipped"
print(
"[vocal extraction] init success: sep=loaded, dereverb=loaded, device=",
"[vocal extraction] init success: sep=loaded, dereverb=%s, device=" % der_status,
device,
)