2026-02-09 10:30:13 +08:00
import os
import random
import shutil
import sys
import traceback
2026-03-04 13:14:09 +08:00
import gc
2026-02-16 01:18:48 +08:00
from datetime import datetime
2026-02-09 10:30:13 +08:00
from pathlib import Path
from typing import Literal , Tuple
import numpy as np
import torch
import librosa
import soundfile as sf
import gradio as gr
from preprocess.pipeline import PreprocessPipeline
2026-02-16 01:18:48 +08:00
from preprocess.tools.midi_parser import MidiParser
2026-02-09 10:30:13 +08:00
from soulxsinger.utils.file_utils import load_config
from cli.inference import build_model as build_svs_model , process as svs_process
ROOT = Path ( __file__ ) . parent
2026-02-16 01:18:48 +08:00
SAMPLE_RATE = 44100
PROMPT_MAX_SEC_DEFAULT = 30
TARGET_MAX_SEC_DEFAULT = 60
PROMPT_MAX_MERGE_DURATION_MS = 30000
TARGET_MAX_MERGE_DURATION_MS = 60000
2026-02-09 10:30:13 +08:00
ENGLISH_EXAMPLE_PROMPT_AUDIO = "example/audio/en_prompt.mp3"
ENGLISH_EXAMPLE_PROMPT_META = "example/audio/en_prompt.json"
ENGLISH_EXAMPLE_TARGET_AUDIO = "example/audio/en_target.mp3"
ENGLISH_EXAMPLE_TARGET_META = "example/audio/en_target.json"
MANDARIN_EXAMPLE_PROMPT_AUDIO = "example/audio/zh_prompt.mp3"
MANDARIN_EXAMPLE_PROMPT_META = "example/audio/zh_prompt.json"
MANDARIN_EXAMPLE_TARGET_AUDIO = "example/audio/zh_target.mp3"
MANDARIN_EXAMPLE_TARGET_META = "example/audio/zh_target.json"
CANTONESE_EXAMPLE_PROMPT_AUDIO = "example/audio/yue_prompt.mp3"
CANTONESE_EXAMPLE_PROMPT_META = "example/audio/yue_prompt.json"
CANTONESE_EXAMPLE_TARGET_AUDIO = "example/audio/yue_target.mp3"
CANTONESE_EXAMPLE_TARGET_META = "example/audio/yue_target.json"
MUSIC_EXAMPLE_TARGET_AUDIO = "example/audio/music.mp3"
MUSIC_EXAMPLE_TARGET_META = "example/audio/music.json"
# Use absolute paths so Examples load correctly (including File components for metadata)
EXAMPLES_LIST = [
[
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_AUDIO ),
str ( ROOT / MANDARIN_EXAMPLE_TARGET_AUDIO ),
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_META ),
str ( ROOT / MANDARIN_EXAMPLE_TARGET_META ),
"Mandarin" ,
"Mandarin" ,
2026-02-16 01:18:48 +08:00
"melody-controlled" ,
"no" ,
"yes" ,
"yes" ,
2026-02-09 10:30:13 +08:00
0 ,
],
[
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_AUDIO ),
str ( ROOT / CANTONESE_EXAMPLE_TARGET_AUDIO ),
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_META ),
str ( ROOT / CANTONESE_EXAMPLE_TARGET_META ),
"Mandarin" ,
"Cantonese" ,
2026-02-16 01:18:48 +08:00
"melody-controlled" ,
"no" ,
"yes" ,
"yes" ,
2026-02-09 10:30:13 +08:00
0 ,
],
[
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_AUDIO ),
str ( ROOT / ENGLISH_EXAMPLE_TARGET_AUDIO ),
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_META ),
str ( ROOT / ENGLISH_EXAMPLE_TARGET_META ),
"Mandarin" ,
"English" ,
2026-02-16 01:18:48 +08:00
"melody-controlled" ,
"no" ,
"yes" ,
"yes" ,
2026-02-09 10:30:13 +08:00
0 ,
],
[
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_AUDIO ),
str ( ROOT / MUSIC_EXAMPLE_TARGET_AUDIO ),
str ( ROOT / MANDARIN_EXAMPLE_PROMPT_META ),
str ( ROOT / MUSIC_EXAMPLE_TARGET_META ),
"Mandarin" ,
"Mandarin" ,
2026-02-16 01:18:48 +08:00
"score-controlled" ,
"no" ,
"yes" ,
"yes" ,
2026-02-09 10:30:13 +08:00
0 ,
],
]
2026-02-16 01:18:48 +08:00
# i18n
_I18N_KEY2LANG = dict (
display_lang_label = dict ( en = "Display Language" , zh = "显示语言" ),
section_input_audio = dict ( en = "Input Audio" , zh = "输入音频" ),
section_transcriptions = dict ( en = "Transcriptions & Metadata" , zh = "转录与元数据" ),
section_synthesis = dict ( en = "Singing Synthesis" , zh = "歌声合成" ),
seed_label = dict ( en = "Seed" , zh = "种子" ),
prompt_audio_label = dict ( en = f "Prompt audio (reference voice), limit to { PROMPT_MAX_SEC_DEFAULT } seconds" , zh = f "Prompt 音频（参考音色），限制在 { PROMPT_MAX_SEC_DEFAULT } 秒以内" ),
target_audio_label = dict ( en = f "Target audio (melody / lyrics source), limit to { TARGET_MAX_SEC_DEFAULT } seconds" , zh = f "Target 音频（旋律/歌词来源），限制在 { TARGET_MAX_SEC_DEFAULT } 秒以内" ),
transcription_btn_label = dict ( en = "Run singing transcription" , zh = "开始歌声转录" ),
synthesis_btn_label = dict ( en = "🎤Generate singing voice" , zh = "🎤歌声合成" ),
prompt_meta_label = dict ( en = "Prompt metadata" , zh = "Prompt 元数据" ),
prompt_midi_label = dict ( en = "Prompt MIDI" , zh = "Prompt MIDI" ),
target_meta_label = dict ( en = "Target metadata" , zh = "Target 元数据" ),
target_midi_label = dict ( en = "Target MIDI" , zh = "Target MIDI" ),
prompt_wav_label = dict ( en = "Prompt WAV (reference)" , zh = "Prompt WAV（参考音色）" ),
generated_audio_label = dict ( en = "Generated merged audio" , zh = "合成结果音频" ),
prompt_lyric_lang_label = dict ( en = "Prompt lyric language" , zh = "Prompt 歌词语种" ),
target_lyric_lang_label = dict ( en = "Target lyric language" , zh = "Target 歌词语种" ),
lyric_lang_mandarin = dict ( en = "Mandarin" , zh = "普通话" ),
lyric_lang_cantonese = dict ( en = "Cantonese" , zh = "粤语" ),
lyric_lang_english = dict ( en = "English" , zh = "英语" ),
warn_missing_synthesis = dict (
en = "Please provide prompt WAV, prompt metadata, and target metadata. Check the content in Transcriptions & Metadata above." ,
zh = "请提供 Prompt WAV、Prompt metadata 与 Target metadata，并检查上方 Transcriptions & Metadata 里的内容。" ,
),
prompt_vocal_sep_label = dict ( en = "Prompt vocal separation" , zh = "Prompt人声分离" ),
target_vocal_sep_label = dict ( en = "Target vocal separation" , zh = "Target人声分离" ),
option_yes = dict ( en = "yes" , zh = "是" ),
option_no = dict ( en = "no" , zh = "否" ),
auto_shift_label = dict ( en = "Auto pitch shift" , zh = "自动变调" ),
pitch_shift_label = dict ( en = "Pitch shift (semitones)" , zh = "指定变调（半音）" ),
control_type_label = dict ( en = "Control type" , zh = "控制类型" ),
control_melody = dict ( en = "melody-controlled" , zh = "旋律控制" ),
control_score = dict ( en = "score-controlled" , zh = "乐谱控制" ),
examples_label = dict ( en = "Reference examples (click to load)" , zh = "参考样例（点击加载）" ),
example_choice_0 = dict ( en = "—" , zh = "—" ),
example_choice_1 = dict ( en = "Example 1: Mandarin → Mandarin (melody), Start singing synthesis!" , zh = "样例 1: 普通话 → 普通话 (melody), 开始歌声合成吧!" ),
example_choice_2 = dict ( en = "Example 2: Mandarin → Cantonese (melody), Start singing synthesis!" , zh = "样例 2: 普通话 → 粤语 (melody), 开始歌声合成吧!" ),
example_choice_3 = dict ( en = "Example 3: Mandarin → English (melody), Start singing synthesis!" , zh = "样例 3: 普通话 → 英语 (melody), 开始歌声合成吧!" ),
example_choice_4 = dict ( en = "Example 4: Mandarin → Music (score), Start singing synthesis!" , zh = "样例 4: 普通话 → 音乐 (score), 开始歌声合成吧!" ),
instruction_title = dict ( en = "Usage" , zh = "使用说明" ),
instruction_p1 = dict (
en = "Upload prompt and target audio, and the corresponding metadata and MIDI files will be automatically transcribed." ,
zh = "上传 Prompt 与 Target 音频，将自动转录生成 Prompt 与 Target 两份 metadata 文件以及对应的 MIDI 文件。" ,
),
instruction_p2 = dict (
en = "Auto-transcribed lyrics and notes are often misaligned, which may lead to suboptimal synthesis results. For best results, import the generated MIDI into the [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) for manual adjustment. After adjustment, re-upload the MIDI file and the metadata will be automatically updated." ,
zh = "自动转录的歌词与音高对齐效果通常不理想，可能导致合成效果不佳，建议将生成的 MIDI 文件导入 [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) 进行手动调整，调整后的 MIDI 文件重新上传后，metadata 将会自动更新。" ,
),
instruction_p3 = dict (
en = "Once prompt audio, prompt metadata, and target metadata are all set, click **🎤Generate singing voice** to run the singing synthesis and generate the final merged audio." ,
zh = "Prompt audio, Prompt metadata 和 Target metadata 都准备好后，点击「🎤歌声合成」开始最终生成。" ,
),
)
_GLOBAL_LANG : Literal [ "zh" , "en" ] = "zh"
def _i18n ( key : str ) -> str :
return _I18N_KEY2LANG [ key ][ _GLOBAL_LANG ]
2026-02-09 10:30:13 +08:00
def _load_example ( choice_value ):
2026-02-16 01:18:48 +08:00
"""Return 11 example values + skip_clear_count.
When loading an example, the next two audio.change events should not clear metadata.
"""
output_count = 11
2026-02-09 10:30:13 +08:00
if choice_value is None :
2026-02-16 01:18:48 +08:00
return [ gr . update ()] * output_count + [ 0 ]
choice_to_index = {
_i18n ( "example_choice_1" ): 1 ,
_i18n ( "example_choice_2" ): 2 ,
_i18n ( "example_choice_3" ): 3 ,
_i18n ( "example_choice_4" ): 4 ,
}
2026-02-09 10:30:13 +08:00
idx = 0
if isinstance ( choice_value , int ):
idx = 0 if choice_value <= 0 else min ( choice_value - 1 , len ( EXAMPLES_LIST ) - 1 )
else :
2026-02-16 01:18:48 +08:00
idx = choice_to_index . get ( choice_value , 0 )
2026-02-09 10:30:13 +08:00
if idx <= 0 :
2026-02-16 01:18:48 +08:00
return [ gr . update ()] * output_count + [ 0 ]
2026-02-09 10:30:13 +08:00
list_idx = idx - 1
if list_idx >= len ( EXAMPLES_LIST ):
2026-02-16 01:18:48 +08:00
return [ gr . update ()] * output_count + [ 0 ]
2026-02-09 10:30:13 +08:00
row = EXAMPLES_LIST [ list_idx ]
return [
row [ 0 ], row [ 1 ], row [ 2 ], row [ 3 ], row [ 4 ], row [ 5 ], row [ 6 ], row [ 7 ], row [ 8 ], row [ 9 ], row [ 10 ],
2 , # skip_clear_metadata_count: next 2 audio.change events (prompt + target) will not clear metadata
]
def _clear_prompt_meta_unless_example ( _audio , skip_count ):
if skip_count and skip_count > 0 :
return gr . skip (), max ( 0 , skip_count - 1 )
return None , 0
def _clear_target_meta_unless_example ( _audio , skip_count ):
if skip_count and skip_count > 0 :
return gr . skip (), max ( 0 , skip_count - 1 )
return None , 0
def _get_device () -> str :
"""Use CUDA if available, else CPU (e.g. for CI or CPU-only environments)."""
return "cuda:0" if torch . cuda . is_available () else "cpu"
2026-02-16 01:18:48 +08:00
def _session_dir () -> Path :
# Use per-call timestamped session dir to avoid cross-request collisions.
timestamp = datetime . now () . strftime ( "%Y%m %d _%H%M%S_ %f " )
return ROOT / "outputs" / "gradio" / timestamp
def _print_exception ( context : str ) -> None :
print ( f "[ { context } ] \n { traceback . format_exc () } " , file = sys . stderr , flush = True )
def _get_lyric_lang_choices ():
"""Lyric language dropdown (display, value) for current UI language."""
return [
( _i18n ( "lyric_lang_mandarin" ), "Mandarin" ),
( _i18n ( "lyric_lang_cantonese" ), "Cantonese" ),
( _i18n ( "lyric_lang_english" ), "English" ),
]
def _resolve_file_path ( x ):
"""Gradio file input can be path string or (path, None) tuple."""
if x is None :
return None
if isinstance ( x , tuple ):
x = x [ 0 ]
return x if ( x and os . path . isfile ( x )) else None
def _normalize_audio_input ( audio ):
"""Normalize Gradio audio input to a filepath string."""
return audio [ 0 ] if isinstance ( audio , tuple ) else audio
def _trim_and_save_audio ( src_audio_path : str , dst_wav_path : Path , max_sec : int , sr : int = SAMPLE_RATE ) -> None :
"""Load audio as mono, trim to max_sec, and save as wav for preprocess."""
audio_data , _ = librosa . load ( src_audio_path , sr = sr , mono = True )
audio_data = audio_data [: max_sec * sr ]
sf . write ( dst_wav_path , audio_data , sr )
def _yes_no_to_bool ( value , default : bool = False ) -> bool :
if isinstance ( value , bool ):
return value
if value is None :
return default
return str ( value ) . strip () . lower () == "yes"
def _control_to_internal ( control : str ) -> str :
if control in ( "melody" , "melody-controlled" ):
return "melody"
return "score"
2026-02-09 10:30:13 +08:00
class AppState :
2026-03-16 13:40:30 +08:00
def __init__ ( self , use_fp16 : bool = False ) -> None :
2026-02-09 10:30:13 +08:00
self . device = _get_device ()
2026-03-16 13:40:30 +08:00
self . use_fp16 = use_fp16 and ( "cuda" in self . device )
2026-02-09 10:30:13 +08:00
self . preprocess_pipeline = PreprocessPipeline (
device = self . device ,
language = "Mandarin" ,
save_dir = str ( ROOT / "outputs" / "gradio" / "_placeholder" / "transcriptions" ),
vocal_sep = True ,
max_merge_duration = 60000 ,
)
config = load_config ( "soulxsinger/config/soulxsinger.yaml" )
self . svs_config = config
self . svs_model = build_svs_model (
model_path = "pretrained_models/SoulX-Singer/model.pt" ,
config = config ,
device = self . device ,
2026-03-16 13:40:30 +08:00
use_fp16 = self . use_fp16 ,
2026-02-09 10:30:13 +08:00
)
self . phoneset_path = "soulxsinger/utils/phoneme/phone_set.json"
2026-02-16 01:18:48 +08:00
self . midi_parser = MidiParser (
rmvpe_model_path = "pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt" ,
device = self . device
)
2026-02-09 10:30:13 +08:00
def run_preprocess (
self ,
2026-02-16 01:18:48 +08:00
audio_path : Path ,
save_path : Path ,
vocal_sep : bool ,
lyric_lang : str ,
max_merge_duration : int
2026-02-09 10:30:13 +08:00
) -> Tuple [ bool , str ]:
try :
2026-02-16 01:18:48 +08:00
self . preprocess_pipeline . save_dir = str ( save_path )
2026-02-09 10:30:13 +08:00
self . preprocess_pipeline . run (
2026-02-16 01:18:48 +08:00
audio_path = str ( audio_path ),
vocal_sep = vocal_sep ,
max_merge_duration = max_merge_duration ,
language = lyric_lang or "Mandarin" ,
2026-02-09 10:30:13 +08:00
)
2026-03-04 13:14:09 +08:00
gc . collect ()
if torch . cuda . is_available ():
torch . cuda . empty_cache ()
2026-02-16 01:18:48 +08:00
return True , f "preprocess { audio_path } done"
2026-02-09 10:30:13 +08:00
except Exception as e :
return False , f "preprocess failed: { e } "
def run_svs (
self ,
control : str ,
session_base : Path ,
auto_shift : bool ,
pitch_shift : int ,
) -> Tuple [ bool , str , Path | None , Path | None , Path | None ]:
if control not in ( "melody" , "score" ):
control = "score"
save_dir = session_base / "generated"
save_dir . mkdir ( parents = True , exist_ok = True )
class Args :
pass
args = Args ()
args . device = self . device
2026-02-09 16:16:14 +08:00
args . model_path = "pretrained_models/SoulX-Singer/model.pt"
2026-02-09 10:30:13 +08:00
args . config = "soulxsinger/config/soulxsinger.yaml"
args . prompt_wav_path = str ( session_base / "audio" / "prompt.wav" )
prompt_meta_path = session_base / "transcriptions" / "prompt" / "metadata.json"
target_meta_path = session_base / "transcriptions" / "target" / "metadata.json"
args . prompt_metadata_path = str ( prompt_meta_path )
args . target_metadata_path = str ( target_meta_path )
args . phoneset_path = self . phoneset_path
args . save_dir = str ( save_dir )
args . auto_shift = auto_shift
args . pitch_shift = int ( pitch_shift )
args . control = control
2026-03-16 13:40:30 +08:00
args . use_fp16 = self . use_fp16
2026-02-09 10:30:13 +08:00
try :
svs_process ( args , self . svs_config , self . svs_model )
2026-03-04 13:14:09 +08:00
gc . collect ()
if torch . cuda . is_available ():
torch . cuda . empty_cache ()
2026-02-09 10:30:13 +08:00
generated = save_dir / "generated.wav"
if not generated . exists ():
return False , f "inference finished but { generated } not found" , None , prompt_meta_path , target_meta_path
return True , "svs inference done" , generated , prompt_meta_path , target_meta_path
except Exception as e :
return False , f "svs inference failed: { e } " , None , prompt_meta_path , target_meta_path
def run_svs_from_paths (
self ,
prompt_wav_path : str ,
prompt_metadata_path : str ,
target_metadata_path : str ,
control : str ,
auto_shift : bool ,
pitch_shift : int ,
save_dir : Path | None = None ,
) -> Tuple [ bool , str , Path | None ]:
"""Run SVS from explicit prompt wav and metadata paths."""
if save_dir is None :
import uuid
save_dir = ROOT / "outputs" / "gradio" / "synthesis" / str ( uuid . uuid4 ())[: 8 ]
save_dir = Path ( save_dir )
audio_dir = save_dir / "audio"
prompt_meta_dir = save_dir / "transcriptions" / "prompt"
target_meta_dir = save_dir / "transcriptions" / "target"
audio_dir . mkdir ( parents = True , exist_ok = True )
prompt_meta_dir . mkdir ( parents = True , exist_ok = True )
target_meta_dir . mkdir ( parents = True , exist_ok = True )
shutil . copy2 ( prompt_wav_path , audio_dir / "prompt.wav" )
shutil . copy2 ( prompt_metadata_path , prompt_meta_dir / "metadata.json" )
shutil . copy2 ( target_metadata_path , target_meta_dir / "metadata.json" )
ok , msg , merged , _ , _ = self . run_svs (
control = control ,
session_base = save_dir ,
auto_shift = auto_shift ,
pitch_shift = pitch_shift ,
)
if not ok or merged is None :
return False , msg or "svs failed" , None
return True , "svs inference done" , merged
2026-03-16 13:40:30 +08:00
APP_STATE = AppState ( use_fp16 = "--fp16" in sys . argv )
2026-02-09 10:30:13 +08:00
2026-02-16 01:18:48 +08:00
def _edit_metadata (
meta ,
midi ,
audio ,
language : str = "Mandarin" ,
2026-02-09 10:30:13 +08:00
):
try :
2026-02-16 01:18:48 +08:00
meta = _resolve_file_path ( meta )
midi = _resolve_file_path ( midi )
if not midi :
return meta
audio = _normalize_audio_input ( audio )
if not meta :
meta = str ( Path ( midi ) . with_name ( "metadata.json" ))
APP_STATE . midi_parser . midi2meta ( midi , meta , audio , language = language )
return meta
except Exception :
_print_exception ( "_edit_metadata" )
return meta
def _transcribe_prompt (
prompt_audio ,
prompt_metadata ,
prompt_lyric_lang : str ,
prompt_vocal_sep ,
prompt_max_sec : int = PROMPT_MAX_SEC_DEFAULT ,
):
try :
prompt_audio = _normalize_audio_input ( prompt_audio )
2026-02-09 10:30:13 +08:00
prompt_meta_resolved = _resolve_file_path ( prompt_metadata )
2026-02-16 01:18:48 +08:00
if prompt_audio is None and prompt_meta_resolved is None :
return None , None
2026-02-09 10:30:13 +08:00
2026-02-16 01:18:48 +08:00
session_base = _session_dir ()
2026-02-09 10:30:13 +08:00
prompt_meta_path = session_base / "transcriptions" / "prompt" / "metadata.json"
2026-02-16 01:18:48 +08:00
prompt_midi_path = session_base / "transcriptions" / "prompt" / "vocal.mid"
if prompt_audio is not None :
audio_dir = session_base / "audio"
audio_dir . mkdir ( parents = True , exist_ok = True )
transfer_prompt_path = audio_dir / "prompt.wav"
_trim_and_save_audio ( prompt_audio , transfer_prompt_path , prompt_max_sec )
prompt_ok , prompt_msg = APP_STATE . run_preprocess (
audio_path = transfer_prompt_path ,
save_path = session_base / "transcriptions" / "prompt" ,
vocal_sep = _yes_no_to_bool ( prompt_vocal_sep , default = False ),
lyric_lang = prompt_lyric_lang or "Mandarin" ,
max_merge_duration = PROMPT_MAX_MERGE_DURATION_MS ,
2026-02-09 10:30:13 +08:00
)
2026-02-16 01:18:48 +08:00
if not prompt_ok :
print ( prompt_msg , file = sys . stderr , flush = True )
2026-02-09 10:30:13 +08:00
return None , None
2026-02-16 01:18:48 +08:00
elif prompt_meta_resolved is not None :
( session_base / "transcriptions" / "prompt" ) . mkdir ( parents = True , exist_ok = True )
shutil . copy2 ( prompt_meta_resolved , prompt_meta_path )
if prompt_meta_path . exists ():
APP_STATE . midi_parser . meta2midi ( prompt_meta_path , prompt_midi_path )
2026-02-09 10:30:13 +08:00
prompt_meta_file = str ( prompt_meta_path ) if prompt_meta_path . exists () else None
2026-02-16 01:18:48 +08:00
prompt_midi_file = str ( prompt_midi_path ) if prompt_midi_path . exists () else None
return prompt_meta_file , prompt_midi_file
2026-02-09 10:30:13 +08:00
except Exception :
2026-02-16 01:18:48 +08:00
_print_exception ( "_transcribe_prompt" )
2026-02-09 10:30:13 +08:00
return None , None
2026-02-16 01:18:48 +08:00
def _transcribe_target (
target_audio ,
target_metadata ,
target_lyric_lang : str ,
target_vocal_sep ,
target_max_sec : int = TARGET_MAX_SEC_DEFAULT ,
):
try :
target_audio = _normalize_audio_input ( target_audio )
target_meta_resolved = _resolve_file_path ( target_metadata )
if target_audio is None and target_meta_resolved is None :
return None , None , None
session_base = _session_dir ()
target_meta_path = session_base / "transcriptions" / "target" / "metadata.json"
target_midi_path = session_base / "transcriptions" / "target" / "vocal.mid"
target_vocal_path = session_base / "transcriptions" / "target" / "vocal.wav"
if target_audio is not None :
audio_dir = session_base / "audio"
audio_dir . mkdir ( parents = True , exist_ok = True )
transfer_target_path = audio_dir / "target.wav"
_trim_and_save_audio ( target_audio , transfer_target_path , target_max_sec )
target_ok , target_msg = APP_STATE . run_preprocess (
audio_path = transfer_target_path ,
save_path = session_base / "transcriptions" / "target" ,
vocal_sep = _yes_no_to_bool ( target_vocal_sep , default = True ),
lyric_lang = target_lyric_lang or "Mandarin" ,
max_merge_duration = TARGET_MAX_MERGE_DURATION_MS ,
)
if not target_ok :
print ( target_msg , file = sys . stderr , flush = True )
return None , None , None
elif target_meta_resolved is not None :
( session_base / "transcriptions" / "target" ) . mkdir ( parents = True , exist_ok = True )
shutil . copy2 ( target_meta_resolved , target_meta_path )
if target_meta_path . exists ():
APP_STATE . midi_parser . meta2midi ( target_meta_path , target_midi_path )
target_meta_file = str ( target_meta_path ) if target_meta_path . exists () else None
target_midi_file = str ( target_midi_path ) if target_midi_path . exists () else None
target_vocal_file = str ( target_vocal_path ) if target_vocal_path . exists () else None
return target_meta_file , target_midi_file , target_vocal_file
except Exception :
_print_exception ( "_transcribe_target" )
return None , None , None
def _run_synthesis (
2026-02-09 10:30:13 +08:00
prompt_audio ,
prompt_metadata ,
target_metadata ,
control : str ,
2026-02-16 01:18:48 +08:00
auto_shift ,
2026-02-09 10:30:13 +08:00
pitch_shift ,
seed : int ,
):
2026-02-16 01:18:48 +08:00
"""Run singing synthesis from prompt audio + prompt metadata + target metadata."""
2026-02-09 10:30:13 +08:00
try :
2026-02-16 01:18:48 +08:00
prompt_audio = _normalize_audio_input ( prompt_audio )
2026-02-09 10:30:13 +08:00
prompt_wav_path = prompt_audio
prompt_meta_path = _resolve_file_path ( prompt_metadata )
target_meta_path = _resolve_file_path ( target_metadata )
if not prompt_wav_path or not os . path . isfile ( prompt_wav_path ):
2026-02-16 01:18:48 +08:00
gr . Warning ( message = _i18n ( "warn_missing_synthesis" ))
2026-02-09 10:30:13 +08:00
return None
if not prompt_meta_path or not os . path . isfile ( prompt_meta_path ):
2026-02-16 01:18:48 +08:00
gr . Warning ( message = _i18n ( "warn_missing_synthesis" ))
2026-02-09 10:30:13 +08:00
return None
if not target_meta_path or not os . path . isfile ( target_meta_path ):
2026-02-16 01:18:48 +08:00
gr . Warning ( message = _i18n ( "warn_missing_synthesis" ))
2026-02-09 10:30:13 +08:00
return None
2026-02-16 01:18:48 +08:00
control = _control_to_internal ( control )
auto_shift = _yes_no_to_bool ( auto_shift , default = True )
2026-02-09 10:30:13 +08:00
seed = int ( seed )
torch . manual_seed ( seed )
np . random . seed ( seed )
random . seed ( seed )
ok , msg , merged = APP_STATE . run_svs_from_paths (
prompt_wav_path = prompt_wav_path ,
prompt_metadata_path = prompt_meta_path ,
target_metadata_path = target_meta_path ,
control = control ,
auto_shift = auto_shift ,
pitch_shift = int ( pitch_shift ),
)
if not ok or merged is None :
print ( msg or "synthesis failed" , file = sys . stderr , flush = True )
return None
return str ( merged )
except Exception :
2026-02-16 01:18:48 +08:00
_print_exception ( "_run_synthesis" )
2026-02-09 10:30:13 +08:00
return None
def _instruction_md () -> str :
"""Markdown content for the instruction panel (supports links)."""
return " \n\n " . join ([
2026-02-16 01:18:48 +08:00
f "**1.** { _i18n ( 'instruction_p1' ) } " ,
f "**2.** { _i18n ( 'instruction_p2' ) } " ,
f "**3.** { _i18n ( 'instruction_p3' ) } " ,
2026-02-09 10:30:13 +08:00
])
def render_interface () -> gr . Blocks :
with gr . Blocks ( title = "SoulX-Singer 歌声合成Demo" , theme = gr . themes . Default ()) as page :
2026-02-09 12:03:12 +08:00
gr . HTML (
'<div style="'
'text-align: center; '
'padding: 1.25rem 0 1.5rem; '
'margin-bottom: 0.5rem;'
'">'
'<div style="'
'display: inline-block; '
'font-size: 1.75rem; '
'font-weight: 700; '
'letter-spacing: 0.02em; '
'color: #1a1a2e; '
'line-height: 1.3;'
'">SoulX-Singer</div>'
'<div style="'
'width: 80px; '
'height: 3px; '
'margin: 1rem auto 0; '
'background: linear-gradient(90deg, transparent, #6366f1, transparent); '
'border-radius: 2px;'
'"></div>'
'</div>'
)
2026-02-09 10:30:13 +08:00
with gr . Row ( equal_height = True ):
lang_choice = gr . Radio (
choices = [ "中文" , "English" ],
value = "中文" ,
2026-02-16 01:18:48 +08:00
label = _i18n ( "display_lang_label" ),
2026-02-09 10:30:13 +08:00
type = "index" ,
interactive = True ,
elem_id = "lang_choice_radio" ,
)
# Instruction panel (usage workflow); updates on language change
2026-02-16 01:18:48 +08:00
instruction_md = gr . Markdown ( f "### { _i18n ( 'instruction_title' ) } \n\n { _instruction_md () } " )
2026-02-09 10:30:13 +08:00
# Reference examples — at the front of operations (handler registered after components exist)
skip_clear_metadata_count = gr . State ( 0 )
with gr . Row ():
2026-02-16 01:18:48 +08:00
_example_choices = [ _i18n ( "example_choice_0" ), _i18n ( "example_choice_1" ), _i18n ( "example_choice_2" ), _i18n ( "example_choice_3" ), _i18n ( "example_choice_4" )]
2026-02-09 10:30:13 +08:00
example_choice = gr . Dropdown (
2026-02-16 01:18:48 +08:00
label = _i18n ( "examples_label" ),
2026-02-09 10:30:13 +08:00
choices = _example_choices ,
value = _example_choices [ 0 ],
interactive = True ,
)
# Step 1: Transcription (audio → metadata)
2026-02-16 01:18:48 +08:00
with gr . Accordion ( _i18n ( "section_input_audio" ), open = True ) as accordion_input_audio :
with gr . Row ( equal_height = True ):
with gr . Column ( scale = 1 ):
prompt_audio = gr . Audio (
label = _i18n ( "prompt_audio_label" ),
type = "filepath" ,
editable = False ,
interactive = True ,
)
with gr . Column ( scale = 1 ):
target_audio = gr . Audio (
label = _i18n ( "target_audio_label" ),
type = "filepath" ,
editable = False ,
interactive = True ,
)
with gr . Accordion ( _i18n ( "section_transcriptions" ), open = True ) as accordion_transcriptions :
with gr . Row ( equal_height = True ):
prompt_lyric_lang = gr . Dropdown (
label = _i18n ( "prompt_lyric_lang_label" ),
choices = _get_lyric_lang_choices (),
value = "Mandarin" ,
2026-02-09 10:30:13 +08:00
interactive = True ,
2026-02-16 01:18:48 +08:00
scale = 1 ,
2026-02-09 10:30:13 +08:00
)
2026-02-16 01:18:48 +08:00
prompt_vocal_sep = gr . Dropdown (
label = _i18n ( "prompt_vocal_sep_label" ),
choices = [( _i18n ( "option_yes" ), "yes" ), ( _i18n ( "option_no" ), "no" )],
value = "no" ,
2026-02-09 10:30:13 +08:00
interactive = True ,
2026-02-16 01:18:48 +08:00
scale = 1 ,
)
target_lyric_lang = gr . Dropdown (
label = _i18n ( "target_lyric_lang_label" ),
choices = _get_lyric_lang_choices (),
value = "Mandarin" ,
interactive = True ,
scale = 1 ,
)
target_vocal_sep = gr . Dropdown (
label = _i18n ( "target_vocal_sep_label" ),
choices = [( _i18n ( "option_yes" ), "yes" ), ( _i18n ( "option_no" ), "no" )],
value = "yes" ,
interactive = True ,
scale = 1 ,
2026-02-09 10:30:13 +08:00
)
2026-02-16 01:18:48 +08:00
with gr . Row ( equal_height = True ):
prompt_metadata = gr . File (
label = _i18n ( "prompt_meta_label" ),
type = "filepath" ,
file_types = [ ".json" ],
height = 140 ,
interactive = True ,
)
prompt_midi = gr . File (
label = _i18n ( "prompt_midi_label" ),
type = "filepath" ,
file_types = [ ".midi" , ".mid" ],
height = 140 ,
interactive = True ,
)
target_metadata = gr . File (
label = _i18n ( "target_meta_label" ),
type = "filepath" ,
file_types = [ ".json" ],
height = 140 ,
interactive = True ,
)
target_midi = gr . File (
label = _i18n ( "target_midi_label" ),
type = "filepath" ,
file_types = [ ".midi" , ".mid" ],
height = 140 ,
interactive = True ,
)
target_vocal = gr . File (
type = "filepath" ,
file_types = [ ".wav" ],
interactive = False ,
visible = False ,
)
with gr . Accordion ( _i18n ( "section_synthesis" ), open = True ) as accordion_synthesis :
with gr . Row ( equal_height = True ):
control_radio = gr . Dropdown (
choices = [( _i18n ( "control_melody" ), "melody-controlled" ), ( _i18n ( "control_score" ), "score-controlled" )],
value = "score-controlled" ,
label = _i18n ( "control_type_label" ),
scale = 1 ,
)
auto_shift = gr . Dropdown (
label = _i18n ( "auto_shift_label" ),
choices = [( _i18n ( "option_yes" ), "yes" ), ( _i18n ( "option_no" ), "no" )],
value = "yes" ,
interactive = True ,
scale = 1 ,
)
pitch_shift = gr . Number (
label = _i18n ( "pitch_shift_label" ),
value = 0 ,
minimum =- 36 ,
maximum = 36 ,
step = 1 ,
interactive = True ,
scale = 1 ,
)
seed_input = gr . Number (
label = _i18n ( "seed_label" ),
value = 12306 ,
step = 1 ,
interactive = True ,
scale = 1 ,
)
with gr . Row ():
synthesis_btn = gr . Button (
value = _i18n ( "synthesis_btn_label" ),
variant = "primary" ,
size = "lg" ,
)
with gr . Row ():
output_audio = gr . Audio (
label = _i18n ( "generated_audio_label" ),
type = "filepath" ,
interactive = False ,
)
2026-02-09 10:30:13 +08:00
example_choice . change (
fn = _load_example ,
inputs = [ example_choice ],
outputs = [
prompt_audio ,
target_audio ,
prompt_metadata ,
target_metadata ,
prompt_lyric_lang ,
target_lyric_lang ,
control_radio ,
prompt_vocal_sep ,
target_vocal_sep ,
auto_shift ,
pitch_shift ,
skip_clear_metadata_count ,
],
)
def _change_component_language ( lang ):
2026-02-16 01:18:48 +08:00
global _GLOBAL_LANG
_GLOBAL_LANG = [ "zh" , "en" ][ lang ]
lyric_choices = _get_lyric_lang_choices ()
yes_no_choices = [( _i18n ( "option_yes" ), "yes" ), ( _i18n ( "option_no" ), "no" )]
control_choices = [( _i18n ( "control_melody" ), "melody-controlled" ), ( _i18n ( "control_score" ), "score-controlled" )]
2026-02-09 10:30:13 +08:00
return [
2026-02-16 01:18:48 +08:00
gr . update ( label = _i18n ( "prompt_audio_label" )),
gr . update ( label = _i18n ( "target_audio_label" )),
gr . update ( label = _i18n ( "prompt_lyric_lang_label" ), choices = lyric_choices ),
gr . update ( label = _i18n ( "target_lyric_lang_label" ), choices = lyric_choices ),
gr . update ( label = _i18n ( "prompt_vocal_sep_label" ), choices = yes_no_choices ),
gr . update ( label = _i18n ( "target_vocal_sep_label" ), choices = yes_no_choices ),
gr . update ( label = _i18n ( "prompt_meta_label" )),
gr . update ( label = _i18n ( "target_meta_label" )),
gr . update ( label = _i18n ( "control_type_label" ), choices = control_choices ),
gr . update ( label = _i18n ( "auto_shift_label" ), choices = yes_no_choices ),
gr . update ( label = _i18n ( "pitch_shift_label" )),
gr . update ( label = _i18n ( "seed_label" )),
gr . update ( value = _i18n ( "synthesis_btn_label" )),
gr . update ( label = _i18n ( "generated_audio_label" )),
gr . update ( label = _i18n ( "display_lang_label" )),
2026-02-09 10:30:13 +08:00
gr . update (
2026-02-16 01:18:48 +08:00
label = _i18n ( "examples_label" ),
choices = [ _i18n ( "example_choice_0" ), _i18n ( "example_choice_1" ), _i18n ( "example_choice_2" ), _i18n ( "example_choice_3" ), _i18n ( "example_choice_4" )],
value = _i18n ( "example_choice_0" ),
2026-02-09 10:30:13 +08:00
),
2026-02-16 01:18:48 +08:00
gr . update ( value = f "### { _i18n ( 'instruction_title' ) } \n\n { _instruction_md () } " ),
gr . update ( label = _i18n ( "section_input_audio" )),
gr . update ( label = _i18n ( "section_transcriptions" )),
gr . update ( label = _i18n ( "section_synthesis" )),
2026-02-09 10:30:13 +08:00
]
lang_choice . change (
fn = _change_component_language ,
inputs = [ lang_choice ],
outputs = [
prompt_audio ,
target_audio ,
prompt_lyric_lang ,
target_lyric_lang ,
prompt_vocal_sep ,
target_vocal_sep ,
prompt_metadata ,
target_metadata ,
control_radio ,
auto_shift ,
pitch_shift ,
seed_input ,
synthesis_btn ,
output_audio ,
lang_choice ,
example_choice ,
instruction_md ,
2026-02-16 01:18:48 +08:00
accordion_input_audio ,
accordion_transcriptions ,
accordion_synthesis ,
2026-02-09 10:30:13 +08:00
],
)
# Upload new prompt/target audio → clear corresponding metadata; skip clear when change came from load example
prompt_audio . change (
fn = _clear_prompt_meta_unless_example ,
inputs = [ prompt_audio , skip_clear_metadata_count ],
outputs = [ prompt_metadata , skip_clear_metadata_count ],
)
2026-02-16 01:18:48 +08:00
prompt_audio . upload (
fn = _transcribe_prompt ,
inputs = [ prompt_audio , prompt_metadata , prompt_lyric_lang , prompt_vocal_sep ],
outputs = [ prompt_metadata , prompt_midi ],
)
prompt_midi . upload (
fn = _edit_metadata ,
inputs = [ prompt_metadata , prompt_midi , prompt_audio , prompt_lyric_lang ],
outputs = [ prompt_metadata ],
)
2026-02-09 10:30:13 +08:00
target_audio . change (
fn = _clear_target_meta_unless_example ,
inputs = [ target_audio , skip_clear_metadata_count ],
outputs = [ target_metadata , skip_clear_metadata_count ],
)
2026-02-16 01:18:48 +08:00
target_audio . upload (
fn = _transcribe_target ,
inputs = [ target_audio , target_metadata , target_lyric_lang , target_vocal_sep ],
outputs = [ target_metadata , target_midi , target_vocal ],
)
target_midi . upload (
fn = _edit_metadata ,
inputs = [ target_metadata , target_midi , target_vocal , target_lyric_lang ],
outputs = [ target_metadata ],
2026-02-09 10:30:13 +08:00
)
synthesis_btn . click (
2026-02-16 01:18:48 +08:00
fn = _run_synthesis ,
2026-02-09 10:30:13 +08:00
inputs = [
prompt_audio ,
prompt_metadata ,
target_metadata ,
control_radio ,
auto_shift ,
pitch_shift ,
seed_input ,
],
outputs = [ output_audio ],
)
return page
if __name__ == "__main__" :
import argparse
parser = argparse . ArgumentParser ()
parser . add_argument ( "--port" , type = int , default = 7860 , help = "Gradio server port" )
parser . add_argument ( "--share" , action = "store_true" , help = "Create public link" )
2026-03-16 13:40:30 +08:00
parser . add_argument ( "--fp16" , action = "store_true" , help = "Use FP16 for SVS model and inference" )
2026-02-09 10:30:13 +08:00
args = parser . parse_args ()
page = render_interface ()
page . queue ()
page . launch ( share = args . share , server_name = "0.0.0.0" , server_port = args . port )
