mei/SoulX-Singer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update webui: add MIDI support, improve the layout and processing flow

Browse Source
This commit is contained in:
jlqian98
2026-02-16 01:18:48 +08:00
parent 889c400dee
commit 0de99ae0ac
3 changed files with 684 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
preprocess/tools/midi_parser.py
+190 -204
View File
@@ -20,15 +20,16 @@ from .f0_extraction import F0Extractor
from .g2p import g2p_transform from .g2p import g2p_transform
# Audio and segmenting constants (used by _edit_data_to_meta) # Audio, MIDI and segmentation constants
SAMPLE_RATE = 44100 SAMPLE_RATE = 44100 # Hz, fixed for all audio processing in this script to ensure consistent timing with MIDI ticks.
DEFAULT_LANGUAGE = "Mandarin" MIDI_TICKS_PER_BEAT = 500
MAX_GAP_SEC = 5.0 # gap (sec) above which we start a new segment MIDI_TEMPO = 500000 # microseconds per beat (120 BPM)
MIDI_TIME_SIGNATURE = (4, 4)
MIDI_VELOCITY = 64
END_EXTENSION_SEC = 0.4 # extend each segment end by this much silence (sec) to give the model more context
MAX_GAP_SEC = 2.0 # gap (sec) above which we start a new segment
MAX_SEGMENT_DUR_SUM_SEC = 60.0 # max cumulative note duration per segment (sec) MAX_SEGMENT_DUR_SUM_SEC = 60.0 # max cumulative note duration per segment (sec)
MIN_GAP_THRESHOLD_SEC = 0.001 # ignore gaps smaller than this SILENCE_THRESHOLD_SEC = 0.2 # treat as separate <SP> if gap larger
LONG_SILENCE_THRESHOLD_SEC = 0.05 # treat as separate <SP> if gap larger
MAX_LEADING_SP_DUR_SEC = 2.0 # cap leading silence in a segment to this (sec)
DEFAULT_RMVPE_MODEL_PATH = "pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt"
@dataclass @dataclass
@@ -44,42 +45,6 @@ class Note:
def end_s(self) -> float: def end_s(self) -> float:
return self.start_s + self.note_dur return self.start_s + self.note_dur
def remove_duplicate_segments(meta_data: List[dict]) -> None:
"""Merge consecutive identical notes (same text, pitch, type) within each segment. Mutates meta_data in place."""
for idx, segment in enumerate(meta_data):
texts = segment["note_text"]
durs = segment["note_dur"]
pitches = segment["note_pitch"]
types = segment["note_type"]
new_texts = []
new_durs = []
new_pitches = []
new_types = []
for i in range(len(texts)):
if i == 0:
new_texts.append(texts[i])
new_durs.append(durs[i])
new_pitches.append(pitches[i])
new_types.append(types[i])
continue
t, d, p, ty = texts[i], durs[i], pitches[i], types[i]
if t == "<SP>" and texts[i - 1] == "<SP>":
new_durs[-1] += d
continue
if t == texts[i - 1] and p == pitches[i - 1] and ty == types[i - 1]:
new_durs[-1] += d
else:
new_texts.append(t)
new_durs.append(d)
new_pitches.append(p)
new_types.append(ty)
meta_data[idx]["note_text"] = new_texts
meta_data[idx]["note_dur"] = new_durs
meta_data[idx]["note_pitch"] = new_pitches
meta_data[idx]["note_type"] = new_types
def meta2notes(meta_path: str) -> List[Note]: def meta2notes(meta_path: str) -> List[Note]:
"""Parse SoulX-Singer metadata JSON into a flat list of Note (absolute start_s).""" """Parse SoulX-Singer metadata JSON into a flat list of Note (absolute start_s)."""
with open(meta_path, "r", encoding="utf-8") as f: with open(meta_path, "r", encoding="utf-8") as f:
@@ -92,7 +57,7 @@ def meta2notes(meta_path: str) -> List[Note]:
notes: List[Note] = [] notes: List[Note] = []
for seg in segments: for seg in segments:
offset_s = seg["time"][0] / 1000 offset_s = seg["time"][0] / 1000
words = [str(x).replace("<AP>", "<SP>") for i, x in enumerate(seg["text"].split())] words = [str(x).replace("<AP>", "<SP>") for x in seg["text"].split()]
word_durs = [float(x) for x in seg["duration"].split()] word_durs = [float(x) for x in seg["duration"].split()]
pitches = [int(x) for x in seg["note_pitch"].split()] pitches = [int(x) for x in seg["note_pitch"].split()]
types = [int(x) if words[i] != "<SP>" else 1 for i, x in enumerate(seg["note_type"].split())] types = [int(x) if words[i] != "<SP>" else 1 for i, x in enumerate(seg["note_type"].split())]
@@ -117,9 +82,10 @@ def meta2notes(meta_path: str) -> List[Note]:
def _append_segment_to_meta( def _append_segment_to_meta(
meta_path_str: str, meta_path_str: str,
cut_wavs_output_dir: str, cut_wavs_output_dir: str | None,
vocal_file: str, vocal_file: str | None,
audio_data: Any, language: str,
audio_data: Any | None,
meta_data: List[dict], meta_data: List[dict],
note_start: List[float], note_start: List[float],
note_end: List[float], note_end: List[float],
@@ -127,28 +93,40 @@ def _append_segment_to_meta(
note_pitch: List[Any], note_pitch: List[Any],
note_type: List[Any], note_type: List[Any],
note_dur: List[float], note_dur: List[float],
end_time_ms_override: float | None = None,
) -> None: ) -> None:
"""Write one segment wav and append one segment dict to meta_data. Caller clears note_* lists after.""" """Write one segment wav and append one segment dict to meta_data. Caller clears note_* lists after."""
if not all((note_start, note_end, note_text, note_pitch, note_type, note_dur)):
return
base_name = os.path.splitext(os.path.basename(meta_path_str))[0] base_name = os.path.splitext(os.path.basename(meta_path_str))[0]
item_name = f"{base_name}_{len(meta_data)}" item_name = f"{base_name}_{len(meta_data)}"
wav_fn = None
if cut_wavs_output_dir and vocal_file and audio_data is not None:
wav_fn = os.path.join(cut_wavs_output_dir, f"{item_name}.wav") wav_fn = os.path.join(cut_wavs_output_dir, f"{item_name}.wav")
start_ms = int(note_start[0] * 1000) end_pad = int(END_EXTENSION_SEC * SAMPLE_RATE)
end_ms = ( start_sample = max(0, int(note_start[0] * SAMPLE_RATE))
int(end_time_ms_override) end_sample = min(len(audio_data), int(note_end[-1] * SAMPLE_RATE) + end_pad)
if end_time_ms_override is not None
else int(note_end[-1] * 1000) end_pad_dur = (end_sample / SAMPLE_RATE - note_end[-1]) if end_sample > int(note_end[-1] * SAMPLE_RATE) else 0.0
) if end_pad_dur > 0:
start_sample = int(note_start[0] * SAMPLE_RATE) note_dur = note_dur + [end_pad_dur]
end_sample = int(note_end[-1] * SAMPLE_RATE) note_text = note_text + ["<SP>"]
note_pitch = note_pitch + [0]
note_type = note_type + [1]
start_ms = int(start_sample / SAMPLE_RATE * 1000)
end_ms = int(end_sample / SAMPLE_RATE * 1000)
write(wav_fn, audio_data[start_sample:end_sample], SAMPLE_RATE) write(wav_fn, audio_data[start_sample:end_sample], SAMPLE_RATE)
else:
start_ms = int(note_start[0] * 1000)
end_ms = int(note_end[-1] * 1000)
meta_data.append({ meta_data.append({
"item_name": item_name, "item_name": item_name,
"wav_fn": wav_fn, "wav_fn": wav_fn,
"origin_wav_fn": vocal_file, "origin_wav_fn": vocal_file,
"start_time_ms": start_ms, "start_time_ms": start_ms,
"end_time_ms": end_ms, "end_time_ms": end_ms,
"language": DEFAULT_LANGUAGE, "language": language,
"note_text": list(note_text), "note_text": list(note_text),
"note_pitch": list(note_pitch), "note_pitch": list(note_pitch),
"note_type": list(note_type), "note_type": list(note_type),
@@ -156,22 +134,25 @@ def _append_segment_to_meta(
}) })
def convert_meta(meta_data: List[dict], rmvpe_model_path, device="cuda"): def convert_meta(meta_data: List[dict], pitch_extractor: F0Extractor | None) -> List[dict]:
pitch_extractor = F0Extractor(rmvpe_model_path, device=device, verbose=False)
converted_data = [] converted_data = []
for item in meta_data: for item in meta_data:
language = item.get("language", "Mandarin")
wav_fn = item.get("wav_fn") wav_fn = item.get("wav_fn")
if pitch_extractor is not None:
if not wav_fn or not os.path.isfile(wav_fn): if not wav_fn or not os.path.isfile(wav_fn):
raise FileNotFoundError(f"Segment wav file not found: {wav_fn}") raise FileNotFoundError(f"Segment wav file not found: {wav_fn}")
f0 = pitch_extractor.process(wav_fn) f0 = pitch_extractor.process(wav_fn)
else:
f0 = []
converted_item = { converted_item = {
"index": item.get("item_name"), "index": item.get("item_name"),
"language": item.get("language"), "language": language,
"time": [item.get("start_time_ms", 0), item.get("end_time_ms", sum(item["note_dur"]) * 1000)], "time": [item.get("start_time_ms", 0), item.get("end_time_ms", sum(item["note_dur"]) * 1000)],
"duration": " ".join(str(round(x, 2)) for x in item.get("note_dur", [])), "duration": " ".join(str(round(x, 2)) for x in item.get("note_dur", [])),
"text": " ".join(item.get("note_text", [])), "text": " ".join(item.get("note_text", [])),
"phoneme": " ".join(g2p_transform(item.get("note_text", []), DEFAULT_LANGUAGE)), "phoneme": " ".join(g2p_transform(item.get("note_text", []), language)),
"note_pitch": " ".join(str(x) for x in item.get("note_pitch", [])), "note_pitch": " ".join(str(x) for x in item.get("note_pitch", [])),
"note_type": " ".join(str(x) for x in item.get("note_type", [])), "note_type": " ".join(str(x) for x in item.get("note_type", [])),
"f0": " ".join(str(round(float(x), 1)) for x in f0), "f0": " ".join(str(round(float(x), 1)) for x in f0),
@@ -184,12 +165,14 @@ def convert_meta(meta_data: List[dict], rmvpe_model_path, device="cuda"):
def _edit_data_to_meta( def _edit_data_to_meta(
meta_path_str: str, meta_path_str: str,
edit_data: List[dict], edit_data: List[dict],
vocal_file: str, vocal_file: str | None,
rmvpe_model_path: str | None = None, language: str,
device: str = "cuda", pitch_extractor: F0Extractor | None,
) -> None: ) -> None:
"""Write SoulX-Singer metadata JSON from edit_data (list of {start, end, note_text, note_pitch, note_type}).""" """Write SoulX-Singer metadata JSON from edit_data (list of {start, end, note_text, note_pitch, note_type})."""
# Use a fixed temporary directory for cut wavs # Store temporary cut wavs beside the source vocal (same folder, fixed subdir name).
cut_wavs_output_dir = None
if vocal_file:
cut_wavs_output_dir = os.path.join(os.path.dirname(vocal_file), "cut_wavs_tmp") cut_wavs_output_dir = os.path.join(os.path.dirname(vocal_file), "cut_wavs_tmp")
os.makedirs(cut_wavs_output_dir, exist_ok=True) os.makedirs(cut_wavs_output_dir, exist_ok=True)
@@ -199,11 +182,36 @@ def _edit_data_to_meta(
note_dur: List[float] = [] note_dur: List[float] = []
note_start: List[float] = [] note_start: List[float] = []
note_end: List[float] = [] note_end: List[float] = []
prev_end = 0.0
meta_data: List[dict] = [] meta_data: List[dict] = []
audio_data = None
if vocal_file:
audio_data, _ = librosa.load(vocal_file, sr=SAMPLE_RATE, mono=True) audio_data, _ = librosa.load(vocal_file, sr=SAMPLE_RATE, mono=True)
dur_sum = 0.0 dur_sum = 0.0
def flush_current_segment() -> None:
nonlocal dur_sum
_append_segment_to_meta(
meta_path_str,
cut_wavs_output_dir,
vocal_file,
language,
audio_data,
meta_data,
note_start,
note_end,
note_text,
note_pitch,
note_type,
note_dur,
)
note_text.clear()
note_pitch.clear()
note_type.clear()
note_dur.clear()
note_start.clear()
note_end.clear()
dur_sum = 0.0
for entry in edit_data: for entry in edit_data:
start = float(entry["start"]) start = float(entry["start"])
end = float(entry["end"]) end = float(entry["end"])
@@ -218,85 +226,27 @@ def _edit_data_to_meta(
note_dur.append(end - start) note_dur.append(end - start)
note_start.append(start) note_start.append(start)
note_end.append(end) note_end.append(end)
prev_end = end
dur_sum += end - start dur_sum += end - start
continue continue
if ( if (
len(note_text) > 0 len(note_text) > 0
and note_text[-1] == "<SP>" and note_text[-1] == "<SP>"
and note_dur[-1] > MAX_LEADING_SP_DUR_SEC and note_dur[-1] > MAX_GAP_SEC
): ):
cut_time = note_dur[-1] - MAX_LEADING_SP_DUR_SEC note_text.pop()
note_dur[-1] = MAX_LEADING_SP_DUR_SEC note_pitch.pop()
end_ms_override = note_end[-1] * 1000 - cut_time * 1000 note_type.pop()
_append_segment_to_meta( note_dur.pop()
meta_path_str, note_start.pop()
cut_wavs_output_dir, note_end.pop()
vocal_file,
audio_data,
meta_data,
note_start,
note_end,
note_text,
note_pitch,
note_type,
note_dur,
end_time_ms_override=end_ms_override,
)
note_text = []
note_pitch = []
note_type = []
note_dur = []
note_start = []
note_end = []
prev_end = start
dur_sum = 0.0
gap_from_prev = start - prev_end dur_sum = sum(note_dur)
gap_from_last_note = (start - note_end[-1]) if note_end else 0.0 flush_current_segment()
if (
gap_from_prev >= MAX_GAP_SEC
or gap_from_last_note >= MAX_GAP_SEC
or dur_sum >= MAX_SEGMENT_DUR_SUM_SEC
):
if len(note_text) > 0:
_append_segment_to_meta(
meta_path_str,
cut_wavs_output_dir,
vocal_file,
audio_data,
meta_data,
note_start,
note_end,
note_text,
note_pitch,
note_type,
note_dur,
)
note_text = []
note_pitch = []
note_type = []
note_dur = []
note_start = []
note_end = []
prev_end = start
dur_sum = 0.0
if start - prev_end > MIN_GAP_THRESHOLD_SEC: if dur_sum + (end - start) > MAX_SEGMENT_DUR_SUM_SEC and len(note_text) > 0:
if start - prev_end > LONG_SILENCE_THRESHOLD_SEC or len(note_text) == 0: flush_current_segment()
note_text.append("<SP>")
note_pitch.append(0)
note_type.append(1)
note_dur.append(start - prev_end)
note_start.append(prev_end)
note_end.append(start)
else:
if len(note_dur) > 0:
note_dur[-1] += start - prev_end
note_end[-1] = start
prev_end = end
note_text.append(text) note_text.append(text)
note_pitch.append(int(pitch)) note_pitch.append(int(pitch))
note_type.append(int(type_)) note_type.append(int(type_))
@@ -305,30 +255,37 @@ def _edit_data_to_meta(
note_end.append(end) note_end.append(end)
dur_sum += end - start dur_sum += end - start
if len(note_text) > 0: if note_text:
_append_segment_to_meta( flush_current_segment()
meta_path_str,
cut_wavs_output_dir,
vocal_file,
audio_data,
meta_data,
note_start,
note_end,
note_text,
note_pitch,
note_type,
note_dur,
)
remove_duplicate_segments(meta_data) # Merge only consecutive <SP> tokens to reduce fragmentation in silence regions.
for segment in meta_data:
phoneme = segment['note_text']
duration = segment['note_dur']
note_pitch = segment['note_pitch']
note_type = segment['note_type']
_rmvpe_path = rmvpe_model_path or DEFAULT_RMVPE_MODEL_PATH merged_items: List[Tuple[str, float, int, int]] = []
converted_data = convert_meta(meta_data, _rmvpe_path, device) prev_item = None
for text, dur, pitch, note_type in zip(phoneme, duration, note_pitch, note_type):
if prev_item and text == "<SP>" and prev_item[0] == "<SP>":
merged_items[-1] = (prev_item[0], prev_item[1] + dur, prev_item[2], prev_item[3])
else:
merged_items.append((text, dur, pitch, note_type))
prev_item = merged_items[-1]
segment['note_text'] = [item[0] for item in merged_items]
segment['note_dur'] = [item[1] for item in merged_items]
segment['note_pitch'] = [item[2] for item in merged_items]
segment['note_type'] = [item[3] for item in merged_items]
converted_data = convert_meta(meta_data, pitch_extractor)
with open(meta_path_str, "w", encoding="utf-8") as f: with open(meta_path_str, "w", encoding="utf-8") as f:
json.dump(converted_data, f, ensure_ascii=False, indent=2) json.dump(converted_data, f, ensure_ascii=False, indent=2)
# Clean up temporary cut wavs directory # Clean up temporary cut wavs directory
if cut_wavs_output_dir:
try: try:
shutil.rmtree(cut_wavs_output_dir, ignore_errors=True) shutil.rmtree(cut_wavs_output_dir, ignore_errors=True)
except Exception: except Exception:
@@ -338,9 +295,9 @@ def _edit_data_to_meta(
def notes2meta( def notes2meta(
notes: List[Note], notes: List[Note],
meta_path: str, meta_path: str,
vocal_file: str, vocal_file: str | None,
rmvpe_model_path: str | None = None, language: str,
device: str = "cuda", pitch_extractor: F0Extractor | None,
) -> None: ) -> None:
"""Write SoulX-Singer metadata JSON from a list of Note (segmenting + wav cuts).""" """Write SoulX-Singer metadata JSON from a list of Note (segmenting + wav cuts)."""
edit_data = [ edit_data = [
@@ -357,30 +314,21 @@ def notes2meta(
str(meta_path), str(meta_path),
edit_data, edit_data,
vocal_file, vocal_file,
rmvpe_model_path=rmvpe_model_path, language,
device=device, pitch_extractor=pitch_extractor,
) )
@dataclass(frozen=True)
class MidiDefaults:
ticks_per_beat: int = 500
tempo: int = 500000 # microseconds per beat (120 BPM)
time_signature: Tuple[int, int] = (4, 4)
velocity: int = 64
def _seconds_to_ticks(seconds: float, ticks_per_beat: int, tempo: int) -> int: def _seconds_to_ticks(seconds: float, ticks_per_beat: int, tempo: int) -> int:
# ticks = seconds * (ticks_per_beat beats) / (tempo microseconds per beat)
return int(round(seconds * ticks_per_beat * 1_000_000 / tempo)) return int(round(seconds * ticks_per_beat * 1_000_000 / tempo))
def notes2midi( def notes2midi(
notes: List[Note], notes: List[Note],
midi_path: str, midi_path: str,
defaults: MidiDefaults | None = None,
) -> None: ) -> None:
"""Write MIDI file from a list of Note.""" """Write MIDI file from a list of Note."""
defaults = defaults or MidiDefaults()
if not notes: if not notes:
raise ValueError("Empty note list.") raise ValueError("Empty note list.")
@@ -392,15 +340,16 @@ def notes2midi(
continue continue
start_ticks = _seconds_to_ticks( start_ticks = _seconds_to_ticks(
start_s, defaults.ticks_per_beat, defaults.tempo start_s, MIDI_TICKS_PER_BEAT, MIDI_TEMPO
) )
end_ticks = _seconds_to_ticks( end_ticks = _seconds_to_ticks(
end_s, defaults.ticks_per_beat, defaults.tempo end_s, MIDI_TICKS_PER_BEAT, MIDI_TEMPO
) )
if end_ticks <= start_ticks: if end_ticks <= start_ticks:
end_ticks = start_ticks + 1 end_ticks = start_ticks + 1
lyric = n.note_text lyric = n.note_text
# Some DAWs store lyric text as latin1-compatible bytes; keep best-effort round-trip.
try: try:
lyric = lyric.encode("utf-8").decode("latin1") lyric = lyric.encode("utf-8").decode("latin1")
except (UnicodeEncodeError, UnicodeDecodeError): except (UnicodeEncodeError, UnicodeDecodeError):
@@ -418,7 +367,7 @@ def notes2midi(
mido.Message( mido.Message(
"note_on", "note_on",
note=n.note_pitch, note=n.note_pitch,
velocity=defaults.velocity, velocity=MIDI_VELOCITY,
time=0, time=0,
), ),
) )
@@ -431,18 +380,19 @@ def notes2midi(
) )
) )
# Keep deterministic ordering at same tick: note_off -> lyric -> note_on.
events.sort(key=lambda x: (x[0], x[1])) events.sort(key=lambda x: (x[0], x[1]))
mid = mido.MidiFile(ticks_per_beat=defaults.ticks_per_beat) mid = mido.MidiFile(ticks_per_beat=MIDI_TICKS_PER_BEAT)
track = mido.MidiTrack() track = mido.MidiTrack()
mid.tracks.append(track) mid.tracks.append(track)
track.append(mido.MetaMessage("set_tempo", tempo=defaults.tempo, time=0)) track.append(mido.MetaMessage("set_tempo", tempo=MIDI_TEMPO, time=0))
track.append( track.append(
mido.MetaMessage( mido.MetaMessage(
"time_signature", "time_signature",
numerator=defaults.time_signature[0], numerator=MIDI_TIME_SIGNATURE[0],
denominator=defaults.time_signature[1], denominator=MIDI_TIME_SIGNATURE[1],
time=0, time=0,
) )
) )
@@ -458,7 +408,10 @@ def notes2midi(
def midi2notes(midi_path: str) -> List[Note]: def midi2notes(midi_path: str) -> List[Note]:
"""Parse MIDI file into a list of Note. Merges all tracks; tempo from last set_tempo event.""" """Parse MIDI file into a list of Note.
Merges all tracks and uses the latest encountered set_tempo as global tempo.
"""
mid = mido.MidiFile(midi_path) mid = mido.MidiFile(midi_path)
ticks_per_beat = mid.ticks_per_beat ticks_per_beat = mid.ticks_per_beat
tempo = 500000 tempo = 500000
@@ -520,6 +473,7 @@ def midi2notes(midi_path: str) -> List[Note]:
lyrics.sort(key=lambda x: x[0]) lyrics.sort(key=lambda x: x[0])
trimmed = [] trimmed = []
# Remove/trim overlaps so generated notes are strictly non-overlapping in tick domain.
for note in raw_notes: for note in raw_notes:
while trimmed: while trimmed:
prev = trimmed[-1] prev = trimmed[-1]
@@ -534,6 +488,7 @@ def midi2notes(midi_path: str) -> List[Note]:
raw_notes = trimmed raw_notes = trimmed
tolerance = ticks_per_beat // 100 tolerance = ticks_per_beat // 100
# Attach lyrics near note_on positions with a small tick tolerance.
lyric_idx = 0 lyric_idx = 0
for note in raw_notes: for note in raw_notes:
while lyric_idx < len(lyrics) and lyrics[lyric_idx][0] < note["start_ticks"] - tolerance: while lyric_idx < len(lyrics) and lyrics[lyric_idx][0] < note["start_ticks"] - tolerance:
@@ -559,6 +514,7 @@ def midi2notes(midi_path: str) -> List[Note]:
continue continue
lyric = n.get("lyric", "") lyric = n.get("lyric", "")
# SoulX-Singer convention mapping from lyric token to note_type/text.
if not lyric: if not lyric:
tp = 2 tp = 2
text = "啦" text = "啦"
@@ -572,6 +528,21 @@ def midi2notes(midi_path: str) -> List[Note]:
tp = 2 tp = 2
text = lyric text = lyric
if start_s - prev_end_s > SILENCE_THRESHOLD_SEC:
# Explicitly represent long gaps as <SP> notes.
result.append(
Note(
start_s=prev_end_s,
note_dur=start_s - prev_end_s,
note_text="<SP>",
note_pitch=0,
note_type=1,
)
)
else:
if len(result) > 0:
result[-1].note_dur = start_s - result[-1].start_s
result.append( result.append(
Note( Note(
start_s=start_s, start_s=start_s,
@@ -586,35 +557,51 @@ def midi2notes(midi_path: str) -> List[Note]:
return result return result
def meta2midi(meta_path: str, midi_path: str, defaults: MidiDefaults | None = None) -> None: class MidiParser:
"""Convert SoulX-Singer metadata JSON to MIDI file (meta -> List[Note] -> midi).""" def __init__(
notes = meta2notes(meta_path) self,
notes2midi(notes, midi_path, defaults) rmvpe_model_path: str,
print(f"Saved MIDI to {midi_path}")
def midi2meta(
midi_path: str,
meta_path: str,
vocal_file: str,
rmvpe_model_path: str | None = None,
device: str = "cuda", device: str = "cuda",
) -> None: ) -> None:
"""Convert MIDI file to SoulX-Singer metadata JSON (midi -> List[Note] -> meta).""" self.rmvpe_model_path = rmvpe_model_path
self.device = device
self.pitch_extractor: F0Extractor | None = None
def _get_pitch_extractor(self) -> F0Extractor:
if self.pitch_extractor is None:
self.pitch_extractor = F0Extractor(
self.rmvpe_model_path,
device=self.device,
verbose=False,
)
return self.pitch_extractor
def midi2meta(
self,
midi_path: str,
meta_path: str,
vocal_file: str | None = None,
language: str = "Mandarin",
) -> None:
meta_dir = os.path.dirname(meta_path) meta_dir = os.path.dirname(meta_path)
if meta_dir: if meta_dir:
os.makedirs(meta_dir, exist_ok=True) os.makedirs(meta_dir, exist_ok=True)
# cut_wavs will be written to a fixed temporary directory inside _edit_data_to_meta
notes = midi2notes(midi_path) notes = midi2notes(midi_path)
pitch_extractor = self._get_pitch_extractor() if vocal_file else None
notes2meta( notes2meta(
notes, notes,
meta_path, meta_path,
vocal_file, vocal_file,
rmvpe_model_path=rmvpe_model_path, language,
device=device, pitch_extractor=pitch_extractor,
) )
print(f"Saved Meta to {meta_path}") print(f"Saved Meta to {meta_path}")
def meta2midi(self, meta_path: str, midi_path: str) -> None:
notes = meta2notes(meta_path)
notes2midi(notes, midi_path)
print(f"Saved MIDI to {midi_path}")
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
@@ -624,7 +611,8 @@ if __name__ == "__main__":
) )
parser.add_argument("--meta", type=str, help="Path to metadata JSON") parser.add_argument("--meta", type=str, help="Path to metadata JSON")
parser.add_argument("--midi", type=str, help="Path to MIDI file") parser.add_argument("--midi", type=str, help="Path to MIDI file")
parser.add_argument("--vocal", type=str, help="Path to vocal wav (for midi2meta)") parser.add_argument("--vocal", type=str, default=None, help="Path to vocal wav (optional for midi2meta)")
parser.add_argument("--language", type=str, default="Mandarin", help="Lyric language for metadata phoneme conversion (default: Mandarin)")
parser.add_argument( parser.add_argument(
"--meta2midi", "--meta2midi",
action="store_true", action="store_true",
@@ -633,7 +621,7 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--midi2meta", "--midi2meta",
action="store_true", action="store_true",
help="Convert midi -> meta (requires --midi, --meta, --vocal, --cut_wavs_dir)", help="Convert midi -> meta (requires --midi and --meta; --vocal is optional)",
) )
parser.add_argument( parser.add_argument(
"--rmvpe_model_path", "--rmvpe_model_path",
@@ -648,22 +636,20 @@ if __name__ == "__main__":
default="cuda", default="cuda",
) )
args = parser.parse_args() args = parser.parse_args()
midi_parser = MidiParser(
rmvpe_model_path=args.rmvpe_model_path,
device=args.device,
)
if args.meta2midi: if args.meta2midi:
if not args.meta or not args.midi: if not args.meta or not args.midi:
parser.error("--meta2midi requires --meta and --midi") parser.error("--meta2midi requires --meta and --midi")
meta2midi(args.meta, args.midi) midi_parser.meta2midi(args.meta, args.midi)
elif args.midi2meta: elif args.midi2meta:
if not args.midi or not args.meta or not args.vocal: if not args.midi or not args.meta:
parser.error( parser.error(
"--midi2meta requires --midi, --meta, --vocal" "--midi2meta requires --midi and --meta"
)
midi2meta(
args.midi,
args.meta,
args.vocal,
rmvpe_model_path=args.rmvpe_model_path,
device=args.device,
) )
midi_parser.midi2meta(args.midi, args.meta, args.vocal, args.language)
else: else:
parser.print_help() parser.print_help()
soulxsinger/utils/data_processor.py
+4 -5
View File
@@ -42,7 +42,7 @@ class DataProcessor:
merged_items = [] merged_items = []
duration = [float(x) for x in meta["duration"].split()] duration = [float(x) for x in meta["duration"].split()]
phoneme = [str(x).replace("<AP>", "<SP>") for i, x in enumerate(meta["phoneme"].split())] phoneme = [str(x).replace("<AP>", "<SP>") for x in meta["phoneme"].split()]
note_pitch = [int(x) for x in meta["note_pitch"].split()] note_pitch = [int(x) for x in meta["note_pitch"].split()]
note_type = [int(x) if phoneme[i] != "<SP>" else 1 for i, x in enumerate(meta["note_type"].split())] note_type = [int(x) if phoneme[i] != "<SP>" else 1 for i, x in enumerate(meta["note_type"].split())]
@@ -52,7 +52,6 @@ class DataProcessor:
else: else:
merged_items.append([phoneme[i], duration[i], note_pitch[i], note_type[i]]) merged_items.append([phoneme[i], duration[i], note_pitch[i], note_type[i]])
single_frame_duration = self.hop_size / self.sample_rate
meta['phoneme'] = [x[0] for x in merged_items] meta['phoneme'] = [x[0] for x in merged_items]
meta['duration'] = [x[1] for x in merged_items] meta['duration'] = [x[1] for x in merged_items]
meta['note_pitch'] = [x[2] for x in merged_items] meta['note_pitch'] = [x[2] for x in merged_items]
@@ -150,9 +149,9 @@ class DataProcessor:
meta["note_type"], meta["note_type"],
) )
f0 = torch.tensor([float(x) for x in meta["f0"].split()]) f0 = [float(x) for x in meta.get("f0", "").split()]
min_frame = min(item["mel2note"].shape[1], f0.shape[0]) min_frame = min(item["mel2note"].shape[1], len(f0)) if len(f0) > 0 else item["mel2note"].shape[1]
item['f0'] = f0[:min_frame].unsqueeze(0).float().to(self.device) item['f0'] = torch.tensor(f0)[:min_frame].unsqueeze(0).float().to(self.device) if len(f0) > 0 else None
item["mel2note"] = item["mel2note"][:, :min_frame] item["mel2note"] = item["mel2note"][:, :min_frame]
if wav_path is not None: if wav_path is not None:
webui.py
+403 -313
View File
@@ -1,9 +1,9 @@
import os import os
import re
import random import random
import shutil import shutil
import sys import sys
import traceback import traceback
from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Literal, Tuple from typing import Literal, Tuple
@@ -14,11 +14,17 @@ import soundfile as sf
import gradio as gr import gradio as gr
from preprocess.pipeline import PreprocessPipeline from preprocess.pipeline import PreprocessPipeline
from preprocess.tools.midi_parser import MidiParser
from soulxsinger.utils.file_utils import load_config from soulxsinger.utils.file_utils import load_config
from cli.inference import build_model as build_svs_model, process as svs_process from cli.inference import build_model as build_svs_model, process as svs_process
ROOT = Path(__file__).parent ROOT = Path(__file__).parent
SAMPLE_RATE = 44100
PROMPT_MAX_SEC_DEFAULT = 30
TARGET_MAX_SEC_DEFAULT = 60
PROMPT_MAX_MERGE_DURATION_MS = 30000
TARGET_MAX_MERGE_DURATION_MS = 60000
ENGLISH_EXAMPLE_PROMPT_AUDIO = "example/audio/en_prompt.mp3" ENGLISH_EXAMPLE_PROMPT_AUDIO = "example/audio/en_prompt.mp3"
ENGLISH_EXAMPLE_PROMPT_META = "example/audio/en_prompt.json" ENGLISH_EXAMPLE_PROMPT_META = "example/audio/en_prompt.json"
@@ -38,8 +44,6 @@ CANTONESE_EXAMPLE_TARGET_META = "example/audio/yue_target.json"
MUSIC_EXAMPLE_TARGET_AUDIO = "example/audio/music.mp3" MUSIC_EXAMPLE_TARGET_AUDIO = "example/audio/music.mp3"
MUSIC_EXAMPLE_TARGET_META = "example/audio/music.json" MUSIC_EXAMPLE_TARGET_META = "example/audio/music.json"
# Lyric language: value (Mandarin/Cantonese/English) is passed to PreprocessPipeline; display labels from i18n via get_lyric_lang_choices()
# Use absolute paths so Examples load correctly (including File components for metadata) # Use absolute paths so Examples load correctly (including File components for metadata)
EXAMPLES_LIST = [ EXAMPLES_LIST = [
[ [
@@ -49,10 +53,10 @@ EXAMPLES_LIST = [
str(ROOT / MANDARIN_EXAMPLE_TARGET_META), str(ROOT / MANDARIN_EXAMPLE_TARGET_META),
"Mandarin", "Mandarin",
"Mandarin", "Mandarin",
"melody", "melody-controlled",
False, "no",
True, "yes",
True, "yes",
0, 0,
], ],
[ [
@@ -62,10 +66,10 @@ EXAMPLES_LIST = [
str(ROOT / CANTONESE_EXAMPLE_TARGET_META), str(ROOT / CANTONESE_EXAMPLE_TARGET_META),
"Mandarin", "Mandarin",
"Cantonese", "Cantonese",
"melody", "melody-controlled",
False, "no",
True, "yes",
True, "yes",
0, 0,
], ],
[ [
@@ -75,10 +79,10 @@ EXAMPLES_LIST = [
str(ROOT / ENGLISH_EXAMPLE_TARGET_META), str(ROOT / ENGLISH_EXAMPLE_TARGET_META),
"Mandarin", "Mandarin",
"English", "English",
"melody", "melody-controlled",
False, "no",
True, "yes",
True, "yes",
0, 0,
], ],
[ [
@@ -88,37 +92,107 @@ EXAMPLES_LIST = [
str(ROOT / MUSIC_EXAMPLE_TARGET_META), str(ROOT / MUSIC_EXAMPLE_TARGET_META),
"Mandarin", "Mandarin",
"Mandarin", "Mandarin",
"melody", "score-controlled",
False, "no",
True, "yes",
True, "yes",
0, 0,
], ],
] ]
# i18n
_I18N_KEY2LANG = dict(
display_lang_label=dict(en="Display Language", zh="显示语言"),
section_input_audio=dict(en="Input Audio", zh="输入音频"),
section_transcriptions=dict(en="Transcriptions & Metadata", zh="转录与元数据"),
section_synthesis=dict(en="Singing Synthesis", zh="歌声合成"),
seed_label=dict(en="Seed", zh="种子"),
prompt_audio_label=dict(en=f"Prompt audio (reference voice), limit to {PROMPT_MAX_SEC_DEFAULT} seconds", zh=f"Prompt 音频（参考音色），限制在 {PROMPT_MAX_SEC_DEFAULT} 秒以内"),
target_audio_label=dict(en=f"Target audio (melody / lyrics source), limit to {TARGET_MAX_SEC_DEFAULT} seconds", zh=f"Target 音频（旋律/歌词来源），限制在 {TARGET_MAX_SEC_DEFAULT} 秒以内"),
transcription_btn_label=dict(en="Run singing transcription", zh="开始歌声转录"),
synthesis_btn_label=dict(en="🎤Generate singing voice", zh="🎤歌声合成"),
prompt_meta_label=dict(en="Prompt metadata", zh="Prompt 元数据"),
prompt_midi_label=dict(en="Prompt MIDI", zh="Prompt MIDI"),
target_meta_label=dict(en="Target metadata", zh="Target 元数据"),
target_midi_label=dict(en="Target MIDI", zh="Target MIDI"),
prompt_wav_label=dict(en="Prompt WAV (reference)", zh="Prompt WAV（参考音色）"),
generated_audio_label=dict(en="Generated merged audio", zh="合成结果音频"),
prompt_lyric_lang_label=dict(en="Prompt lyric language", zh="Prompt 歌词语种"),
target_lyric_lang_label=dict(en="Target lyric language", zh="Target 歌词语种"),
lyric_lang_mandarin=dict(en="Mandarin", zh="普通话"),
lyric_lang_cantonese=dict(en="Cantonese", zh="粤语"),
lyric_lang_english=dict(en="English", zh="英语"),
warn_missing_synthesis=dict(
en="Please provide prompt WAV, prompt metadata, and target metadata. Check the content in Transcriptions & Metadata above.",
zh="请提供 Prompt WAV、Prompt metadata 与 Target metadata，并检查上方 Transcriptions & Metadata 里的内容。",
),
prompt_vocal_sep_label=dict(en="Prompt vocal separation", zh="Prompt人声分离"),
target_vocal_sep_label=dict(en="Target vocal separation", zh="Target人声分离"),
option_yes=dict(en="yes", zh="是"),
option_no=dict(en="no", zh="否"),
auto_shift_label=dict(en="Auto pitch shift", zh="自动变调"),
pitch_shift_label=dict(en="Pitch shift (semitones)", zh="指定变调（半音）"),
control_type_label=dict(en="Control type", zh="控制类型"),
control_melody=dict(en="melody-controlled", zh="旋律控制"),
control_score=dict(en="score-controlled", zh="乐谱控制"),
examples_label=dict(en="Reference examples (click to load)", zh="参考样例（点击加载）"),
example_choice_0=dict(en="—", zh="—"),
example_choice_1=dict(en="Example 1: Mandarin → Mandarin (melody), Start singing synthesis!", zh="样例 1: 普通话 → 普通话 (melody), 开始歌声合成吧!"),
example_choice_2=dict(en="Example 2: Mandarin → Cantonese (melody), Start singing synthesis!", zh="样例 2: 普通话 → 粤语 (melody), 开始歌声合成吧!"),
example_choice_3=dict(en="Example 3: Mandarin → English (melody), Start singing synthesis!", zh="样例 3: 普通话 → 英语 (melody), 开始歌声合成吧!"),
example_choice_4=dict(en="Example 4: Mandarin → Music (score), Start singing synthesis!", zh="样例 4: 普通话 → 音乐 (score), 开始歌声合成吧!"),
instruction_title=dict(en="Usage", zh="使用说明"),
instruction_p1=dict(
en="Upload prompt and target audio, and the corresponding metadata and MIDI files will be automatically transcribed.",
zh="上传 Prompt 与 Target 音频，将自动转录生成 Prompt 与 Target 两份 metadata 文件以及对应的 MIDI 文件。",
),
instruction_p2=dict(
en="Auto-transcribed lyrics and notes are often misaligned, which may lead to suboptimal synthesis results. For best results, import the generated MIDI into the [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) for manual adjustment. After adjustment, re-upload the MIDI file and the metadata will be automatically updated.",
zh="自动转录的歌词与音高对齐效果通常不理想，可能导致合成效果不佳，建议将生成的 MIDI 文件导入 [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) 进行手动调整，调整后的 MIDI 文件重新上传后，metadata 将会自动更新。",
),
instruction_p3=dict(
en="Once prompt audio, prompt metadata, and target metadata are all set, click **🎤Generate singing voice** to run the singing synthesis and generate the final merged audio.",
zh="Prompt audio, Prompt metadata 和 Target metadata 都准备好后，点击「🎤歌声合成」开始最终生成。",
),
)
_GLOBAL_LANG: Literal["zh", "en"] = "zh"
def _i18n(key: str) -> str:
return _I18N_KEY2LANG[key][_GLOBAL_LANG]
def _load_example(choice_value): def _load_example(choice_value):
"""Return 11 example values + skip_clear_count (2 when loading example so next 2 audio.change events don't clear metadata). """Return 11 example values + skip_clear_count.
choice_value: selected dropdown string (or index in older flow); map to example index 0/1/2."""
When loading an example, the next two audio.change events should not clear metadata.
"""
output_count = 11
if choice_value is None: if choice_value is None:
return [gr.update()] * 11 + [0] return [gr.update()] * output_count + [0]
choice_to_index = {
_i18n("example_choice_1"): 1,
_i18n("example_choice_2"): 2,
_i18n("example_choice_3"): 3,
_i18n("example_choice_4"): 4,
}
idx = 0 idx = 0
if isinstance(choice_value, int): if isinstance(choice_value, int):
idx = 0 if choice_value <= 0 else min(choice_value - 1, len(EXAMPLES_LIST) - 1) idx = 0 if choice_value <= 0 else min(choice_value - 1, len(EXAMPLES_LIST) - 1)
else: else:
if choice_value == i18n("example_choice_1"): idx = choice_to_index.get(choice_value, 0)
idx = 1
elif choice_value == i18n("example_choice_2"):
idx = 2
elif choice_value == i18n("example_choice_3"):
idx = 3
elif choice_value == i18n("example_choice_4"):
idx = 4
if idx <= 0: if idx <= 0:
return [gr.update()] * 11 + [0] return [gr.update()] * output_count + [0]
list_idx = idx - 1 list_idx = idx - 1
if list_idx >= len(EXAMPLES_LIST): if list_idx >= len(EXAMPLES_LIST):
return [gr.update()] * 11 + [0] return [gr.update()] * output_count + [0]
row = EXAMPLES_LIST[list_idx] row = EXAMPLES_LIST[list_idx]
return [ return [
row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10],
@@ -143,11 +217,58 @@ def _get_device() -> str:
return "cuda:0" if torch.cuda.is_available() else "cpu" return "cuda:0" if torch.cuda.is_available() else "cpu"
def _session_dir_from_target(target_audio_path: str) -> Path: def _session_dir() -> Path:
stem = Path(target_audio_path).stem # Use per-call timestamped session dir to avoid cross-request collisions.
safe = re.sub(r"[^\w\-]", "_", stem) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
safe = re.sub(r"_+", "_", safe).strip("_") or "session" return ROOT / "outputs" / "gradio" / timestamp
return ROOT / "outputs" / "gradio" / safe[:64]
def _print_exception(context: str) -> None:
print(f"[{context}]\n{traceback.format_exc()}", file=sys.stderr, flush=True)
def _get_lyric_lang_choices():
"""Lyric language dropdown (display, value) for current UI language."""
return [
(_i18n("lyric_lang_mandarin"), "Mandarin"),
(_i18n("lyric_lang_cantonese"), "Cantonese"),
(_i18n("lyric_lang_english"), "English"),
]
def _resolve_file_path(x):
"""Gradio file input can be path string or (path, None) tuple."""
if x is None:
return None
if isinstance(x, tuple):
x = x[0]
return x if (x and os.path.isfile(x)) else None
def _normalize_audio_input(audio):
"""Normalize Gradio audio input to a filepath string."""
return audio[0] if isinstance(audio, tuple) else audio
def _trim_and_save_audio(src_audio_path: str, dst_wav_path: Path, max_sec: int, sr: int = SAMPLE_RATE) -> None:
"""Load audio as mono, trim to max_sec, and save as wav for preprocess."""
audio_data, _ = librosa.load(src_audio_path, sr=sr, mono=True)
audio_data = audio_data[: max_sec * sr]
sf.write(dst_wav_path, audio_data, sr)
def _yes_no_to_bool(value, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if value is None:
return default
return str(value).strip().lower() == "yes"
def _control_to_internal(control: str) -> str:
if control in ("melody", "melody-controlled"):
return "melody"
return "score"
class AppState: class AppState:
@@ -168,33 +289,28 @@ class AppState:
device=self.device, device=self.device,
) )
self.phoneset_path = "soulxsinger/utils/phoneme/phone_set.json" self.phoneset_path = "soulxsinger/utils/phoneme/phone_set.json"
self.midi_parser = MidiParser(
rmvpe_model_path="pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt",
device=self.device
)
def run_preprocess( def run_preprocess(
self, self,
prompt_path: Path, audio_path: Path,
target_path: Path, save_path: Path,
session_base: Path, vocal_sep: bool,
prompt_vocal_sep: bool, lyric_lang: str,
target_vocal_sep: bool, max_merge_duration: int
prompt_lyric_lang: str,
target_lyric_lang: str,
) -> Tuple[bool, str]: ) -> Tuple[bool, str]:
try: try:
self.preprocess_pipeline.save_dir = str(session_base / "transcriptions" / "prompt") self.preprocess_pipeline.save_dir = str(save_path)
self.preprocess_pipeline.run( self.preprocess_pipeline.run(
audio_path=str(prompt_path), audio_path=str(audio_path),
vocal_sep=prompt_vocal_sep, vocal_sep=vocal_sep,
max_merge_duration=20000, max_merge_duration=max_merge_duration,
language=prompt_lyric_lang or "Mandarin", language=lyric_lang or "Mandarin",
) )
self.preprocess_pipeline.save_dir = str(session_base / "transcriptions" / "target") return True, f"preprocess {audio_path} done"
self.preprocess_pipeline.run(
audio_path=str(target_path),
vocal_sep=target_vocal_sep,
max_merge_duration=60000,
language=target_lyric_lang or "Mandarin",
)
return True, "preprocess done"
except Exception as e: except Exception as e:
return False, f"preprocess failed: {e}" return False, f"preprocess failed: {e}"
@@ -271,191 +387,153 @@ class AppState:
APP_STATE = AppState() APP_STATE = AppState()
def _edit_metadata(
# i18n meta,
_i18n_key2lang_dict = dict( midi,
display_lang_label=dict(en="Display Language", zh="显示语言"), audio,
seed_label=dict(en="Seed", zh="种子"), language: str = "Mandarin",
prompt_audio_label=dict(en="Prompt audio (reference voice), limit to 30 seconds", zh="Prompt 音频（参考音色），限制在 30 秒以内"),
target_audio_label=dict(en="Target audio (melody / lyrics source), limit to 60 seconds", zh="Target 音频（旋律/歌词来源），限制在 60 秒以内"),
generate_btn_label=dict(en="Start SVS", zh="开始 SVS"),
transcription_btn_label=dict(en="Run singing transcription", zh="开始歌声转录"),
synthesis_btn_label=dict(en="Run singing synthesis", zh="歌声合成"),
prompt_meta_label=dict(en="Prompt metadata", zh="Prompt metadata"),
target_meta_label=dict(en="Target metadata", zh="Target metadata"),
edit_tutorial_html=dict(
en='<p class="mb-0">Refer to <a href="https://github.com/Soul-AILab/SoulX-Singer/tree/main/preprocess#step-2-edit-in-the-midi-editor" target="_blank" rel="noopener">Edit Tutorial</a> for metadata editing (Important Note: The generated metadata may not perfectly align the singing audio with the corresponding lyrics and musical notes. For better results, we strongly recommend manually correcting the alignment. You can directly use <a href="https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor" target="_blank" rel="noopener">SoulX-Singer-Midi-Editor</a> to edit) </p>',
zh='<p class="mb-0">metadata 编辑请参考 <a href="https://github.com/Soul-AILab/SoulX-Singer/tree/main/preprocess#step-2-edit-in-the-midi-editor" target="_blank" rel="noopener">编辑教程</a> (重要提示：自动生成的 metadata 在音频与歌词、音高对齐效果通常不理想。为了获得更好的结果，我们强烈建议手动纠正对齐，否则会导致合成效果不佳。 你可以直接使用 <a href="https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor" target="_blank" rel="noopener">SoulX-Singer-Midi-Editor</a> 进行编辑) </p>',
),
prompt_wav_label=dict(en="Prompt WAV (reference)", zh="Prompt WAV（参考音色）"),
generated_audio_label=dict(en="Generated merged audio", zh="合成结果音频"),
prompt_lyric_lang_label=dict(en="Prompt lyric language", zh="Prompt 歌词语种"),
target_lyric_lang_label=dict(en="Target lyric language", zh="Target 歌词语种"),
lyric_lang_mandarin=dict(en="Mandarin", zh="普通话"),
lyric_lang_cantonese=dict(en="Cantonese", zh="粤语"),
lyric_lang_english=dict(en="English", zh="英语"),
warn_missing_synthesis=dict(en="Please provide prompt WAV, prompt metadata, and target metadata", zh="请提供 Prompt WAV、Prompt metadata 与 Target metadata"),
prompt_vocal_sep_label=dict(en="Prompt vocal separation", zh="Prompt人声分离"),
target_vocal_sep_label=dict(en="Target vocal separation", zh="Target人声分离"),
auto_shift_label=dict(en="Auto pitch shift", zh="自动变调"),
pitch_shift_label=dict(en="Pitch shift (semitones)", zh="指定变调（半音）"),
control_type_label=dict(en="Control type", zh="控制类型"),
examples_label=dict(en="Reference examples (click to load)", zh="参考样例（点击加载）"),
example_choice_0=dict(en="—", zh="—"),
example_choice_1=dict(en="Example 1: Mandarin → Mandarin (melody), Start singing synthesis!", zh="样例 1: 普通话 → 普通话 (melody), 开始歌声合成吧!"),
example_choice_2=dict(en="Example 2: Mandarin → Cantonese (melody), Start singing synthesis!", zh="样例 2: 普通话 → 粤语 (melody), 开始歌声合成吧!"),
example_choice_3=dict(en="Example 3: Mandarin → English (melody), Start singing synthesis!", zh="样例 3: 普通话 → 英语 (melody), 开始歌声合成吧!"),
example_choice_4=dict(en="Example 4: Mandarin → Music (score), Start singing synthesis!", zh="样例 4: 普通话 → 音乐 (score), 开始歌声合成吧!"),
warn_missing_audio=dict(
en="Please upload both prompt audio and target audio",
zh="请上传 Prompt 音频与 Target 音频",
),
# Instruction panel (workflow description)
instruction_title=dict(en="Usage", zh="使用说明"),
instruction_p1=dict(
en="After uploading prompt and target audio and clicking **Run singing transcription**, the system generates two metadata files (prompt and target).",
zh="上传 Prompt 与 Target 音频并点击「开始歌声转录」后，将生成 Prompt 与 Target 两份 metadata 文件。",
),
instruction_p2=dict(
en="Auto-transcribed lyrics and notes are often misaligned. For better results, import the generated metadata into the **MIDI Editor** for manual adjustment: [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor).",
zh="自动转录的歌词与音高对齐效果通常不理想，建议将生成的 metadata 导入 **MIDI 编辑器** 进行手动调整：[SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor)。",
),
instruction_p3=dict(
en="Re-upload the adjusted metadata to the corresponding Prompt / Target Meta fields, then click **Run singing synthesis** to generate the final audio.",
zh="将调整后的 metadata 重新上传至对应的 Prompt / Target Meta 位置后，点击「歌声合成」开始最终生成。",
),
)
def _detect_initial_lang() -> Literal["zh", "en"]:
"""Detect initial UI language from server locale (browser language applied later via JS)."""
try:
import locale
loc = (locale.getdefaultlocale()[0] or os.environ.get("LANG", "") or "").lower()
return "en" if loc.startswith("en") else "zh"
except Exception:
return "zh"
global_lang: Literal["zh", "en"] = _detect_initial_lang()
def i18n(key: str) -> str:
return _i18n_key2lang_dict[key][global_lang]
def get_lyric_lang_choices():
"""Lyric language dropdown (display, value) for current UI language."""
return [
(i18n("lyric_lang_mandarin"), "Mandarin"),
(i18n("lyric_lang_cantonese"), "Cantonese"),
(i18n("lyric_lang_english"), "English"),
]
def _resolve_file_path(x):
"""Gradio file input can be path string or (path, None) tuple."""
if x is None:
return None
if isinstance(x, tuple):
x = x[0]
return x if (x and os.path.isfile(x)) else None
def transcription_function(
prompt_audio,
target_audio,
prompt_metadata,
target_metadata,
prompt_lyric_lang: str,
target_lyric_lang: str,
prompt_vocal_sep: bool,
target_vocal_sep: bool,
): ):
"""Step 1: Run transcription only; output (prompt_meta_path, target_meta_path)."""
try: try:
if isinstance(prompt_audio, tuple): meta = _resolve_file_path(meta)
prompt_audio = prompt_audio[0] midi = _resolve_file_path(midi)
if isinstance(target_audio, tuple): if not midi:
target_audio = target_audio[0] return meta
if prompt_audio is None or target_audio is None: audio = _normalize_audio_input(audio)
gr.Warning(message=i18n("warn_missing_audio"))
return None, None
prompt_meta_resolved = _resolve_file_path(prompt_metadata)
target_meta_resolved = _resolve_file_path(target_metadata)
use_input_metadata = prompt_meta_resolved is not None and target_meta_resolved is not None
session_base = _session_dir_from_target(target_audio) if not meta:
meta = str(Path(midi).with_name("metadata.json"))
APP_STATE.midi_parser.midi2meta(midi, meta, audio, language=language)
return meta
except Exception:
_print_exception("_edit_metadata")
return meta
def _transcribe_prompt(
prompt_audio,
prompt_metadata,
prompt_lyric_lang: str,
prompt_vocal_sep,
prompt_max_sec: int = PROMPT_MAX_SEC_DEFAULT,
):
try:
prompt_audio = _normalize_audio_input(prompt_audio)
prompt_meta_resolved = _resolve_file_path(prompt_metadata)
if prompt_audio is None and prompt_meta_resolved is None:
return None, None
session_base = _session_dir()
prompt_meta_path = session_base / "transcriptions" / "prompt" / "metadata.json"
prompt_midi_path = session_base / "transcriptions" / "prompt" / "vocal.mid"
if prompt_audio is not None:
audio_dir = session_base / "audio" audio_dir = session_base / "audio"
audio_dir.mkdir(parents=True, exist_ok=True) audio_dir.mkdir(parents=True, exist_ok=True)
transfer_prompt_path = audio_dir / "prompt.wav" transfer_prompt_path = audio_dir / "prompt.wav"
transfer_target_path = audio_dir / "target.wav" _trim_and_save_audio(prompt_audio, transfer_prompt_path, prompt_max_sec)
SR = 44100
PROMPT_MAX_SEC = 30
TARGET_MAX_SEC = 60
prompt_audio_data, _ = librosa.load(prompt_audio, sr=SR, mono=True)
target_audio_data, _ = librosa.load(target_audio, sr=SR, mono=True)
prompt_audio_data = prompt_audio_data[: PROMPT_MAX_SEC * SR]
target_audio_data = target_audio_data[: TARGET_MAX_SEC * SR]
sf.write(transfer_prompt_path, prompt_audio_data, SR)
sf.write(transfer_target_path, target_audio_data, SR)
prompt_meta_path = session_base / "transcriptions" / "prompt" / "metadata.json" prompt_ok, prompt_msg = APP_STATE.run_preprocess(
target_meta_path = session_base / "transcriptions" / "target" / "metadata.json" audio_path=transfer_prompt_path,
if use_input_metadata: save_path=session_base / "transcriptions" / "prompt",
(session_base / "transcriptions" / "prompt").mkdir(parents=True, exist_ok=True) vocal_sep=_yes_no_to_bool(prompt_vocal_sep, default=False),
(session_base / "transcriptions" / "target").mkdir(parents=True, exist_ok=True) lyric_lang=prompt_lyric_lang or "Mandarin",
shutil.copy2(prompt_meta_resolved, prompt_meta_path) max_merge_duration=PROMPT_MAX_MERGE_DURATION_MS,
shutil.copy2(target_meta_resolved, target_meta_path)
else:
ok, msg = APP_STATE.run_preprocess(
transfer_prompt_path,
transfer_target_path,
session_base,
prompt_vocal_sep=prompt_vocal_sep,
target_vocal_sep=target_vocal_sep,
prompt_lyric_lang=prompt_lyric_lang or "Mandarin",
target_lyric_lang=target_lyric_lang or "Mandarin",
) )
if not ok: if not prompt_ok:
print(msg, file=sys.stderr, flush=True) print(prompt_msg, file=sys.stderr, flush=True)
return None, None return None, None
elif prompt_meta_resolved is not None:
(session_base / "transcriptions" / "prompt").mkdir(parents=True, exist_ok=True)
shutil.copy2(prompt_meta_resolved, prompt_meta_path)
if prompt_meta_path.exists():
APP_STATE.midi_parser.meta2midi(prompt_meta_path, prompt_midi_path)
prompt_meta_file = str(prompt_meta_path) if prompt_meta_path.exists() else None prompt_meta_file = str(prompt_meta_path) if prompt_meta_path.exists() else None
target_meta_file = str(target_meta_path) if target_meta_path.exists() else None prompt_midi_file = str(prompt_midi_path) if prompt_midi_path.exists() else None
return prompt_meta_file, target_meta_file return prompt_meta_file, prompt_midi_file
except Exception: except Exception:
print(traceback.format_exc(), file=sys.stderr, flush=True) _print_exception("_transcribe_prompt")
return None, None return None, None
def synthesis_function( def _transcribe_target(
target_audio,
target_metadata,
target_lyric_lang: str,
target_vocal_sep,
target_max_sec: int = TARGET_MAX_SEC_DEFAULT,
):
try:
target_audio = _normalize_audio_input(target_audio)
target_meta_resolved = _resolve_file_path(target_metadata)
if target_audio is None and target_meta_resolved is None:
return None, None, None
session_base = _session_dir()
target_meta_path = session_base / "transcriptions" / "target" / "metadata.json"
target_midi_path = session_base / "transcriptions" / "target" / "vocal.mid"
target_vocal_path = session_base / "transcriptions" / "target" / "vocal.wav"
if target_audio is not None:
audio_dir = session_base / "audio"
audio_dir.mkdir(parents=True, exist_ok=True)
transfer_target_path = audio_dir / "target.wav"
_trim_and_save_audio(target_audio, transfer_target_path, target_max_sec)
target_ok, target_msg = APP_STATE.run_preprocess(
audio_path=transfer_target_path,
save_path=session_base / "transcriptions" / "target",
vocal_sep=_yes_no_to_bool(target_vocal_sep, default=True),
lyric_lang=target_lyric_lang or "Mandarin",
max_merge_duration=TARGET_MAX_MERGE_DURATION_MS,
)
if not target_ok:
print(target_msg, file=sys.stderr, flush=True)
return None, None, None
elif target_meta_resolved is not None:
(session_base / "transcriptions" / "target").mkdir(parents=True, exist_ok=True)
shutil.copy2(target_meta_resolved, target_meta_path)
if target_meta_path.exists():
APP_STATE.midi_parser.meta2midi(target_meta_path, target_midi_path)
target_meta_file = str(target_meta_path) if target_meta_path.exists() else None
target_midi_file = str(target_midi_path) if target_midi_path.exists() else None
target_vocal_file = str(target_vocal_path) if target_vocal_path.exists() else None
return target_meta_file, target_midi_file, target_vocal_file
except Exception:
_print_exception("_transcribe_target")
return None, None, None
def _run_synthesis(
prompt_audio, prompt_audio,
prompt_metadata, prompt_metadata,
target_metadata, target_metadata,
control: str, control: str,
auto_shift: bool, auto_shift,
pitch_shift, pitch_shift,
seed: int, seed: int,
): ):
"""Step 2: Run SVS from top prompt_audio + prompt_metadata + target_metadata.""" """Run singing synthesis from prompt audio + prompt metadata + target metadata."""
try: try:
if isinstance(prompt_audio, tuple): prompt_audio = _normalize_audio_input(prompt_audio)
prompt_audio = prompt_audio[0]
prompt_wav_path = prompt_audio prompt_wav_path = prompt_audio
prompt_meta_path = _resolve_file_path(prompt_metadata) prompt_meta_path = _resolve_file_path(prompt_metadata)
target_meta_path = _resolve_file_path(target_metadata) target_meta_path = _resolve_file_path(target_metadata)
if not prompt_wav_path or not os.path.isfile(prompt_wav_path): if not prompt_wav_path or not os.path.isfile(prompt_wav_path):
gr.Warning(message=i18n("warn_missing_synthesis")) gr.Warning(message=_i18n("warn_missing_synthesis"))
return None return None
if not prompt_meta_path or not os.path.isfile(prompt_meta_path): if not prompt_meta_path or not os.path.isfile(prompt_meta_path):
gr.Warning(message=i18n("warn_missing_synthesis")) gr.Warning(message=_i18n("warn_missing_synthesis"))
return None return None
if not target_meta_path or not os.path.isfile(target_meta_path): if not target_meta_path or not os.path.isfile(target_meta_path):
gr.Warning(message=i18n("warn_missing_synthesis")) gr.Warning(message=_i18n("warn_missing_synthesis"))
return None return None
if control not in ("melody", "score"): control = _control_to_internal(control)
control = "score" auto_shift = _yes_no_to_bool(auto_shift, default=True)
seed = int(seed) seed = int(seed)
torch.manual_seed(seed) torch.manual_seed(seed)
np.random.seed(seed) np.random.seed(seed)
@@ -473,16 +551,16 @@ def synthesis_function(
return None return None
return str(merged) return str(merged)
except Exception: except Exception:
print(traceback.format_exc(), file=sys.stderr, flush=True) _print_exception("_run_synthesis")
return None return None
def _instruction_md() -> str: def _instruction_md() -> str:
"""Markdown content for the instruction panel (supports links).""" """Markdown content for the instruction panel (supports links)."""
return "\n\n".join([ return "\n\n".join([
f"**1.** {i18n('instruction_p1')}", f"**1.** {_i18n('instruction_p1')}",
f"**2.** {i18n('instruction_p2')}", f"**2.** {_i18n('instruction_p2')}",
f"**3.** {i18n('instruction_p3')}", f"**3.** {_i18n('instruction_p3')}",
]) ])
@@ -511,127 +589,130 @@ def render_interface() -> gr.Blocks:
'"></div>' '"></div>'
'</div>' '</div>'
) )
# Auto-detect browser language: run after Gradio mounts
gr.HTML(
'<script type="text/javascript">'
'(function(){'
'function setLang(){'
'var lang=(navigator.language||navigator.userLanguage||"").toLowerCase();'
'if(lang.startsWith("en")){'
'var inputs=document.querySelectorAll("#lang_choice_radio input");'
'if(inputs.length>1)inputs[1].click();'
'}'
'}'
'if(document.readyState==="complete")setTimeout(setLang,800);'
'else window.addEventListener("load",function(){setTimeout(setLang,800);});'
'})();'
'</script>',
visible=False,
)
with gr.Row(equal_height=True): with gr.Row(equal_height=True):
lang_choice = gr.Radio( lang_choice = gr.Radio(
choices=["中文", "English"], choices=["中文", "English"],
value="中文", value="中文",
label=i18n("display_lang_label"), label=_i18n("display_lang_label"),
type="index", type="index",
interactive=True, interactive=True,
elem_id="lang_choice_radio", elem_id="lang_choice_radio",
) )
# Instruction panel (usage workflow); updates on language change # Instruction panel (usage workflow); updates on language change
instruction_md = gr.Markdown(f"### {i18n('instruction_title')}\n\n{_instruction_md()}") instruction_md = gr.Markdown(f"### {_i18n('instruction_title')}\n\n{_instruction_md()}")
# Reference examples — at the front of operations (handler registered after components exist) # Reference examples — at the front of operations (handler registered after components exist)
skip_clear_metadata_count = gr.State(0) skip_clear_metadata_count = gr.State(0)
with gr.Row(): with gr.Row():
_example_choices = [i18n("example_choice_0"), i18n("example_choice_1"), i18n("example_choice_2"), i18n("example_choice_3"), i18n("example_choice_4")] _example_choices = [_i18n("example_choice_0"), _i18n("example_choice_1"), _i18n("example_choice_2"), _i18n("example_choice_3"), _i18n("example_choice_4")]
example_choice = gr.Dropdown( example_choice = gr.Dropdown(
label=i18n("examples_label"), label=_i18n("examples_label"),
choices=_example_choices, choices=_example_choices,
value=_example_choices[0], value=_example_choices[0],
interactive=True, interactive=True,
) )
# Step 1: Transcription (audio → metadata) # Step 1: Transcription (audio → metadata)
with gr.Accordion(_i18n("section_input_audio"), open=True) as accordion_input_audio:
with gr.Row(equal_height=True): with gr.Row(equal_height=True):
with gr.Column(scale=1): with gr.Column(scale=1):
prompt_audio = gr.Audio( prompt_audio = gr.Audio(
label=i18n("prompt_audio_label"), label=_i18n("prompt_audio_label"),
type="filepath", type="filepath",
editable=False, editable=False,
interactive=True, interactive=True,
) )
with gr.Column(scale=1): with gr.Column(scale=1):
target_audio = gr.Audio( target_audio = gr.Audio(
label=i18n("target_audio_label"), label=_i18n("target_audio_label"),
type="filepath", type="filepath",
editable=False, editable=False,
interactive=True, interactive=True,
) )
with gr.Accordion(_i18n("section_transcriptions"), open=True) as accordion_transcriptions:
with gr.Row(equal_height=True): with gr.Row(equal_height=True):
prompt_lyric_lang = gr.Dropdown( prompt_lyric_lang = gr.Dropdown(
label=i18n("prompt_lyric_lang_label"), label=_i18n("prompt_lyric_lang_label"),
choices=get_lyric_lang_choices(), choices=_get_lyric_lang_choices(),
value="Mandarin", value="Mandarin",
interactive=True, interactive=True,
scale=1, scale=1,
) )
prompt_vocal_sep = gr.Dropdown(
label=_i18n("prompt_vocal_sep_label"),
choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")],
value="no",
interactive=True,
scale=1,
)
target_lyric_lang = gr.Dropdown( target_lyric_lang = gr.Dropdown(
label=i18n("target_lyric_lang_label"), label=_i18n("target_lyric_lang_label"),
choices=get_lyric_lang_choices(), choices=_get_lyric_lang_choices(),
value="Mandarin", value="Mandarin",
interactive=True, interactive=True,
scale=1, scale=1,
) )
prompt_vocal_sep = gr.Checkbox( target_vocal_sep = gr.Dropdown(
label=i18n("prompt_vocal_sep_label"), label=_i18n("target_vocal_sep_label"),
value=False, choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")],
value="yes",
interactive=True, interactive=True,
scale=1, scale=1,
) )
target_vocal_sep = gr.Checkbox(
label=i18n("target_vocal_sep_label"),
value=True,
interactive=True,
scale=1,
)
with gr.Row():
transcription_btn = gr.Button(
value=i18n("transcription_btn_label"),
variant="primary",
size="lg",
)
# Edit tutorial link (gr.HTML supports links; component labels do not)
metadata_tutorial_html = gr.HTML(value=i18n("edit_tutorial_html"))
# Synthesis: params row, then synthesis button on next row
with gr.Row(equal_height=True): with gr.Row(equal_height=True):
prompt_metadata = gr.File( prompt_metadata = gr.File(
label=i18n("prompt_meta_label"), label=_i18n("prompt_meta_label"),
type="filepath", type="filepath",
file_types=[".json"], file_types=[".json"],
height=140,
interactive=True,
)
prompt_midi = gr.File(
label=_i18n("prompt_midi_label"),
type="filepath",
file_types=[".midi", ".mid"],
height=140,
interactive=True, interactive=True,
) )
target_metadata = gr.File( target_metadata = gr.File(
label=i18n("target_meta_label"), label=_i18n("target_meta_label"),
type="filepath", type="filepath",
file_types=[".json"], file_types=[".json"],
height=140,
interactive=True, interactive=True,
) )
control_radio = gr.Radio( target_midi = gr.File(
choices=["melody", "score"], label=_i18n("target_midi_label"),
value="score", type="filepath",
label=i18n("control_type_label"), file_types=[".midi", ".mid"],
height=140,
interactive=True,
)
target_vocal = gr.File(
type="filepath",
file_types=[".wav"],
interactive=False,
visible=False,
)
with gr.Accordion(_i18n("section_synthesis"), open=True) as accordion_synthesis:
with gr.Row(equal_height=True):
control_radio = gr.Dropdown(
choices=[(_i18n("control_melody"), "melody-controlled"), (_i18n("control_score"), "score-controlled")],
value="score-controlled",
label=_i18n("control_type_label"),
scale=1, scale=1,
) )
auto_shift = gr.Checkbox( auto_shift = gr.Dropdown(
label=i18n("auto_shift_label"), label=_i18n("auto_shift_label"),
value=True, choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")],
value="yes",
interactive=True, interactive=True,
scale=1, scale=1,
) )
pitch_shift = gr.Number( pitch_shift = gr.Number(
label=i18n("pitch_shift_label"), label=_i18n("pitch_shift_label"),
value=0, value=0,
minimum=-36, minimum=-36,
maximum=36, maximum=36,
@@ -640,7 +721,7 @@ def render_interface() -> gr.Blocks:
scale=1, scale=1,
) )
seed_input = gr.Number( seed_input = gr.Number(
label=i18n("seed_label"), label=_i18n("seed_label"),
value=12306, value=12306,
step=1, step=1,
interactive=True, interactive=True,
@@ -648,13 +729,13 @@ def render_interface() -> gr.Blocks:
) )
with gr.Row(): with gr.Row():
synthesis_btn = gr.Button( synthesis_btn = gr.Button(
value=i18n("synthesis_btn_label"), value=_i18n("synthesis_btn_label"),
variant="primary", variant="primary",
size="lg", size="lg",
) )
with gr.Row(): with gr.Row():
output_audio = gr.Audio( output_audio = gr.Audio(
label=i18n("generated_audio_label"), label=_i18n("generated_audio_label"),
type="filepath", type="filepath",
interactive=False, interactive=False,
) )
@@ -679,33 +760,36 @@ def render_interface() -> gr.Blocks:
) )
def _change_component_language(lang): def _change_component_language(lang):
global global_lang global _GLOBAL_LANG
global_lang = ["zh", "en"][lang] _GLOBAL_LANG = ["zh", "en"][lang]
choices = get_lyric_lang_choices() lyric_choices = _get_lyric_lang_choices()
yes_no_choices = [(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")]
control_choices = [(_i18n("control_melody"), "melody-controlled"), (_i18n("control_score"), "score-controlled")]
return [ return [
gr.update(label=i18n("prompt_audio_label")), gr.update(label=_i18n("prompt_audio_label")),
gr.update(label=i18n("target_audio_label")), gr.update(label=_i18n("target_audio_label")),
gr.update(label=i18n("prompt_lyric_lang_label"), choices=choices), gr.update(label=_i18n("prompt_lyric_lang_label"), choices=lyric_choices),
gr.update(label=i18n("target_lyric_lang_label"), choices=choices), gr.update(label=_i18n("target_lyric_lang_label"), choices=lyric_choices),
gr.update(label=i18n("prompt_vocal_sep_label")), gr.update(label=_i18n("prompt_vocal_sep_label"), choices=yes_no_choices),
gr.update(label=i18n("target_vocal_sep_label")), gr.update(label=_i18n("target_vocal_sep_label"), choices=yes_no_choices),
gr.update(value=i18n("transcription_btn_label")), gr.update(label=_i18n("prompt_meta_label")),
gr.update(label=i18n("prompt_meta_label")), gr.update(label=_i18n("target_meta_label")),
gr.update(label=i18n("target_meta_label")), gr.update(label=_i18n("control_type_label"), choices=control_choices),
gr.update(value=i18n("edit_tutorial_html")), gr.update(label=_i18n("auto_shift_label"), choices=yes_no_choices),
gr.update(label=i18n("control_type_label")), gr.update(label=_i18n("pitch_shift_label")),
gr.update(label=i18n("auto_shift_label")), gr.update(label=_i18n("seed_label")),
gr.update(label=i18n("pitch_shift_label")), gr.update(value=_i18n("synthesis_btn_label")),
gr.update(label=i18n("seed_label")), gr.update(label=_i18n("generated_audio_label")),
gr.update(value=i18n("synthesis_btn_label")), gr.update(label=_i18n("display_lang_label")),
gr.update(label=i18n("generated_audio_label")),
gr.update(label=i18n("display_lang_label")),
gr.update( gr.update(
label=i18n("examples_label"), label=_i18n("examples_label"),
choices=[i18n("example_choice_0"), i18n("example_choice_1"), i18n("example_choice_2"), i18n("example_choice_3"), i18n("example_choice_4")], choices=[_i18n("example_choice_0"), _i18n("example_choice_1"), _i18n("example_choice_2"), _i18n("example_choice_3"), _i18n("example_choice_4")],
value=i18n("example_choice_0"), value=_i18n("example_choice_0"),
), ),
gr.update(value=f"### {i18n('instruction_title')}\n\n{_instruction_md()}"), gr.update(value=f"### {_i18n('instruction_title')}\n\n{_instruction_md()}"),
gr.update(label=_i18n("section_input_audio")),
gr.update(label=_i18n("section_transcriptions")),
gr.update(label=_i18n("section_synthesis")),
] ]
lang_choice.change( lang_choice.change(
@@ -718,10 +802,8 @@ def render_interface() -> gr.Blocks:
target_lyric_lang, target_lyric_lang,
prompt_vocal_sep, prompt_vocal_sep,
target_vocal_sep, target_vocal_sep,
transcription_btn,
prompt_metadata, prompt_metadata,
target_metadata, target_metadata,
metadata_tutorial_html,
control_radio, control_radio,
auto_shift, auto_shift,
pitch_shift, pitch_shift,
@@ -731,6 +813,9 @@ def render_interface() -> gr.Blocks:
lang_choice, lang_choice,
example_choice, example_choice,
instruction_md, instruction_md,
accordion_input_audio,
accordion_transcriptions,
accordion_synthesis,
], ],
) )
@@ -740,29 +825,34 @@ def render_interface() -> gr.Blocks:
inputs=[prompt_audio, skip_clear_metadata_count], inputs=[prompt_audio, skip_clear_metadata_count],
outputs=[prompt_metadata, skip_clear_metadata_count], outputs=[prompt_metadata, skip_clear_metadata_count],
) )
prompt_audio.upload(
fn=_transcribe_prompt,
inputs=[prompt_audio, prompt_metadata, prompt_lyric_lang, prompt_vocal_sep],
outputs=[prompt_metadata, prompt_midi],
)
prompt_midi.upload(
fn=_edit_metadata,
inputs=[prompt_metadata, prompt_midi, prompt_audio, prompt_lyric_lang],
outputs=[prompt_metadata],
)
target_audio.change( target_audio.change(
fn=_clear_target_meta_unless_example, fn=_clear_target_meta_unless_example,
inputs=[target_audio, skip_clear_metadata_count], inputs=[target_audio, skip_clear_metadata_count],
outputs=[target_metadata, skip_clear_metadata_count], outputs=[target_metadata, skip_clear_metadata_count],
) )
target_audio.upload(
transcription_btn.click( fn=_transcribe_target,
fn=transcription_function, inputs=[target_audio, target_metadata, target_lyric_lang, target_vocal_sep],
inputs=[ outputs=[target_metadata, target_midi, target_vocal],
prompt_audio, )
target_audio, target_midi.upload(
prompt_metadata, fn=_edit_metadata,
target_metadata, inputs=[target_metadata, target_midi, target_vocal, target_lyric_lang],
prompt_lyric_lang, outputs=[target_metadata],
target_lyric_lang,
prompt_vocal_sep,
target_vocal_sep,
],
outputs=[prompt_metadata, target_metadata],
) )
synthesis_btn.click( synthesis_btn.click(
fn=synthesis_function, fn=_run_synthesis,
inputs=[ inputs=[
prompt_audio, prompt_audio,
prompt_metadata, prompt_metadata,