diff --git a/preprocess/tools/midi_parser.py b/preprocess/tools/midi_parser.py index 3c0ed3b..dd7c694 100644 --- a/preprocess/tools/midi_parser.py +++ b/preprocess/tools/midi_parser.py @@ -20,15 +20,16 @@ from .f0_extraction import F0Extractor from .g2p import g2p_transform -# Audio and segmenting constants (used by _edit_data_to_meta) -SAMPLE_RATE = 44100 -DEFAULT_LANGUAGE = "Mandarin" -MAX_GAP_SEC = 5.0 # gap (sec) above which we start a new segment +# Audio, MIDI and segmentation constants +SAMPLE_RATE = 44100 # Hz, fixed for all audio processing in this script to ensure consistent timing with MIDI ticks. +MIDI_TICKS_PER_BEAT = 500 +MIDI_TEMPO = 500000 # microseconds per beat (120 BPM) +MIDI_TIME_SIGNATURE = (4, 4) +MIDI_VELOCITY = 64 +END_EXTENSION_SEC = 0.4 # extend each segment end by this much silence (sec) to give the model more context +MAX_GAP_SEC = 2.0 # gap (sec) above which we start a new segment MAX_SEGMENT_DUR_SUM_SEC = 60.0 # max cumulative note duration per segment (sec) -MIN_GAP_THRESHOLD_SEC = 0.001 # ignore gaps smaller than this -LONG_SILENCE_THRESHOLD_SEC = 0.05 # treat as separate if gap larger -MAX_LEADING_SP_DUR_SEC = 2.0 # cap leading silence in a segment to this (sec) -DEFAULT_RMVPE_MODEL_PATH = "pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt" +SILENCE_THRESHOLD_SEC = 0.2 # treat as separate if gap larger @dataclass @@ -44,42 +45,6 @@ class Note: def end_s(self) -> float: return self.start_s + self.note_dur - - -def remove_duplicate_segments(meta_data: List[dict]) -> None: - """Merge consecutive identical notes (same text, pitch, type) within each segment. Mutates meta_data in place.""" - for idx, segment in enumerate(meta_data): - texts = segment["note_text"] - durs = segment["note_dur"] - pitches = segment["note_pitch"] - types = segment["note_type"] - new_texts = [] - new_durs = [] - new_pitches = [] - new_types = [] - for i in range(len(texts)): - if i == 0: - new_texts.append(texts[i]) - new_durs.append(durs[i]) - new_pitches.append(pitches[i]) - new_types.append(types[i]) - continue - t, d, p, ty = texts[i], durs[i], pitches[i], types[i] - if t == "" and texts[i - 1] == "": - new_durs[-1] += d - continue - if t == texts[i - 1] and p == pitches[i - 1] and ty == types[i - 1]: - new_durs[-1] += d - else: - new_texts.append(t) - new_durs.append(d) - new_pitches.append(p) - new_types.append(ty) - meta_data[idx]["note_text"] = new_texts - meta_data[idx]["note_dur"] = new_durs - meta_data[idx]["note_pitch"] = new_pitches - meta_data[idx]["note_type"] = new_types - def meta2notes(meta_path: str) -> List[Note]: """Parse SoulX-Singer metadata JSON into a flat list of Note (absolute start_s).""" with open(meta_path, "r", encoding="utf-8") as f: @@ -92,7 +57,7 @@ def meta2notes(meta_path: str) -> List[Note]: notes: List[Note] = [] for seg in segments: offset_s = seg["time"][0] / 1000 - words = [str(x).replace("", "") for i, x in enumerate(seg["text"].split())] + words = [str(x).replace("", "") for x in seg["text"].split()] word_durs = [float(x) for x in seg["duration"].split()] pitches = [int(x) for x in seg["note_pitch"].split()] types = [int(x) if words[i] != "" else 1 for i, x in enumerate(seg["note_type"].split())] @@ -117,9 +82,10 @@ def meta2notes(meta_path: str) -> List[Note]: def _append_segment_to_meta( meta_path_str: str, - cut_wavs_output_dir: str, - vocal_file: str, - audio_data: Any, + cut_wavs_output_dir: str | None, + vocal_file: str | None, + language: str, + audio_data: Any | None, meta_data: List[dict], note_start: List[float], note_end: List[float], @@ -127,28 +93,40 @@ def _append_segment_to_meta( note_pitch: List[Any], note_type: List[Any], note_dur: List[float], - end_time_ms_override: float | None = None, ) -> None: """Write one segment wav and append one segment dict to meta_data. Caller clears note_* lists after.""" + if not all((note_start, note_end, note_text, note_pitch, note_type, note_dur)): + return + base_name = os.path.splitext(os.path.basename(meta_path_str))[0] item_name = f"{base_name}_{len(meta_data)}" - wav_fn = os.path.join(cut_wavs_output_dir, f"{item_name}.wav") - start_ms = int(note_start[0] * 1000) - end_ms = ( - int(end_time_ms_override) - if end_time_ms_override is not None - else int(note_end[-1] * 1000) - ) - start_sample = int(note_start[0] * SAMPLE_RATE) - end_sample = int(note_end[-1] * SAMPLE_RATE) - write(wav_fn, audio_data[start_sample:end_sample], SAMPLE_RATE) + wav_fn = None + if cut_wavs_output_dir and vocal_file and audio_data is not None: + wav_fn = os.path.join(cut_wavs_output_dir, f"{item_name}.wav") + end_pad = int(END_EXTENSION_SEC * SAMPLE_RATE) + start_sample = max(0, int(note_start[0] * SAMPLE_RATE)) + end_sample = min(len(audio_data), int(note_end[-1] * SAMPLE_RATE) + end_pad) + + end_pad_dur = (end_sample / SAMPLE_RATE - note_end[-1]) if end_sample > int(note_end[-1] * SAMPLE_RATE) else 0.0 + if end_pad_dur > 0: + note_dur = note_dur + [end_pad_dur] + note_text = note_text + [""] + note_pitch = note_pitch + [0] + note_type = note_type + [1] + start_ms = int(start_sample / SAMPLE_RATE * 1000) + end_ms = int(end_sample / SAMPLE_RATE * 1000) + write(wav_fn, audio_data[start_sample:end_sample], SAMPLE_RATE) + else: + start_ms = int(note_start[0] * 1000) + end_ms = int(note_end[-1] * 1000) + meta_data.append({ "item_name": item_name, "wav_fn": wav_fn, "origin_wav_fn": vocal_file, "start_time_ms": start_ms, "end_time_ms": end_ms, - "language": DEFAULT_LANGUAGE, + "language": language, "note_text": list(note_text), "note_pitch": list(note_pitch), "note_type": list(note_type), @@ -156,22 +134,25 @@ def _append_segment_to_meta( }) -def convert_meta(meta_data: List[dict], rmvpe_model_path, device="cuda"): - pitch_extractor = F0Extractor(rmvpe_model_path, device=device, verbose=False) +def convert_meta(meta_data: List[dict], pitch_extractor: F0Extractor | None) -> List[dict]: converted_data = [] for item in meta_data: + language = item.get("language", "Mandarin") wav_fn = item.get("wav_fn") - if not wav_fn or not os.path.isfile(wav_fn): - raise FileNotFoundError(f"Segment wav file not found: {wav_fn}") - f0 = pitch_extractor.process(wav_fn) + if pitch_extractor is not None: + if not wav_fn or not os.path.isfile(wav_fn): + raise FileNotFoundError(f"Segment wav file not found: {wav_fn}") + f0 = pitch_extractor.process(wav_fn) + else: + f0 = [] converted_item = { "index": item.get("item_name"), - "language": item.get("language"), + "language": language, "time": [item.get("start_time_ms", 0), item.get("end_time_ms", sum(item["note_dur"]) * 1000)], "duration": " ".join(str(round(x, 2)) for x in item.get("note_dur", [])), "text": " ".join(item.get("note_text", [])), - "phoneme": " ".join(g2p_transform(item.get("note_text", []), DEFAULT_LANGUAGE)), + "phoneme": " ".join(g2p_transform(item.get("note_text", []), language)), "note_pitch": " ".join(str(x) for x in item.get("note_pitch", [])), "note_type": " ".join(str(x) for x in item.get("note_type", [])), "f0": " ".join(str(round(float(x), 1)) for x in f0), @@ -184,14 +165,16 @@ def convert_meta(meta_data: List[dict], rmvpe_model_path, device="cuda"): def _edit_data_to_meta( meta_path_str: str, edit_data: List[dict], - vocal_file: str, - rmvpe_model_path: str | None = None, - device: str = "cuda", + vocal_file: str | None, + language: str, + pitch_extractor: F0Extractor | None, ) -> None: """Write SoulX-Singer metadata JSON from edit_data (list of {start, end, note_text, note_pitch, note_type}).""" - # Use a fixed temporary directory for cut wavs - cut_wavs_output_dir = os.path.join(os.path.dirname(vocal_file), "cut_wavs_tmp") - os.makedirs(cut_wavs_output_dir, exist_ok=True) + # Store temporary cut wavs beside the source vocal (same folder, fixed subdir name). + cut_wavs_output_dir = None + if vocal_file: + cut_wavs_output_dir = os.path.join(os.path.dirname(vocal_file), "cut_wavs_tmp") + os.makedirs(cut_wavs_output_dir, exist_ok=True) note_text: List[Any] = [] note_pitch: List[Any] = [] @@ -199,11 +182,36 @@ def _edit_data_to_meta( note_dur: List[float] = [] note_start: List[float] = [] note_end: List[float] = [] - prev_end = 0.0 meta_data: List[dict] = [] - audio_data, _ = librosa.load(vocal_file, sr=SAMPLE_RATE, mono=True) + audio_data = None + if vocal_file: + audio_data, _ = librosa.load(vocal_file, sr=SAMPLE_RATE, mono=True) dur_sum = 0.0 + def flush_current_segment() -> None: + nonlocal dur_sum + _append_segment_to_meta( + meta_path_str, + cut_wavs_output_dir, + vocal_file, + language, + audio_data, + meta_data, + note_start, + note_end, + note_text, + note_pitch, + note_type, + note_dur, + ) + note_text.clear() + note_pitch.clear() + note_type.clear() + note_dur.clear() + note_start.clear() + note_end.clear() + dur_sum = 0.0 + for entry in edit_data: start = float(entry["start"]) end = float(entry["end"]) @@ -218,85 +226,27 @@ def _edit_data_to_meta( note_dur.append(end - start) note_start.append(start) note_end.append(end) - prev_end = end dur_sum += end - start continue if ( len(note_text) > 0 and note_text[-1] == "" - and note_dur[-1] > MAX_LEADING_SP_DUR_SEC + and note_dur[-1] > MAX_GAP_SEC ): - cut_time = note_dur[-1] - MAX_LEADING_SP_DUR_SEC - note_dur[-1] = MAX_LEADING_SP_DUR_SEC - end_ms_override = note_end[-1] * 1000 - cut_time * 1000 - _append_segment_to_meta( - meta_path_str, - cut_wavs_output_dir, - vocal_file, - audio_data, - meta_data, - note_start, - note_end, - note_text, - note_pitch, - note_type, - note_dur, - end_time_ms_override=end_ms_override, - ) - note_text = [] - note_pitch = [] - note_type = [] - note_dur = [] - note_start = [] - note_end = [] - prev_end = start - dur_sum = 0.0 + note_text.pop() + note_pitch.pop() + note_type.pop() + note_dur.pop() + note_start.pop() + note_end.pop() - gap_from_prev = start - prev_end - gap_from_last_note = (start - note_end[-1]) if note_end else 0.0 - if ( - gap_from_prev >= MAX_GAP_SEC - or gap_from_last_note >= MAX_GAP_SEC - or dur_sum >= MAX_SEGMENT_DUR_SUM_SEC - ): - if len(note_text) > 0: - _append_segment_to_meta( - meta_path_str, - cut_wavs_output_dir, - vocal_file, - audio_data, - meta_data, - note_start, - note_end, - note_text, - note_pitch, - note_type, - note_dur, - ) - note_text = [] - note_pitch = [] - note_type = [] - note_dur = [] - note_start = [] - note_end = [] - prev_end = start - dur_sum = 0.0 + dur_sum = sum(note_dur) + flush_current_segment() - if start - prev_end > MIN_GAP_THRESHOLD_SEC: - if start - prev_end > LONG_SILENCE_THRESHOLD_SEC or len(note_text) == 0: - note_text.append("") - note_pitch.append(0) - note_type.append(1) - note_dur.append(start - prev_end) - note_start.append(prev_end) - note_end.append(start) - else: - if len(note_dur) > 0: - note_dur[-1] += start - prev_end - note_end[-1] = start + if dur_sum + (end - start) > MAX_SEGMENT_DUR_SUM_SEC and len(note_text) > 0: + flush_current_segment() - prev_end = end note_text.append(text) note_pitch.append(int(pitch)) note_type.append(int(type_)) @@ -305,42 +255,49 @@ def _edit_data_to_meta( note_end.append(end) dur_sum += end - start - if len(note_text) > 0: - _append_segment_to_meta( - meta_path_str, - cut_wavs_output_dir, - vocal_file, - audio_data, - meta_data, - note_start, - note_end, - note_text, - note_pitch, - note_type, - note_dur, - ) + if note_text: + flush_current_segment() - remove_duplicate_segments(meta_data) + # Merge only consecutive tokens to reduce fragmentation in silence regions. + for segment in meta_data: + phoneme = segment['note_text'] + duration = segment['note_dur'] + note_pitch = segment['note_pitch'] + note_type = segment['note_type'] - _rmvpe_path = rmvpe_model_path or DEFAULT_RMVPE_MODEL_PATH - converted_data = convert_meta(meta_data, _rmvpe_path, device) + merged_items: List[Tuple[str, float, int, int]] = [] + prev_item = None + for text, dur, pitch, note_type in zip(phoneme, duration, note_pitch, note_type): + if prev_item and text == "" and prev_item[0] == "": + merged_items[-1] = (prev_item[0], prev_item[1] + dur, prev_item[2], prev_item[3]) + else: + merged_items.append((text, dur, pitch, note_type)) + prev_item = merged_items[-1] + + segment['note_text'] = [item[0] for item in merged_items] + segment['note_dur'] = [item[1] for item in merged_items] + segment['note_pitch'] = [item[2] for item in merged_items] + segment['note_type'] = [item[3] for item in merged_items] + + converted_data = convert_meta(meta_data, pitch_extractor) with open(meta_path_str, "w", encoding="utf-8") as f: json.dump(converted_data, f, ensure_ascii=False, indent=2) # Clean up temporary cut wavs directory - try: - shutil.rmtree(cut_wavs_output_dir, ignore_errors=True) - except Exception: - pass + if cut_wavs_output_dir: + try: + shutil.rmtree(cut_wavs_output_dir, ignore_errors=True) + except Exception: + pass def notes2meta( notes: List[Note], meta_path: str, - vocal_file: str, - rmvpe_model_path: str | None = None, - device: str = "cuda", + vocal_file: str | None, + language: str, + pitch_extractor: F0Extractor | None, ) -> None: """Write SoulX-Singer metadata JSON from a list of Note (segmenting + wav cuts).""" edit_data = [ @@ -357,30 +314,21 @@ def notes2meta( str(meta_path), edit_data, vocal_file, - rmvpe_model_path=rmvpe_model_path, - device=device, + language, + pitch_extractor=pitch_extractor, ) -@dataclass(frozen=True) -class MidiDefaults: - ticks_per_beat: int = 500 - tempo: int = 500000 # microseconds per beat (120 BPM) - time_signature: Tuple[int, int] = (4, 4) - velocity: int = 64 - - def _seconds_to_ticks(seconds: float, ticks_per_beat: int, tempo: int) -> int: + # ticks = seconds * (ticks_per_beat beats) / (tempo microseconds per beat) return int(round(seconds * ticks_per_beat * 1_000_000 / tempo)) def notes2midi( notes: List[Note], midi_path: str, - defaults: MidiDefaults | None = None, ) -> None: """Write MIDI file from a list of Note.""" - defaults = defaults or MidiDefaults() if not notes: raise ValueError("Empty note list.") @@ -392,15 +340,16 @@ def notes2midi( continue start_ticks = _seconds_to_ticks( - start_s, defaults.ticks_per_beat, defaults.tempo + start_s, MIDI_TICKS_PER_BEAT, MIDI_TEMPO ) end_ticks = _seconds_to_ticks( - end_s, defaults.ticks_per_beat, defaults.tempo + end_s, MIDI_TICKS_PER_BEAT, MIDI_TEMPO ) if end_ticks <= start_ticks: end_ticks = start_ticks + 1 lyric = n.note_text + # Some DAWs store lyric text as latin1-compatible bytes; keep best-effort round-trip. try: lyric = lyric.encode("utf-8").decode("latin1") except (UnicodeEncodeError, UnicodeDecodeError): @@ -418,7 +367,7 @@ def notes2midi( mido.Message( "note_on", note=n.note_pitch, - velocity=defaults.velocity, + velocity=MIDI_VELOCITY, time=0, ), ) @@ -431,18 +380,19 @@ def notes2midi( ) ) + # Keep deterministic ordering at same tick: note_off -> lyric -> note_on. events.sort(key=lambda x: (x[0], x[1])) - mid = mido.MidiFile(ticks_per_beat=defaults.ticks_per_beat) + mid = mido.MidiFile(ticks_per_beat=MIDI_TICKS_PER_BEAT) track = mido.MidiTrack() mid.tracks.append(track) - track.append(mido.MetaMessage("set_tempo", tempo=defaults.tempo, time=0)) + track.append(mido.MetaMessage("set_tempo", tempo=MIDI_TEMPO, time=0)) track.append( mido.MetaMessage( "time_signature", - numerator=defaults.time_signature[0], - denominator=defaults.time_signature[1], + numerator=MIDI_TIME_SIGNATURE[0], + denominator=MIDI_TIME_SIGNATURE[1], time=0, ) ) @@ -458,7 +408,10 @@ def notes2midi( def midi2notes(midi_path: str) -> List[Note]: - """Parse MIDI file into a list of Note. Merges all tracks; tempo from last set_tempo event.""" + """Parse MIDI file into a list of Note. + + Merges all tracks and uses the latest encountered set_tempo as global tempo. + """ mid = mido.MidiFile(midi_path) ticks_per_beat = mid.ticks_per_beat tempo = 500000 @@ -520,6 +473,7 @@ def midi2notes(midi_path: str) -> List[Note]: lyrics.sort(key=lambda x: x[0]) trimmed = [] + # Remove/trim overlaps so generated notes are strictly non-overlapping in tick domain. for note in raw_notes: while trimmed: prev = trimmed[-1] @@ -534,6 +488,7 @@ def midi2notes(midi_path: str) -> List[Note]: raw_notes = trimmed tolerance = ticks_per_beat // 100 + # Attach lyrics near note_on positions with a small tick tolerance. lyric_idx = 0 for note in raw_notes: while lyric_idx < len(lyrics) and lyrics[lyric_idx][0] < note["start_ticks"] - tolerance: @@ -559,6 +514,7 @@ def midi2notes(midi_path: str) -> List[Note]: continue lyric = n.get("lyric", "") + # SoulX-Singer convention mapping from lyric token to note_type/text. if not lyric: tp = 2 text = "啦" @@ -572,6 +528,21 @@ def midi2notes(midi_path: str) -> List[Note]: tp = 2 text = lyric + if start_s - prev_end_s > SILENCE_THRESHOLD_SEC: + # Explicitly represent long gaps as notes. + result.append( + Note( + start_s=prev_end_s, + note_dur=start_s - prev_end_s, + note_text="", + note_pitch=0, + note_type=1, + ) + ) + else: + if len(result) > 0: + result[-1].note_dur = start_s - result[-1].start_s + result.append( Note( start_s=start_s, @@ -586,35 +557,51 @@ def midi2notes(midi_path: str) -> List[Note]: return result -def meta2midi(meta_path: str, midi_path: str, defaults: MidiDefaults | None = None) -> None: - """Convert SoulX-Singer metadata JSON to MIDI file (meta -> List[Note] -> midi).""" - notes = meta2notes(meta_path) - notes2midi(notes, midi_path, defaults) - print(f"Saved MIDI to {midi_path}") +class MidiParser: + def __init__( + self, + rmvpe_model_path: str, + device: str = "cuda", + ) -> None: + self.rmvpe_model_path = rmvpe_model_path + self.device = device + self.pitch_extractor: F0Extractor | None = None + def _get_pitch_extractor(self) -> F0Extractor: + if self.pitch_extractor is None: + self.pitch_extractor = F0Extractor( + self.rmvpe_model_path, + device=self.device, + verbose=False, + ) + return self.pitch_extractor -def midi2meta( - midi_path: str, - meta_path: str, - vocal_file: str, - rmvpe_model_path: str | None = None, - device: str = "cuda", -) -> None: - """Convert MIDI file to SoulX-Singer metadata JSON (midi -> List[Note] -> meta).""" - meta_dir = os.path.dirname(meta_path) - if meta_dir: - os.makedirs(meta_dir, exist_ok=True) - # cut_wavs will be written to a fixed temporary directory inside _edit_data_to_meta - notes = midi2notes(midi_path) - notes2meta( - notes, - meta_path, - vocal_file, - rmvpe_model_path=rmvpe_model_path, - device=device, - ) - print(f"Saved Meta to {meta_path}") + def midi2meta( + self, + midi_path: str, + meta_path: str, + vocal_file: str | None = None, + language: str = "Mandarin", + ) -> None: + meta_dir = os.path.dirname(meta_path) + if meta_dir: + os.makedirs(meta_dir, exist_ok=True) + notes = midi2notes(midi_path) + pitch_extractor = self._get_pitch_extractor() if vocal_file else None + notes2meta( + notes, + meta_path, + vocal_file, + language, + pitch_extractor=pitch_extractor, + ) + print(f"Saved Meta to {meta_path}") + + def meta2midi(self, meta_path: str, midi_path: str) -> None: + notes = meta2notes(meta_path) + notes2midi(notes, midi_path) + print(f"Saved MIDI to {midi_path}") if __name__ == "__main__": import argparse @@ -624,7 +611,8 @@ if __name__ == "__main__": ) parser.add_argument("--meta", type=str, help="Path to metadata JSON") parser.add_argument("--midi", type=str, help="Path to MIDI file") - parser.add_argument("--vocal", type=str, help="Path to vocal wav (for midi2meta)") + parser.add_argument("--vocal", type=str, default=None, help="Path to vocal wav (optional for midi2meta)") + parser.add_argument("--language", type=str, default="Mandarin", help="Lyric language for metadata phoneme conversion (default: Mandarin)") parser.add_argument( "--meta2midi", action="store_true", @@ -633,7 +621,7 @@ if __name__ == "__main__": parser.add_argument( "--midi2meta", action="store_true", - help="Convert midi -> meta (requires --midi, --meta, --vocal, --cut_wavs_dir)", + help="Convert midi -> meta (requires --midi and --meta; --vocal is optional)", ) parser.add_argument( "--rmvpe_model_path", @@ -648,22 +636,20 @@ if __name__ == "__main__": default="cuda", ) args = parser.parse_args() + midi_parser = MidiParser( + rmvpe_model_path=args.rmvpe_model_path, + device=args.device, + ) if args.meta2midi: if not args.meta or not args.midi: parser.error("--meta2midi requires --meta and --midi") - meta2midi(args.meta, args.midi) + midi_parser.meta2midi(args.meta, args.midi) elif args.midi2meta: - if not args.midi or not args.meta or not args.vocal: + if not args.midi or not args.meta: parser.error( - "--midi2meta requires --midi, --meta, --vocal" + "--midi2meta requires --midi and --meta" ) - midi2meta( - args.midi, - args.meta, - args.vocal, - rmvpe_model_path=args.rmvpe_model_path, - device=args.device, - ) + midi_parser.midi2meta(args.midi, args.meta, args.vocal, args.language) else: parser.print_help() \ No newline at end of file diff --git a/soulxsinger/utils/data_processor.py b/soulxsinger/utils/data_processor.py index 50c366e..5dc18a0 100644 --- a/soulxsinger/utils/data_processor.py +++ b/soulxsinger/utils/data_processor.py @@ -42,7 +42,7 @@ class DataProcessor: merged_items = [] duration = [float(x) for x in meta["duration"].split()] - phoneme = [str(x).replace("", "") for i, x in enumerate(meta["phoneme"].split())] + phoneme = [str(x).replace("", "") for x in meta["phoneme"].split()] note_pitch = [int(x) for x in meta["note_pitch"].split()] note_type = [int(x) if phoneme[i] != "" else 1 for i, x in enumerate(meta["note_type"].split())] @@ -52,7 +52,6 @@ class DataProcessor: else: merged_items.append([phoneme[i], duration[i], note_pitch[i], note_type[i]]) - single_frame_duration = self.hop_size / self.sample_rate meta['phoneme'] = [x[0] for x in merged_items] meta['duration'] = [x[1] for x in merged_items] meta['note_pitch'] = [x[2] for x in merged_items] @@ -150,9 +149,9 @@ class DataProcessor: meta["note_type"], ) - f0 = torch.tensor([float(x) for x in meta["f0"].split()]) - min_frame = min(item["mel2note"].shape[1], f0.shape[0]) - item['f0'] = f0[:min_frame].unsqueeze(0).float().to(self.device) + f0 = [float(x) for x in meta.get("f0", "").split()] + min_frame = min(item["mel2note"].shape[1], len(f0)) if len(f0) > 0 else item["mel2note"].shape[1] + item['f0'] = torch.tensor(f0)[:min_frame].unsqueeze(0).float().to(self.device) if len(f0) > 0 else None item["mel2note"] = item["mel2note"][:, :min_frame] if wav_path is not None: diff --git a/webui.py b/webui.py index f9ce397..ac9db6b 100644 --- a/webui.py +++ b/webui.py @@ -1,9 +1,9 @@ import os -import re import random import shutil import sys import traceback +from datetime import datetime from pathlib import Path from typing import Literal, Tuple @@ -14,11 +14,17 @@ import soundfile as sf import gradio as gr from preprocess.pipeline import PreprocessPipeline +from preprocess.tools.midi_parser import MidiParser from soulxsinger.utils.file_utils import load_config from cli.inference import build_model as build_svs_model, process as svs_process ROOT = Path(__file__).parent +SAMPLE_RATE = 44100 +PROMPT_MAX_SEC_DEFAULT = 30 +TARGET_MAX_SEC_DEFAULT = 60 +PROMPT_MAX_MERGE_DURATION_MS = 30000 +TARGET_MAX_MERGE_DURATION_MS = 60000 ENGLISH_EXAMPLE_PROMPT_AUDIO = "example/audio/en_prompt.mp3" ENGLISH_EXAMPLE_PROMPT_META = "example/audio/en_prompt.json" @@ -38,8 +44,6 @@ CANTONESE_EXAMPLE_TARGET_META = "example/audio/yue_target.json" MUSIC_EXAMPLE_TARGET_AUDIO = "example/audio/music.mp3" MUSIC_EXAMPLE_TARGET_META = "example/audio/music.json" -# Lyric language: value (Mandarin/Cantonese/English) is passed to PreprocessPipeline; display labels from i18n via get_lyric_lang_choices() - # Use absolute paths so Examples load correctly (including File components for metadata) EXAMPLES_LIST = [ [ @@ -49,10 +53,10 @@ EXAMPLES_LIST = [ str(ROOT / MANDARIN_EXAMPLE_TARGET_META), "Mandarin", "Mandarin", - "melody", - False, - True, - True, + "melody-controlled", + "no", + "yes", + "yes", 0, ], [ @@ -62,10 +66,10 @@ EXAMPLES_LIST = [ str(ROOT / CANTONESE_EXAMPLE_TARGET_META), "Mandarin", "Cantonese", - "melody", - False, - True, - True, + "melody-controlled", + "no", + "yes", + "yes", 0, ], [ @@ -75,10 +79,10 @@ EXAMPLES_LIST = [ str(ROOT / ENGLISH_EXAMPLE_TARGET_META), "Mandarin", "English", - "melody", - False, - True, - True, + "melody-controlled", + "no", + "yes", + "yes", 0, ], [ @@ -88,37 +92,107 @@ EXAMPLES_LIST = [ str(ROOT / MUSIC_EXAMPLE_TARGET_META), "Mandarin", "Mandarin", - "melody", - False, - True, - True, + "score-controlled", + "no", + "yes", + "yes", 0, ], ] +# i18n +_I18N_KEY2LANG = dict( + display_lang_label=dict(en="Display Language", zh="显示语言"), + section_input_audio=dict(en="Input Audio", zh="输入音频"), + section_transcriptions=dict(en="Transcriptions & Metadata", zh="转录与元数据"), + section_synthesis=dict(en="Singing Synthesis", zh="歌声合成"), + seed_label=dict(en="Seed", zh="种子"), + prompt_audio_label=dict(en=f"Prompt audio (reference voice), limit to {PROMPT_MAX_SEC_DEFAULT} seconds", zh=f"Prompt 音频（参考音色），限制在 {PROMPT_MAX_SEC_DEFAULT} 秒以内"), + target_audio_label=dict(en=f"Target audio (melody / lyrics source), limit to {TARGET_MAX_SEC_DEFAULT} seconds", zh=f"Target 音频（旋律/歌词来源），限制在 {TARGET_MAX_SEC_DEFAULT} 秒以内"), + transcription_btn_label=dict(en="Run singing transcription", zh="开始歌声转录"), + synthesis_btn_label=dict(en="🎤Generate singing voice", zh="🎤歌声合成"), + prompt_meta_label=dict(en="Prompt metadata", zh="Prompt 元数据"), + prompt_midi_label=dict(en="Prompt MIDI", zh="Prompt MIDI"), + target_meta_label=dict(en="Target metadata", zh="Target 元数据"), + target_midi_label=dict(en="Target MIDI", zh="Target MIDI"), + prompt_wav_label=dict(en="Prompt WAV (reference)", zh="Prompt WAV（参考音色）"), + generated_audio_label=dict(en="Generated merged audio", zh="合成结果音频"), + prompt_lyric_lang_label=dict(en="Prompt lyric language", zh="Prompt 歌词语种"), + target_lyric_lang_label=dict(en="Target lyric language", zh="Target 歌词语种"), + lyric_lang_mandarin=dict(en="Mandarin", zh="普通话"), + lyric_lang_cantonese=dict(en="Cantonese", zh="粤语"), + lyric_lang_english=dict(en="English", zh="英语"), + warn_missing_synthesis=dict( + en="Please provide prompt WAV, prompt metadata, and target metadata. Check the content in Transcriptions & Metadata above.", + zh="请提供 Prompt WAV、Prompt metadata 与 Target metadata，并检查上方 Transcriptions & Metadata 里的内容。", + ), + prompt_vocal_sep_label=dict(en="Prompt vocal separation", zh="Prompt人声分离"), + target_vocal_sep_label=dict(en="Target vocal separation", zh="Target人声分离"), + option_yes=dict(en="yes", zh="是"), + option_no=dict(en="no", zh="否"), + auto_shift_label=dict(en="Auto pitch shift", zh="自动变调"), + pitch_shift_label=dict(en="Pitch shift (semitones)", zh="指定变调（半音）"), + control_type_label=dict(en="Control type", zh="控制类型"), + control_melody=dict(en="melody-controlled", zh="旋律控制"), + control_score=dict(en="score-controlled", zh="乐谱控制"), + examples_label=dict(en="Reference examples (click to load)", zh="参考样例（点击加载）"), + example_choice_0=dict(en="—", zh="—"), + example_choice_1=dict(en="Example 1: Mandarin → Mandarin (melody), Start singing synthesis!", zh="样例 1: 普通话 → 普通话 (melody), 开始歌声合成吧!"), + example_choice_2=dict(en="Example 2: Mandarin → Cantonese (melody), Start singing synthesis!", zh="样例 2: 普通话 → 粤语 (melody), 开始歌声合成吧!"), + example_choice_3=dict(en="Example 3: Mandarin → English (melody), Start singing synthesis!", zh="样例 3: 普通话 → 英语 (melody), 开始歌声合成吧!"), + example_choice_4=dict(en="Example 4: Mandarin → Music (score), Start singing synthesis!", zh="样例 4: 普通话 → 音乐 (score), 开始歌声合成吧!"), + instruction_title=dict(en="Usage", zh="使用说明"), + instruction_p1=dict( + en="Upload prompt and target audio, and the corresponding metadata and MIDI files will be automatically transcribed.", + zh="上传 Prompt 与 Target 音频，将自动转录生成 Prompt 与 Target 两份 metadata 文件以及对应的 MIDI 文件。", + ), + instruction_p2=dict( + en="Auto-transcribed lyrics and notes are often misaligned, which may lead to suboptimal synthesis results. For best results, import the generated MIDI into the [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) for manual adjustment. After adjustment, re-upload the MIDI file and the metadata will be automatically updated.", + zh="自动转录的歌词与音高对齐效果通常不理想，可能导致合成效果不佳，建议将生成的 MIDI 文件导入 [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor) 进行手动调整，调整后的 MIDI 文件重新上传后，metadata 将会自动更新。", + ), + instruction_p3=dict( + en="Once prompt audio, prompt metadata, and target metadata are all set, click **🎤Generate singing voice** to run the singing synthesis and generate the final merged audio.", + zh="Prompt audio, Prompt metadata 和 Target metadata 都准备好后，点击「🎤歌声合成」开始最终生成。", + ), +) + +_GLOBAL_LANG: Literal["zh", "en"] = "zh" + + +def _i18n(key: str) -> str: + return _I18N_KEY2LANG[key][_GLOBAL_LANG] + + def _load_example(choice_value): - """Return 11 example values + skip_clear_count (2 when loading example so next 2 audio.change events don't clear metadata). - choice_value: selected dropdown string (or index in older flow); map to example index 0/1/2.""" + """Return 11 example values + skip_clear_count. + + When loading an example, the next two audio.change events should not clear metadata. + """ + output_count = 11 if choice_value is None: - return [gr.update()] * 11 + [0] + return [gr.update()] * output_count + [0] + + choice_to_index = { + _i18n("example_choice_1"): 1, + _i18n("example_choice_2"): 2, + _i18n("example_choice_3"): 3, + _i18n("example_choice_4"): 4, + } + idx = 0 if isinstance(choice_value, int): idx = 0 if choice_value <= 0 else min(choice_value - 1, len(EXAMPLES_LIST) - 1) else: - if choice_value == i18n("example_choice_1"): - idx = 1 - elif choice_value == i18n("example_choice_2"): - idx = 2 - elif choice_value == i18n("example_choice_3"): - idx = 3 - elif choice_value == i18n("example_choice_4"): - idx = 4 + idx = choice_to_index.get(choice_value, 0) + if idx <= 0: - return [gr.update()] * 11 + [0] + return [gr.update()] * output_count + [0] + list_idx = idx - 1 if list_idx >= len(EXAMPLES_LIST): - return [gr.update()] * 11 + [0] + return [gr.update()] * output_count + [0] + row = EXAMPLES_LIST[list_idx] return [ row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9], row[10], @@ -143,11 +217,58 @@ def _get_device() -> str: return "cuda:0" if torch.cuda.is_available() else "cpu" -def _session_dir_from_target(target_audio_path: str) -> Path: - stem = Path(target_audio_path).stem - safe = re.sub(r"[^\w\-]", "_", stem) - safe = re.sub(r"_+", "_", safe).strip("_") or "session" - return ROOT / "outputs" / "gradio" / safe[:64] +def _session_dir() -> Path: + # Use per-call timestamped session dir to avoid cross-request collisions. + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") + return ROOT / "outputs" / "gradio" / timestamp + + +def _print_exception(context: str) -> None: + print(f"[{context}]\n{traceback.format_exc()}", file=sys.stderr, flush=True) + + +def _get_lyric_lang_choices(): + """Lyric language dropdown (display, value) for current UI language.""" + return [ + (_i18n("lyric_lang_mandarin"), "Mandarin"), + (_i18n("lyric_lang_cantonese"), "Cantonese"), + (_i18n("lyric_lang_english"), "English"), + ] + + +def _resolve_file_path(x): + """Gradio file input can be path string or (path, None) tuple.""" + if x is None: + return None + if isinstance(x, tuple): + x = x[0] + return x if (x and os.path.isfile(x)) else None + + +def _normalize_audio_input(audio): + """Normalize Gradio audio input to a filepath string.""" + return audio[0] if isinstance(audio, tuple) else audio + + +def _trim_and_save_audio(src_audio_path: str, dst_wav_path: Path, max_sec: int, sr: int = SAMPLE_RATE) -> None: + """Load audio as mono, trim to max_sec, and save as wav for preprocess.""" + audio_data, _ = librosa.load(src_audio_path, sr=sr, mono=True) + audio_data = audio_data[: max_sec * sr] + sf.write(dst_wav_path, audio_data, sr) + + +def _yes_no_to_bool(value, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + return str(value).strip().lower() == "yes" + + +def _control_to_internal(control: str) -> str: + if control in ("melody", "melody-controlled"): + return "melody" + return "score" class AppState: @@ -168,33 +289,28 @@ class AppState: device=self.device, ) self.phoneset_path = "soulxsinger/utils/phoneme/phone_set.json" + self.midi_parser = MidiParser( + rmvpe_model_path="pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt", + device=self.device + ) def run_preprocess( self, - prompt_path: Path, - target_path: Path, - session_base: Path, - prompt_vocal_sep: bool, - target_vocal_sep: bool, - prompt_lyric_lang: str, - target_lyric_lang: str, + audio_path: Path, + save_path: Path, + vocal_sep: bool, + lyric_lang: str, + max_merge_duration: int ) -> Tuple[bool, str]: try: - self.preprocess_pipeline.save_dir = str(session_base / "transcriptions" / "prompt") + self.preprocess_pipeline.save_dir = str(save_path) self.preprocess_pipeline.run( - audio_path=str(prompt_path), - vocal_sep=prompt_vocal_sep, - max_merge_duration=20000, - language=prompt_lyric_lang or "Mandarin", + audio_path=str(audio_path), + vocal_sep=vocal_sep, + max_merge_duration=max_merge_duration, + language=lyric_lang or "Mandarin", ) - self.preprocess_pipeline.save_dir = str(session_base / "transcriptions" / "target") - self.preprocess_pipeline.run( - audio_path=str(target_path), - vocal_sep=target_vocal_sep, - max_merge_duration=60000, - language=target_lyric_lang or "Mandarin", - ) - return True, "preprocess done" + return True, f"preprocess {audio_path} done" except Exception as e: return False, f"preprocess failed: {e}" @@ -271,191 +387,153 @@ class AppState: APP_STATE = AppState() - -# i18n -_i18n_key2lang_dict = dict( - display_lang_label=dict(en="Display Language", zh="显示语言"), - seed_label=dict(en="Seed", zh="种子"), - prompt_audio_label=dict(en="Prompt audio (reference voice), limit to 30 seconds", zh="Prompt 音频（参考音色），限制在 30 秒以内"), - target_audio_label=dict(en="Target audio (melody / lyrics source), limit to 60 seconds", zh="Target 音频（旋律/歌词来源），限制在 60 秒以内"), - generate_btn_label=dict(en="Start SVS", zh="开始 SVS"), - transcription_btn_label=dict(en="Run singing transcription", zh="开始歌声转录"), - synthesis_btn_label=dict(en="Run singing synthesis", zh="歌声合成"), - prompt_meta_label=dict(en="Prompt metadata", zh="Prompt metadata"), - target_meta_label=dict(en="Target metadata", zh="Target metadata"), - - edit_tutorial_html=dict( - en='

Refer to Edit Tutorial for metadata editing (Important Note: The generated metadata may not perfectly align the singing audio with the corresponding lyrics and musical notes. For better results, we strongly recommend manually correcting the alignment. You can directly use SoulX-Singer-Midi-Editor to edit)

', - zh='

metadata 编辑请参考 编辑教程 (重要提示：自动生成的 metadata 在音频与歌词、音高对齐效果通常不理想。为了获得更好的结果，我们强烈建议手动纠正对齐，否则会导致合成效果不佳。 你可以直接使用 SoulX-Singer-Midi-Editor 进行编辑)

', - ), - prompt_wav_label=dict(en="Prompt WAV (reference)", zh="Prompt WAV（参考音色）"), - generated_audio_label=dict(en="Generated merged audio", zh="合成结果音频"), - prompt_lyric_lang_label=dict(en="Prompt lyric language", zh="Prompt 歌词语种"), - target_lyric_lang_label=dict(en="Target lyric language", zh="Target 歌词语种"), - lyric_lang_mandarin=dict(en="Mandarin", zh="普通话"), - lyric_lang_cantonese=dict(en="Cantonese", zh="粤语"), - lyric_lang_english=dict(en="English", zh="英语"), - warn_missing_synthesis=dict(en="Please provide prompt WAV, prompt metadata, and target metadata", zh="请提供 Prompt WAV、Prompt metadata 与 Target metadata"), - prompt_vocal_sep_label=dict(en="Prompt vocal separation", zh="Prompt人声分离"), - target_vocal_sep_label=dict(en="Target vocal separation", zh="Target人声分离"), - auto_shift_label=dict(en="Auto pitch shift", zh="自动变调"), - pitch_shift_label=dict(en="Pitch shift (semitones)", zh="指定变调（半音）"), - control_type_label=dict(en="Control type", zh="控制类型"), - examples_label=dict(en="Reference examples (click to load)", zh="参考样例（点击加载）"), - example_choice_0=dict(en="—", zh="—"), - example_choice_1=dict(en="Example 1: Mandarin → Mandarin (melody), Start singing synthesis!", zh="样例 1: 普通话 → 普通话 (melody), 开始歌声合成吧!"), - example_choice_2=dict(en="Example 2: Mandarin → Cantonese (melody), Start singing synthesis!", zh="样例 2: 普通话 → 粤语 (melody), 开始歌声合成吧!"), - example_choice_3=dict(en="Example 3: Mandarin → English (melody), Start singing synthesis!", zh="样例 3: 普通话 → 英语 (melody), 开始歌声合成吧!"), - example_choice_4=dict(en="Example 4: Mandarin → Music (score), Start singing synthesis!", zh="样例 4: 普通话 → 音乐 (score), 开始歌声合成吧!"), - warn_missing_audio=dict( - en="Please upload both prompt audio and target audio", - zh="请上传 Prompt 音频与 Target 音频", - ), - # Instruction panel (workflow description) - instruction_title=dict(en="Usage", zh="使用说明"), - instruction_p1=dict( - en="After uploading prompt and target audio and clicking **Run singing transcription**, the system generates two metadata files (prompt and target).", - zh="上传 Prompt 与 Target 音频并点击「开始歌声转录」后，将生成 Prompt 与 Target 两份 metadata 文件。", - ), - instruction_p2=dict( - en="Auto-transcribed lyrics and notes are often misaligned. For better results, import the generated metadata into the **MIDI Editor** for manual adjustment: [SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor).", - zh="自动转录的歌词与音高对齐效果通常不理想，建议将生成的 metadata 导入 **MIDI 编辑器** 进行手动调整：[SoulX-Singer-Midi-Editor](https://huggingface.co/spaces/Soul-AILab/SoulX-Singer-Midi-Editor)。", - ), - instruction_p3=dict( - en="Re-upload the adjusted metadata to the corresponding Prompt / Target Meta fields, then click **Run singing synthesis** to generate the final audio.", - zh="将调整后的 metadata 重新上传至对应的 Prompt / Target Meta 位置后，点击「歌声合成」开始最终生成。", - ), -) - -def _detect_initial_lang() -> Literal["zh", "en"]: - """Detect initial UI language from server locale (browser language applied later via JS).""" - try: - import locale - loc = (locale.getdefaultlocale()[0] or os.environ.get("LANG", "") or "").lower() - return "en" if loc.startswith("en") else "zh" - except Exception: - return "zh" - - -global_lang: Literal["zh", "en"] = _detect_initial_lang() - - -def i18n(key: str) -> str: - return _i18n_key2lang_dict[key][global_lang] - - -def get_lyric_lang_choices(): - """Lyric language dropdown (display, value) for current UI language.""" - return [ - (i18n("lyric_lang_mandarin"), "Mandarin"), - (i18n("lyric_lang_cantonese"), "Cantonese"), - (i18n("lyric_lang_english"), "English"), - ] - - -def _resolve_file_path(x): - """Gradio file input can be path string or (path, None) tuple.""" - if x is None: - return None - if isinstance(x, tuple): - x = x[0] - return x if (x and os.path.isfile(x)) else None - - -def transcription_function( - prompt_audio, - target_audio, - prompt_metadata, - target_metadata, - prompt_lyric_lang: str, - target_lyric_lang: str, - prompt_vocal_sep: bool, - target_vocal_sep: bool, +def _edit_metadata( + meta, + midi, + audio, + language: str = "Mandarin", ): - """Step 1: Run transcription only; output (prompt_meta_path, target_meta_path).""" try: - if isinstance(prompt_audio, tuple): - prompt_audio = prompt_audio[0] - if isinstance(target_audio, tuple): - target_audio = target_audio[0] - if prompt_audio is None or target_audio is None: - gr.Warning(message=i18n("warn_missing_audio")) - return None, None + meta = _resolve_file_path(meta) + midi = _resolve_file_path(midi) + if not midi: + return meta + audio = _normalize_audio_input(audio) + + if not meta: + meta = str(Path(midi).with_name("metadata.json")) + + APP_STATE.midi_parser.midi2meta(midi, meta, audio, language=language) + return meta + except Exception: + _print_exception("_edit_metadata") + return meta + + +def _transcribe_prompt( + prompt_audio, + prompt_metadata, + prompt_lyric_lang: str, + prompt_vocal_sep, + prompt_max_sec: int = PROMPT_MAX_SEC_DEFAULT, +): + try: + prompt_audio = _normalize_audio_input(prompt_audio) prompt_meta_resolved = _resolve_file_path(prompt_metadata) - target_meta_resolved = _resolve_file_path(target_metadata) - use_input_metadata = prompt_meta_resolved is not None and target_meta_resolved is not None - - session_base = _session_dir_from_target(target_audio) - audio_dir = session_base / "audio" - audio_dir.mkdir(parents=True, exist_ok=True) - transfer_prompt_path = audio_dir / "prompt.wav" - transfer_target_path = audio_dir / "target.wav" - SR = 44100 - PROMPT_MAX_SEC = 30 - TARGET_MAX_SEC = 60 - prompt_audio_data, _ = librosa.load(prompt_audio, sr=SR, mono=True) - target_audio_data, _ = librosa.load(target_audio, sr=SR, mono=True) - prompt_audio_data = prompt_audio_data[: PROMPT_MAX_SEC * SR] - target_audio_data = target_audio_data[: TARGET_MAX_SEC * SR] - sf.write(transfer_prompt_path, prompt_audio_data, SR) - sf.write(transfer_target_path, target_audio_data, SR) + if prompt_audio is None and prompt_meta_resolved is None: + return None, None + session_base = _session_dir() prompt_meta_path = session_base / "transcriptions" / "prompt" / "metadata.json" - target_meta_path = session_base / "transcriptions" / "target" / "metadata.json" - if use_input_metadata: - (session_base / "transcriptions" / "prompt").mkdir(parents=True, exist_ok=True) - (session_base / "transcriptions" / "target").mkdir(parents=True, exist_ok=True) - shutil.copy2(prompt_meta_resolved, prompt_meta_path) - shutil.copy2(target_meta_resolved, target_meta_path) - else: - ok, msg = APP_STATE.run_preprocess( - transfer_prompt_path, - transfer_target_path, - session_base, - prompt_vocal_sep=prompt_vocal_sep, - target_vocal_sep=target_vocal_sep, - prompt_lyric_lang=prompt_lyric_lang or "Mandarin", - target_lyric_lang=target_lyric_lang or "Mandarin", + prompt_midi_path = session_base / "transcriptions" / "prompt" / "vocal.mid" + + if prompt_audio is not None: + audio_dir = session_base / "audio" + audio_dir.mkdir(parents=True, exist_ok=True) + transfer_prompt_path = audio_dir / "prompt.wav" + _trim_and_save_audio(prompt_audio, transfer_prompt_path, prompt_max_sec) + + prompt_ok, prompt_msg = APP_STATE.run_preprocess( + audio_path=transfer_prompt_path, + save_path=session_base / "transcriptions" / "prompt", + vocal_sep=_yes_no_to_bool(prompt_vocal_sep, default=False), + lyric_lang=prompt_lyric_lang or "Mandarin", + max_merge_duration=PROMPT_MAX_MERGE_DURATION_MS, ) - if not ok: - print(msg, file=sys.stderr, flush=True) + if not prompt_ok: + print(prompt_msg, file=sys.stderr, flush=True) return None, None + elif prompt_meta_resolved is not None: + (session_base / "transcriptions" / "prompt").mkdir(parents=True, exist_ok=True) + shutil.copy2(prompt_meta_resolved, prompt_meta_path) + + if prompt_meta_path.exists(): + APP_STATE.midi_parser.meta2midi(prompt_meta_path, prompt_midi_path) prompt_meta_file = str(prompt_meta_path) if prompt_meta_path.exists() else None - target_meta_file = str(target_meta_path) if target_meta_path.exists() else None - return prompt_meta_file, target_meta_file + prompt_midi_file = str(prompt_midi_path) if prompt_midi_path.exists() else None + return prompt_meta_file, prompt_midi_file except Exception: - print(traceback.format_exc(), file=sys.stderr, flush=True) + _print_exception("_transcribe_prompt") return None, None -def synthesis_function( +def _transcribe_target( + target_audio, + target_metadata, + target_lyric_lang: str, + target_vocal_sep, + target_max_sec: int = TARGET_MAX_SEC_DEFAULT, +): + try: + target_audio = _normalize_audio_input(target_audio) + target_meta_resolved = _resolve_file_path(target_metadata) + if target_audio is None and target_meta_resolved is None: + return None, None, None + + session_base = _session_dir() + target_meta_path = session_base / "transcriptions" / "target" / "metadata.json" + target_midi_path = session_base / "transcriptions" / "target" / "vocal.mid" + target_vocal_path = session_base / "transcriptions" / "target" / "vocal.wav" + + if target_audio is not None: + audio_dir = session_base / "audio" + audio_dir.mkdir(parents=True, exist_ok=True) + transfer_target_path = audio_dir / "target.wav" + _trim_and_save_audio(target_audio, transfer_target_path, target_max_sec) + + target_ok, target_msg = APP_STATE.run_preprocess( + audio_path=transfer_target_path, + save_path=session_base / "transcriptions" / "target", + vocal_sep=_yes_no_to_bool(target_vocal_sep, default=True), + lyric_lang=target_lyric_lang or "Mandarin", + max_merge_duration=TARGET_MAX_MERGE_DURATION_MS, + ) + if not target_ok: + print(target_msg, file=sys.stderr, flush=True) + return None, None, None + elif target_meta_resolved is not None: + (session_base / "transcriptions" / "target").mkdir(parents=True, exist_ok=True) + shutil.copy2(target_meta_resolved, target_meta_path) + + if target_meta_path.exists(): + APP_STATE.midi_parser.meta2midi(target_meta_path, target_midi_path) + + target_meta_file = str(target_meta_path) if target_meta_path.exists() else None + target_midi_file = str(target_midi_path) if target_midi_path.exists() else None + target_vocal_file = str(target_vocal_path) if target_vocal_path.exists() else None + return target_meta_file, target_midi_file, target_vocal_file + except Exception: + _print_exception("_transcribe_target") + return None, None, None + + +def _run_synthesis( prompt_audio, prompt_metadata, target_metadata, control: str, - auto_shift: bool, + auto_shift, pitch_shift, seed: int, ): - """Step 2: Run SVS from top prompt_audio + prompt_metadata + target_metadata.""" + """Run singing synthesis from prompt audio + prompt metadata + target metadata.""" try: - if isinstance(prompt_audio, tuple): - prompt_audio = prompt_audio[0] + prompt_audio = _normalize_audio_input(prompt_audio) prompt_wav_path = prompt_audio prompt_meta_path = _resolve_file_path(prompt_metadata) target_meta_path = _resolve_file_path(target_metadata) if not prompt_wav_path or not os.path.isfile(prompt_wav_path): - gr.Warning(message=i18n("warn_missing_synthesis")) + gr.Warning(message=_i18n("warn_missing_synthesis")) return None if not prompt_meta_path or not os.path.isfile(prompt_meta_path): - gr.Warning(message=i18n("warn_missing_synthesis")) + gr.Warning(message=_i18n("warn_missing_synthesis")) return None if not target_meta_path or not os.path.isfile(target_meta_path): - gr.Warning(message=i18n("warn_missing_synthesis")) + gr.Warning(message=_i18n("warn_missing_synthesis")) return None - if control not in ("melody", "score"): - control = "score" + control = _control_to_internal(control) + auto_shift = _yes_no_to_bool(auto_shift, default=True) seed = int(seed) torch.manual_seed(seed) np.random.seed(seed) @@ -473,16 +551,16 @@ def synthesis_function( return None return str(merged) except Exception: - print(traceback.format_exc(), file=sys.stderr, flush=True) + _print_exception("_run_synthesis") return None def _instruction_md() -> str: """Markdown content for the instruction panel (supports links).""" return "\n\n".join([ - f"**1.** {i18n('instruction_p1')}", - f"**2.** {i18n('instruction_p2')}", - f"**3.** {i18n('instruction_p3')}", + f"**1.** {_i18n('instruction_p1')}", + f"**2.** {_i18n('instruction_p2')}", + f"**3.** {_i18n('instruction_p3')}", ]) @@ -511,153 +589,156 @@ def render_interface() -> gr.Blocks: '">' '' ) - # Auto-detect browser language: run after Gradio mounts - gr.HTML( - '', - visible=False, - ) with gr.Row(equal_height=True): lang_choice = gr.Radio( choices=["中文", "English"], value="中文", - label=i18n("display_lang_label"), + label=_i18n("display_lang_label"), type="index", interactive=True, elem_id="lang_choice_radio", ) # Instruction panel (usage workflow); updates on language change - instruction_md = gr.Markdown(f"### {i18n('instruction_title')}\n\n{_instruction_md()}") + instruction_md = gr.Markdown(f"### {_i18n('instruction_title')}\n\n{_instruction_md()}") # Reference examples — at the front of operations (handler registered after components exist) skip_clear_metadata_count = gr.State(0) with gr.Row(): - _example_choices = [i18n("example_choice_0"), i18n("example_choice_1"), i18n("example_choice_2"), i18n("example_choice_3"), i18n("example_choice_4")] + _example_choices = [_i18n("example_choice_0"), _i18n("example_choice_1"), _i18n("example_choice_2"), _i18n("example_choice_3"), _i18n("example_choice_4")] example_choice = gr.Dropdown( - label=i18n("examples_label"), + label=_i18n("examples_label"), choices=_example_choices, value=_example_choices[0], interactive=True, ) # Step 1: Transcription (audio → metadata) - with gr.Row(equal_height=True): - with gr.Column(scale=1): - prompt_audio = gr.Audio( - label=i18n("prompt_audio_label"), - type="filepath", - editable=False, + with gr.Accordion(_i18n("section_input_audio"), open=True) as accordion_input_audio: + with gr.Row(equal_height=True): + with gr.Column(scale=1): + prompt_audio = gr.Audio( + label=_i18n("prompt_audio_label"), + type="filepath", + editable=False, + interactive=True, + ) + with gr.Column(scale=1): + target_audio = gr.Audio( + label=_i18n("target_audio_label"), + type="filepath", + editable=False, + interactive=True, + ) + with gr.Accordion(_i18n("section_transcriptions"), open=True) as accordion_transcriptions: + with gr.Row(equal_height=True): + prompt_lyric_lang = gr.Dropdown( + label=_i18n("prompt_lyric_lang_label"), + choices=_get_lyric_lang_choices(), + value="Mandarin", interactive=True, + scale=1, ) - with gr.Column(scale=1): - target_audio = gr.Audio( - label=i18n("target_audio_label"), - type="filepath", - editable=False, + prompt_vocal_sep = gr.Dropdown( + label=_i18n("prompt_vocal_sep_label"), + choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")], + value="no", interactive=True, + scale=1, + ) + target_lyric_lang = gr.Dropdown( + label=_i18n("target_lyric_lang_label"), + choices=_get_lyric_lang_choices(), + value="Mandarin", + interactive=True, + scale=1, + ) + target_vocal_sep = gr.Dropdown( + label=_i18n("target_vocal_sep_label"), + choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")], + value="yes", + interactive=True, + scale=1, ) - with gr.Row(equal_height=True): - prompt_lyric_lang = gr.Dropdown( - label=i18n("prompt_lyric_lang_label"), - choices=get_lyric_lang_choices(), - value="Mandarin", - interactive=True, - scale=1, - ) - target_lyric_lang = gr.Dropdown( - label=i18n("target_lyric_lang_label"), - choices=get_lyric_lang_choices(), - value="Mandarin", - interactive=True, - scale=1, - ) - prompt_vocal_sep = gr.Checkbox( - label=i18n("prompt_vocal_sep_label"), - value=False, - interactive=True, - scale=1, - ) - target_vocal_sep = gr.Checkbox( - label=i18n("target_vocal_sep_label"), - value=True, - interactive=True, - scale=1, - ) - with gr.Row(): - transcription_btn = gr.Button( - value=i18n("transcription_btn_label"), - variant="primary", - size="lg", - ) - # Edit tutorial link (gr.HTML supports links; component labels do not) - metadata_tutorial_html = gr.HTML(value=i18n("edit_tutorial_html")) - # Synthesis: params row, then synthesis button on next row - with gr.Row(equal_height=True): - prompt_metadata = gr.File( - label=i18n("prompt_meta_label"), - type="filepath", - file_types=[".json"], - interactive=True, - ) - target_metadata = gr.File( - label=i18n("target_meta_label"), - type="filepath", - file_types=[".json"], - interactive=True, - ) - control_radio = gr.Radio( - choices=["melody", "score"], - value="score", - label=i18n("control_type_label"), - scale=1, - ) - auto_shift = gr.Checkbox( - label=i18n("auto_shift_label"), - value=True, - interactive=True, - scale=1, - ) - pitch_shift = gr.Number( - label=i18n("pitch_shift_label"), - value=0, - minimum=-36, - maximum=36, - step=1, - interactive=True, - scale=1, - ) - seed_input = gr.Number( - label=i18n("seed_label"), - value=12306, - step=1, - interactive=True, - scale=1, - ) - with gr.Row(): - synthesis_btn = gr.Button( - value=i18n("synthesis_btn_label"), - variant="primary", - size="lg", - ) - with gr.Row(): - output_audio = gr.Audio( - label=i18n("generated_audio_label"), - type="filepath", - interactive=False, - ) + with gr.Row(equal_height=True): + prompt_metadata = gr.File( + label=_i18n("prompt_meta_label"), + type="filepath", + file_types=[".json"], + height=140, + interactive=True, + ) + prompt_midi = gr.File( + label=_i18n("prompt_midi_label"), + type="filepath", + file_types=[".midi", ".mid"], + height=140, + interactive=True, + ) + target_metadata = gr.File( + label=_i18n("target_meta_label"), + type="filepath", + file_types=[".json"], + height=140, + interactive=True, + ) + target_midi = gr.File( + label=_i18n("target_midi_label"), + type="filepath", + file_types=[".midi", ".mid"], + height=140, + interactive=True, + ) + target_vocal = gr.File( + type="filepath", + file_types=[".wav"], + interactive=False, + visible=False, + ) + with gr.Accordion(_i18n("section_synthesis"), open=True) as accordion_synthesis: + with gr.Row(equal_height=True): + control_radio = gr.Dropdown( + choices=[(_i18n("control_melody"), "melody-controlled"), (_i18n("control_score"), "score-controlled")], + value="score-controlled", + label=_i18n("control_type_label"), + scale=1, + ) + auto_shift = gr.Dropdown( + label=_i18n("auto_shift_label"), + choices=[(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")], + value="yes", + interactive=True, + scale=1, + ) + pitch_shift = gr.Number( + label=_i18n("pitch_shift_label"), + value=0, + minimum=-36, + maximum=36, + step=1, + interactive=True, + scale=1, + ) + seed_input = gr.Number( + label=_i18n("seed_label"), + value=12306, + step=1, + interactive=True, + scale=1, + ) + with gr.Row(): + synthesis_btn = gr.Button( + value=_i18n("synthesis_btn_label"), + variant="primary", + size="lg", + ) + with gr.Row(): + output_audio = gr.Audio( + label=_i18n("generated_audio_label"), + type="filepath", + interactive=False, + ) example_choice.change( fn=_load_example, @@ -679,33 +760,36 @@ def render_interface() -> gr.Blocks: ) def _change_component_language(lang): - global global_lang - global_lang = ["zh", "en"][lang] - choices = get_lyric_lang_choices() + global _GLOBAL_LANG + _GLOBAL_LANG = ["zh", "en"][lang] + lyric_choices = _get_lyric_lang_choices() + yes_no_choices = [(_i18n("option_yes"), "yes"), (_i18n("option_no"), "no")] + control_choices = [(_i18n("control_melody"), "melody-controlled"), (_i18n("control_score"), "score-controlled")] return [ - gr.update(label=i18n("prompt_audio_label")), - gr.update(label=i18n("target_audio_label")), - gr.update(label=i18n("prompt_lyric_lang_label"), choices=choices), - gr.update(label=i18n("target_lyric_lang_label"), choices=choices), - gr.update(label=i18n("prompt_vocal_sep_label")), - gr.update(label=i18n("target_vocal_sep_label")), - gr.update(value=i18n("transcription_btn_label")), - gr.update(label=i18n("prompt_meta_label")), - gr.update(label=i18n("target_meta_label")), - gr.update(value=i18n("edit_tutorial_html")), - gr.update(label=i18n("control_type_label")), - gr.update(label=i18n("auto_shift_label")), - gr.update(label=i18n("pitch_shift_label")), - gr.update(label=i18n("seed_label")), - gr.update(value=i18n("synthesis_btn_label")), - gr.update(label=i18n("generated_audio_label")), - gr.update(label=i18n("display_lang_label")), + gr.update(label=_i18n("prompt_audio_label")), + gr.update(label=_i18n("target_audio_label")), + gr.update(label=_i18n("prompt_lyric_lang_label"), choices=lyric_choices), + gr.update(label=_i18n("target_lyric_lang_label"), choices=lyric_choices), + gr.update(label=_i18n("prompt_vocal_sep_label"), choices=yes_no_choices), + gr.update(label=_i18n("target_vocal_sep_label"), choices=yes_no_choices), + gr.update(label=_i18n("prompt_meta_label")), + gr.update(label=_i18n("target_meta_label")), + gr.update(label=_i18n("control_type_label"), choices=control_choices), + gr.update(label=_i18n("auto_shift_label"), choices=yes_no_choices), + gr.update(label=_i18n("pitch_shift_label")), + gr.update(label=_i18n("seed_label")), + gr.update(value=_i18n("synthesis_btn_label")), + gr.update(label=_i18n("generated_audio_label")), + gr.update(label=_i18n("display_lang_label")), gr.update( - label=i18n("examples_label"), - choices=[i18n("example_choice_0"), i18n("example_choice_1"), i18n("example_choice_2"), i18n("example_choice_3"), i18n("example_choice_4")], - value=i18n("example_choice_0"), + label=_i18n("examples_label"), + choices=[_i18n("example_choice_0"), _i18n("example_choice_1"), _i18n("example_choice_2"), _i18n("example_choice_3"), _i18n("example_choice_4")], + value=_i18n("example_choice_0"), ), - gr.update(value=f"### {i18n('instruction_title')}\n\n{_instruction_md()}"), + gr.update(value=f"### {_i18n('instruction_title')}\n\n{_instruction_md()}"), + gr.update(label=_i18n("section_input_audio")), + gr.update(label=_i18n("section_transcriptions")), + gr.update(label=_i18n("section_synthesis")), ] lang_choice.change( @@ -718,10 +802,8 @@ def render_interface() -> gr.Blocks: target_lyric_lang, prompt_vocal_sep, target_vocal_sep, - transcription_btn, prompt_metadata, target_metadata, - metadata_tutorial_html, control_radio, auto_shift, pitch_shift, @@ -731,6 +813,9 @@ def render_interface() -> gr.Blocks: lang_choice, example_choice, instruction_md, + accordion_input_audio, + accordion_transcriptions, + accordion_synthesis, ], ) @@ -740,29 +825,34 @@ def render_interface() -> gr.Blocks: inputs=[prompt_audio, skip_clear_metadata_count], outputs=[prompt_metadata, skip_clear_metadata_count], ) + prompt_audio.upload( + fn=_transcribe_prompt, + inputs=[prompt_audio, prompt_metadata, prompt_lyric_lang, prompt_vocal_sep], + outputs=[prompt_metadata, prompt_midi], + ) + prompt_midi.upload( + fn=_edit_metadata, + inputs=[prompt_metadata, prompt_midi, prompt_audio, prompt_lyric_lang], + outputs=[prompt_metadata], + ) target_audio.change( fn=_clear_target_meta_unless_example, inputs=[target_audio, skip_clear_metadata_count], outputs=[target_metadata, skip_clear_metadata_count], ) - - transcription_btn.click( - fn=transcription_function, - inputs=[ - prompt_audio, - target_audio, - prompt_metadata, - target_metadata, - prompt_lyric_lang, - target_lyric_lang, - prompt_vocal_sep, - target_vocal_sep, - ], - outputs=[prompt_metadata, target_metadata], + target_audio.upload( + fn=_transcribe_target, + inputs=[target_audio, target_metadata, target_lyric_lang, target_vocal_sep], + outputs=[target_metadata, target_midi, target_vocal], + ) + target_midi.upload( + fn=_edit_metadata, + inputs=[target_metadata, target_midi, target_vocal, target_lyric_lang], + outputs=[target_metadata], ) synthesis_btn.click( - fn=synthesis_function, + fn=_run_synthesis, inputs=[ prompt_audio, prompt_metadata,