fix(llama.py): 适配cuda13.x及更新的库版本

cli/inference.py

@@ -114,7 +114,8 @@ def process(args, config, model: torch.nn.Module):
             )
 
         generated_audio = generated_audio.squeeze().cpu().numpy()
-        generated_merged[start_sample_idx : start_sample_idx + generated_audio.shape[0]] = generated_audio
+        gen_len = min(generated_audio.shape[0], generated_merged.shape[0] - start_sample_idx)
+        generated_merged[start_sample_idx: start_sample_idx + gen_len] = generated_audio[:gen_len]
 
     merged_path = os.path.join(args.save_dir, "generated.wav")
     sf.write(merged_path, generated_merged, 24000)

preprocess/requirements.txt

@@ -23,6 +23,7 @@ sageattention==1.0.6
 scikit_learn==1.7.2
 scipy==1.15.3
 six==1.17.0
+setuptools==81.0.0
 scikit_image==0.25.2
 soundfile==0.13.1
 ToJyutping==3.2.0

preprocess/tools/f0_extraction.py

@@ -516,12 +516,12 @@ class F0Extractor:
 
 if __name__ == "__main__":
     model_path = (
-        "pretrained_models/rmvpe/rmvpe.pt"
+        "pretrained_models/SoulX-Singer-Preprocess/rmvpe/rmvpe.pt"
     )
-    audio_path = "./outputs/transcription/test.wav"
+    audio_path = "example/audio/zh_prompt.mp3"
 
     pe = F0Extractor(
         model_path,
         device="cuda",
     )
-    f0 = pe.process(audio_path)
+    f0 = pe.process(audio_path, f0_path="example/audio/zh_prompt_f0.npy")

preprocess/tools/lyric_transcription.py

@@ -120,9 +120,10 @@ class _ASRZhModel:
         raw_timestamps = [[t[0] / 1000, t[1] / 1000] for t in out["timestamp"]]
         words, word_durs = _build_words_with_gaps(raw_words, raw_timestamps, wav_fn)
 
-        if os.path.exists(wav_fn.replace(".wav", "_f0.npy")):
+        f0_path = os.path.splitext(wav_fn)[0] + "_f0.npy"
+        if os.path.exists(f0_path):
             words, word_durs = _word_dur_post_process(
-                words, word_durs, np.load(wav_fn.replace(".wav", "_f0.npy"))
+                words, word_durs, np.load(f0_path)
             )
 
         return words, word_durs
@@ -179,9 +180,10 @@ class _ASREnModel:
 
         words, durs = _build_words_with_gaps(raw_words, raw_timestamps, wav_fn)
 
-        if os.path.exists(wav_fn.replace(".wav", "_f0.npy")):
+        f0_path = os.path.splitext(wav_fn)[0] + "_f0.npy"
+        if os.path.exists(f0_path):
             words, durs = _word_dur_post_process(
-                words, durs, np.load(wav_fn.replace(".wav", "_f0.npy"))
+                words, durs, np.load(f0_path)
             )
 
         return words, durs
@@ -248,8 +250,10 @@ class LyricTranscriber:
             if self.en_model is None:
                 # Lazy-load NeMo model only when English is actually used.
                 if v:
-                    print("[lyric transcription] init English ASR, please make sure NeMo is installed")
+                    print("[lyric transcription] init English ASR start, please make sure NeMo is installed and wait for a while")
                 self.en_model = _ASREnModel(model_path=self.en_model_path, device=self.device)
+                if v:
+                    print("[lyric transcription] init English ASR success")
             out = self.en_model.process(wav_fn)
         else:
             out = self.zh_model.process(wav_fn)
@@ -271,9 +275,9 @@ class LyricTranscriber:
 
 if __name__ == "__main__":
     m = LyricTranscriber(
-        zh_model_path="pretrained_models/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
-        en_model_path="pretrained_models/parakeet-tdt-0.6b-v2/parakeet-tdt-0.6b-v2.nemo",
+        zh_model_path="pretrained_models/SoulX-Singer-Preprocess/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+        en_model_path="pretrained_models/SoulX-Singer-Preprocess/parakeet-tdt-0.6b-v2/parakeet-tdt-0.6b-v2.nemo",
         device="cuda"
     )
-    print(m.process("example/test/asr_zh.wav", language="Mandarin"))
-    print(m.process("example/test/asr_en.wav", language="English"))
+    print(m.process("example/audio/zh_prompt.mp3", language="Mandarin"))
+    print(m.process("example/audio/en_prompt.mp3", language="English"))

preprocess/tools/note_transcription/model.py

@@ -509,14 +509,23 @@ class NoteTranscriber:
 
 if __name__ == "__main__":
 
-    items = json.load(open("example/test/rosvot_input.json", "r"))
-    item = items[0]
+    item = {
+        'item_name': 'vocal_0', 
+        'wav_fn': 'example/audio/zh_prompt.mp3', 
+        'start_time_ms': 320, 
+        'end_time_ms': 10687, 
+        'origin_wav_fn': 'example/audio/zh_prompt.mp3', 
+        'duration': 10367, 
+        'words': ['<SP>', '除', '了', '想', '你', '<SP>', '除', '了', '爱', '你', '<SP>', '我', '什', '么', '什', '么', '都', '愿', '意'], 
+        'word_durs': [0.21, 0.36, 0.26, 0.7000000000000001, 0.96, 0.3800000000000001, 0.43999999999999995, 0.3799999999999999, 0.6400000000000001, 0.9600000000000002, 1.1199999999999999, 0.28000000000000025, 0.3799999999999999, 0.3199999999999994, 0.3200000000000003, 0.3799999999999999, 0.3200000000000003, 0.5, 1.457981859410431], 
+        'language': 'Mandarin'
+    }
 
     m = NoteTranscriber(
-        rosvot_model_path="pretrained_models/rosvot/rosvot/model.pt", 
-        rwbd_model_path="pretrained_models/rosvot/rwbd/model.pt", 
+        rosvot_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rosvot/model.pt", 
+        rwbd_model_path="pretrained_models/SoulX-Singer-Preprocess/rosvot/rwbd/model.pt", 
         device="cuda"
     )
-    out = m.process(item)
+    out = m.process(item, segment_info=item)
 
     print(out)

preprocess/tools/vocal_detection.py

@@ -289,6 +289,9 @@ class VocalDetector:
             key = f"{base_name_no_ext}_{idx}"
             start_sample = librosa.time_to_samples(start_ms / 1000, sr=sr)
             end_sample = librosa.time_to_samples(end_ms / 1000, sr=sr)
+            # Use Python int to avoid 32-bit overflow on Windows
+            start_sample = int(start_sample)
+            end_sample = int(end_sample)
             segment = y[start_sample:end_sample]
 
             write(f"{self.cut_wavs_output_dir}/{key}.wav", segment, sr)
@@ -316,5 +319,5 @@ class VocalDetector:
 
 if __name__ == "__main__":
     m = VocalDetector(cut_wavs_output_dir="outputs/transcription/cut_wavs")
-    segment_infos = m.process("./outputs/transcription/test.wav")
+    segment_infos = m.process("example/audio/zh_prompt.mp3", np.load("example/audio/zh_prompt_f0.npy"))
     print(segment_infos)

preprocess/tools/vocal_separation/model.py

@@ -226,12 +226,12 @@ class VocalSeparator:
 if __name__ == "__main__":
     
     m = VocalSeparator(
-        sep_model_path="pretrained_models/mel-band-roformer-karaoke/mel_band_roformer_karaoke_becruily.ckpt",
-        sep_config_path="pretrained_models/mel-band-roformer-karaoke/config_karaoke_becruily.yaml",
-        der_model_path="pretrained_models/dereverb_mel_band_roformer/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt",
-        der_config_path="pretrained_models/dereverb_mel_band_roformer/dereverb_mel_band_roformer_anvuew.yaml",
+        sep_model_path="pretrained_models/SoulX-Singer-Preprocess/mel-band-roformer-karaoke/mel_band_roformer_karaoke_becruily.ckpt",
+        sep_config_path="pretrained_models/SoulX-Singer-Preprocess/mel-band-roformer-karaoke/config_karaoke_becruily.yaml",
+        der_model_path="pretrained_models/SoulX-Singer-Preprocess/dereverb_mel_band_roformer/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt",
+        der_config_path="pretrained_models/SoulX-Singer-Preprocess/dereverb_mel_band_roformer/dereverb_mel_band_roformer_anvuew.yaml",
         device="cuda"
     )
 
-    out = m.process("example/test/separation_test.mp3")
+    out = m.process("example/audio/zh_prompt.mp3")
     print(out.vocals_dereverbed.shape)

preprocess/utils.py

@@ -109,10 +109,11 @@ def _merge_group(
     )
 
     # ---------- time & audio ----------
-    start_ms = get_attr(segments[0], "start_time_ms", 0)
-    end_ms = get_attr(segments[-1], "end_time_ms", 0) + end_extension_ms
-    start_sample = start_ms * sample_rate // 1000
-    end_sample = end_ms * sample_rate // 1000
+    # Use Python int to avoid 32-bit overflow on Windows
+    start_ms = int(get_attr(segments[0], "start_time_ms", 0))
+    end_ms = int(get_attr(segments[-1], "end_time_ms", 0)) + end_extension_ms
+    start_sample = int(start_ms * sample_rate // 1000)
+    end_sample = int(end_ms * sample_rate // 1000)
 
     # ---------- naming ----------
     first_item_name = get_attr(segments[0], "item_name", "segment")
@@ -195,12 +196,11 @@ def merge_short_segments(
 
     for seg in segments:
         if isinstance(seg, dict):
-            start_time = seg.get("start_time_ms", 0)
-            end_time = seg.get("end_time_ms", 0)
+            start_time = int(seg.get("start_time_ms", 0))
+            end_time = int(seg.get("end_time_ms", 0))
         else:
-            start_time = seg.start_time_ms
-            end_time = seg.end_time_ms
-        
+            start_time = int(seg.start_time_ms)
+            end_time = int(seg.end_time_ms)
         if (
             current_group
             and (start_time - prev_end > max_gap_ms

requirements.txt

@@ -26,6 +26,7 @@ sageattention==1.0.6
 scikit_learn==1.7.2
 scipy==1.15.3
 six==1.17.0
+setuptools==81.0.0
 scikit_image==0.25.2
 soundfile==0.13.1
 ToJyutping==3.2.0

soulxsinger/models/modules/llama.py

@@ -56,7 +56,7 @@ class LlamaNARDecoderLayer(LlamaDecoderLayer):
             config.hidden_size, eps=config.rms_norm_eps, dim_cond=config.hidden_size
         )
 
-    # add `cond` in forward function
+    # 修改点：添加 position_embeddings 参数和 **kwargs
     def forward(
         self,
         hidden_states: torch.Tensor,
@@ -66,38 +66,39 @@ class LlamaNARDecoderLayer(LlamaDecoderLayer):
         past_key_value: Optional[Tuple[torch.Tensor]] = None,
         output_attentions: Optional[bool] = False,
         use_cache: Optional[bool] = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs,
     ) -> Tuple[
         torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
     ]:
-        """
-        Args:
-            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
-            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
-                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
-            output_attentions (`bool`, *optional*):
-                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
-                returned tensors for more detail.
-            use_cache (`bool`, *optional*):
-                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
-                (see `past_key_values`).
-            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
-        """
-
         residual = hidden_states
 
         hidden_states = self.input_layernorm(
             hidden_states, cond_embedding=cond_embedding
         )
 
-        # Self Attention
-        hidden_states, self_attn_weights, present_key_value = self.self_attn(
+        # [MODIFIED] 兼容新版 transformers 的返回值解包逻辑
+        # 因为新版在 output_attentions=False 时可能只返回 (hidden_states, present_key_value)
+        attn_outputs = self.self_attn(
             hidden_states=hidden_states,
             attention_mask=attention_mask,
             position_ids=position_ids,
             past_key_value=past_key_value,
             output_attentions=output_attentions,
             use_cache=use_cache,
+            position_embeddings=position_embeddings,
         )
+
+        hidden_states = attn_outputs[0]
+        
+        # 动态处理权重和 KV 缓存的解包
+        if output_attentions:
+            self_attn_weights = attn_outputs[1]
+            present_key_value = attn_outputs[2] if use_cache else None
+        else:
+            self_attn_weights = None
+            present_key_value = attn_outputs[1] if use_cache else None
+
         hidden_states = residual + hidden_states
 
         # Fully Connected
@@ -129,7 +130,15 @@ class DiffLlama(LlamaModel):
         dropout=0.1,
         ffn_dropout=0.1,
         attention_dropout=0.0,
-        config=LlamaConfig(0, 256, 1024, 1, 1),
+        config=LlamaConfig(
+            vocab_size=0, 
+            hidden_size=256, 
+            intermediate_size=1024, 
+            num_hidden_layers=1, 
+            num_attention_heads=4,   # 确保维度对齐 (256/4=64)
+            num_key_value_heads=4,
+            attn_implementation="eager",
+        ),
     ):
         super().__init__(config)
 
@@ -293,6 +302,13 @@ class DiffLlama(LlamaModel):
         else:
             position_ids = position_ids.view(-1, seq_length).long()
 
+        # ==============================================================
+        # [NEW] 修复位置：为适配新版 transformers 生成 position_embeddings
+        # ==============================================================
+        # self.rotary_emb 继承自 LlamaModel
+        position_embeddings = self.rotary_emb(hidden_states if 'hidden_states' in locals() else inputs_embeds, position_ids)
+        # ==============================================================
+
         # embed positions
         if attention_mask is None:
             attention_mask = torch.ones(
@@ -300,6 +316,9 @@ class DiffLlama(LlamaModel):
                 dtype=torch.bool,
                 device=inputs_embeds.device,
             )
+        
+        # 处理 attention_mask 格式
+        # 注意：某些新版本 transformers 此处返回值结构有变，如报错请检查此函数
         attention_mask = self._prepare_decoder_attention_mask(
             attention_mask,
             (batch_size, seq_length),
@@ -329,23 +348,11 @@ class DiffLlama(LlamaModel):
             )
 
             if self.gradient_checkpointing and self.training:
+                # 原始代码此处有 NotImplementedError，如果需要开启梯度检查点，
+                # 也需要将 position_embeddings 传入 custom_forward
                 raise NotImplementedError
-
-                def create_custom_forward(module):
-                    def custom_forward(*inputs):
-                        # None for past_key_value
-                        return module(*inputs, output_attentions, None)
-
-                    return custom_forward
-
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    create_custom_forward(decoder_layer),
-                    hidden_states,
-                    attention_mask,
-                    position_ids,
-                    None,
-                )
             else:
+                # [MODIFIED] 在调用 decoder_layer 时传入 position_embeddings
                 layer_outputs = decoder_layer(
                     hidden_states,
                     attention_mask=attention_mask,
@@ -354,6 +361,7 @@ class DiffLlama(LlamaModel):
                     output_attentions=output_attentions,
                     use_cache=use_cache,
                     cond_embedding=diffusion_step,
+                    position_embeddings=position_embeddings, # 新增参数
                 )
 
             hidden_states = layer_outputs[0]
@@ -375,18 +383,10 @@ class DiffLlama(LlamaModel):
 
         hidden_states = self.mel_out_mlp(hidden_states)
 
-        # if not return_dict:
-        #     return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
-        # return BaseModelOutputWithPast(
-        #     last_hidden_state=hidden_states,
-        #     past_key_values=next_cache,
-        #     hidden_states=all_hidden_states,
-        #     attentions=all_self_attns,
-        # )
         if return_dict:
             return {
                 "output": hidden_states,
                 "hidden_states": all_layer_hidden_states,
             }
 
-        return hidden_states
+        return hidden_states

webui.py

@@ -3,6 +3,7 @@ import random
 import shutil
 import sys
 import traceback
+import gc
 from datetime import datetime
 from pathlib import Path
 from typing import Literal, Tuple
@@ -312,6 +313,9 @@ class AppState:
                 max_merge_duration=max_merge_duration,
                 language=lyric_lang or "Mandarin",
             )
+            gc.collect()
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
             return True, f"preprocess {audio_path} done"
         except Exception as e:
             return False, f"preprocess failed: {e}"
@@ -346,6 +350,9 @@ class AppState:
         args.use_fp16 = self.use_fp16
         try:
             svs_process(args, self.svs_config, self.svs_model)
+            gc.collect()
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
             generated = save_dir / "generated.wav"
             if not generated.exists():
                 return False, f"inference finished but {generated} not found", None, prompt_meta_path, target_meta_path

webui_svc.py

@@ -1,6 +1,7 @@
 import random
 import sys
 import traceback
+import gc
 from datetime import datetime
 from pathlib import Path
 from typing import Literal
@@ -165,6 +166,9 @@ class AppState:
 			vocal_f0 = save_path / "vocal_f0.npy"
 			if not vocal_wav.exists() or not vocal_f0.exists():
 				return False, f"preprocess output missing: {vocal_wav} or {vocal_f0}", None, None
+			gc.collect()
+			if torch.cuda.is_available():
+				torch.cuda.empty_cache()
 			return True, "ok", vocal_wav, vocal_f0
 		except Exception as e:
 			return False, f"preprocess failed: {e}", None, None
@@ -241,7 +245,9 @@ class AppState:
 						mixed_path = save_dir / "generated_mixed.wav"
 						sf.write(str(mixed_path), mixed, mix_sr)
 						generated = mixed_path
-
+			gc.collect()
+			if torch.cuda.is_available():
+				torch.cuda.empty_cache()
 			return True, "svc inference done", generated
 		except Exception as e:
 			return False, f"svc inference failed: {e}", None