multimodalart HF Staff commited on
Commit
ec2c57c
·
verified ·
1 Parent(s): f466359

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +86 -26
app.py CHANGED
@@ -4,6 +4,12 @@ Pipeline: 9B speech-translation LM (ST LM, frozen) -> 26M Hidden2CVMapper
4
  (lang-conditioned) -> CosyVoice3 0.5B (frozen, source audio as voice prompt).
5
  6 directions: zh->en/es/ja + en->zh/es/ja. Source language is auto-detected
6
  from the ST transcript (CJK chars => zh source, else en source).
 
 
 
 
 
 
7
  """
8
  import spaces # MUST come before any torch / CUDA-touching import
9
 
@@ -12,7 +18,6 @@ import sys
12
  import time
13
 
14
  import gradio as gr
15
- import numpy as np
16
  import torch
17
  from huggingface_hub import snapshot_download
18
 
@@ -20,12 +25,83 @@ REPO_ID = "IndexTeam/Index-Echo-S2ST-9B"
20
  HERE = os.path.dirname(os.path.abspath(__file__))
21
  MODEL_DIR = os.path.join(HERE, "dubbing_9b_fulldir_cv3")
22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  print(f"[app] downloading {REPO_ID} snapshot (~31 GB, first boot only)...", flush=True)
24
  _t0 = time.time()
25
  snapshot_download(
26
  repo_id=REPO_ID,
27
  local_dir=MODEL_DIR,
28
  max_workers=16,
 
29
  )
30
  print(f"[app] snapshot done in {time.time() - _t0:.0f}s", flush=True)
31
 
@@ -38,24 +114,7 @@ print("[app] loading pipeline (ST LM 9B + mapper + CosyVoice3)...", flush=True)
38
  _t0 = time.time()
39
  model = DubbingBridgeModel.from_pretrained(MODEL_DIR, device="cuda")
40
  print(f"[app] pipeline ready in {time.time() - _t0:.0f}s", flush=True)
41
-
42
- # ---- ZeroGPU disk hygiene -------------------------------------------------
43
- # The weights are now packed to disk by ZeroGPU; the raw snapshot blobs are no
44
- # longer needed. Keep everything that is read lazily at inference time:
45
- # wetext FSTs (wetext_* / wetext_repo), vendored ja_ext, BlankEN tokenizer
46
- # files, configs and the sample wavs. Unlinking is safe on Linux: mmapped
47
- # weights stay valid after unlink.
48
- for root, _dirs, files in os.walk(MODEL_DIR, topdown=True):
49
- if os.sep + "ja_ext" + os.sep in root + os.sep or "wetext" in root:
50
- continue
51
- for fn in files:
52
- if fn.endswith(".safetensors"):
53
- p = os.path.join(root, fn)
54
- try:
55
- os.unlink(p)
56
- print(f"[app] freed {p}", flush=True)
57
- except OSError:
58
- pass
59
 
60
  SAMPLE_RATE = model.sample_rate # 24000
61
  LANG_NAMES = {"en": "English", "es": "Spanish", "ja": "Japanese", "zh": "Chinese"}
@@ -71,12 +130,12 @@ def dub(audio_path: str, target_lang: str):
71
  """Dub source speech into a target language, preserving the original voice.
72
 
73
  Args:
74
- audio_path: path to source speech (Chinese or English, ≤ 30 s).
75
- target_lang: target language — "English", "Spanish", "Japanese",
76
- "Chinese" (or the code en/es/ja/zh).
77
 
78
  Returns:
79
- (audio tuple (sample_rate, waveform), source transcript markdown,
80
  translation markdown) for the dubbed speech.
81
  """
82
  if not audio_path:
@@ -89,11 +148,12 @@ def dub(audio_path: str, target_lang: str):
89
  wav, sr, info = model.dub(audio_path, lang=lang, return_info=True)
90
  except AssertionError as e:
91
  msg = str(e) or "unsupported dubbing direction"
92
- if "30" in msg or "support" in msg:
93
  raise gr.Error("Input audio is too long — please keep it under 30 seconds.")
94
  raise gr.Error(
95
- f"This direction is not supported. The source language was detected "
96
- f"as the same as the target. Supported: zh→en/es/ja, en→zh/es/ja. ({msg})"
 
97
  )
98
  elapsed = time.time() - t0
99
  src_text, tgt_raw, tgt_cv = info["zh"], info["tgt_raw"], info["tgt_cv"]
 
4
  (lang-conditioned) -> CosyVoice3 0.5B (frozen, source audio as voice prompt).
5
  6 directions: zh->en/es/ja + en->zh/es/ja. Source language is auto-detected
6
  from the ST transcript (CJK chars => zh source, else en source).
7
+
8
+ Disk strategy: the raw snapshot is ~31 GB and the ZeroGPU weight pack adds
9
+ ~25 GB, which together would exceed the Space's ephemeral disk. Each weight
10
+ blob is therefore unlinked from disk as soon as its consumer has read it into
11
+ RAM (safe on Linux: open mmaps and even fully-read files keep working after
12
+ unlink), so the pack never overlaps with the raw snapshot on disk.
13
  """
14
  import spaces # MUST come before any torch / CUDA-touching import
15
 
 
18
  import time
19
 
20
  import gradio as gr
 
21
  import torch
22
  from huggingface_hub import snapshot_download
23
 
 
25
  HERE = os.path.dirname(os.path.abspath(__file__))
26
  MODEL_DIR = os.path.join(HERE, "dubbing_9b_fulldir_cv3")
27
 
28
+ # ---- interposition: unlink weight blobs once loaded into RAM --------------
29
+ _CONSUMED = set()
30
+
31
+
32
+ def _unlink(path):
33
+ """Delete a fully-read weight blob inside MODEL_DIR; mappings stay valid."""
34
+ if not os.path.realpath(path).startswith(os.path.realpath(MODEL_DIR) + os.sep):
35
+ return
36
+ try:
37
+ size_gb = os.path.getsize(path) / 1e9
38
+ os.unlink(path)
39
+ _CONSUMED.add(path)
40
+ print(f"[app] freed {os.path.basename(path)} ({size_gb:.2f} GB)", flush=True)
41
+ except OSError:
42
+ pass
43
+
44
+
45
+ def _install_progressive_cleanup():
46
+ """Patch the loaders the vendored pipeline uses so each safetensors/torch
47
+ blob is unlinked right after its contents reach RAM."""
48
+ import safetensors.torch as st
49
+
50
+ _orig_load_file = st.load_file
51
+
52
+ def load_file_and_unlink(filename, **kw):
53
+ out = _orig_load_file(filename, **kw)
54
+ if isinstance(filename, str) and filename.endswith(".safetensors"):
55
+ _unlink(filename)
56
+ return out
57
+
58
+ st.load_file = load_file_and_unlink
59
+ # code/stlm/stlm_api.py and code/stlm/audio_model.py and CV3 model.py do
60
+ # `from safetensors.torch import load_file`, i.e. bind a fresh reference
61
+ # at import time; patching the module attr before those imports run is
62
+ # enough because they happen later, inside DubPipeline.__init__.
63
+
64
+ _orig_torch_load = torch.load
65
+
66
+ def torch_load_and_unlink(f, *a, **kw):
67
+ out = _orig_torch_load(f, *a, **kw)
68
+ if isinstance(f, str) and f.endswith(".pt"):
69
+ _unlink(f)
70
+ return out
71
+
72
+ torch.load = torch_load_and_unlink
73
+
74
+ # transformers 5.6 keeps the mmap open on from_pretrained safetensors
75
+ # blobs (stlm_llm 17.9 GB via AutoModelForCausalLM, CosyVoice-BlankEN
76
+ # 1 GB via Qwen2ForCausalLM); unlink right after each call returns —
77
+ # the open mappings stay valid on Linux.
78
+ from transformers import PreTrainedModel
79
+
80
+ _orig_from_pretrained = PreTrainedModel.from_pretrained.__func__
81
+
82
+ def from_pretrained_and_unlink(cls, path, *a, **kw):
83
+ out = _orig_from_pretrained(cls, path, *a, **kw)
84
+ try:
85
+ if os.path.isdir(str(path)):
86
+ f = os.path.join(path, "model.safetensors")
87
+ if os.path.isfile(f):
88
+ _unlink(f)
89
+ except OSError:
90
+ pass
91
+ return out
92
+
93
+ PreTrainedModel.from_pretrained = classmethod(from_pretrained_and_unlink)
94
+
95
+
96
+ _install_progressive_cleanup()
97
+
98
  print(f"[app] downloading {REPO_ID} snapshot (~31 GB, first boot only)...", flush=True)
99
  _t0 = time.time()
100
  snapshot_download(
101
  repo_id=REPO_ID,
102
  local_dir=MODEL_DIR,
103
  max_workers=16,
104
+ ignore_patterns=["bridge/mapper_best.pt", "__pycache__/*", "*.pyc"],
105
  )
106
  print(f"[app] snapshot done in {time.time() - _t0:.0f}s", flush=True)
107
 
 
114
  _t0 = time.time()
115
  model = DubbingBridgeModel.from_pretrained(MODEL_DIR, device="cuda")
116
  print(f"[app] pipeline ready in {time.time() - _t0:.0f}s", flush=True)
117
+ print(f"[app] unlinked {len(_CONSUMED)} raw blobs", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
 
119
  SAMPLE_RATE = model.sample_rate # 24000
120
  LANG_NAMES = {"en": "English", "es": "Spanish", "ja": "Japanese", "zh": "Chinese"}
 
130
  """Dub source speech into a target language, preserving the original voice.
131
 
132
  Args:
133
+ audio_path: path to source speech (Chinese or English, <= 30 s).
134
+ target_lang: target language — "English", "Spanish", "Japanese" or
135
+ "Chinese" (a code en/es/ja/zh also works).
136
 
137
  Returns:
138
+ (audio (sample_rate, waveform), source transcript markdown,
139
  translation markdown) for the dubbed speech.
140
  """
141
  if not audio_path:
 
148
  wav, sr, info = model.dub(audio_path, lang=lang, return_info=True)
149
  except AssertionError as e:
150
  msg = str(e) or "unsupported dubbing direction"
151
+ if "30" in msg or "extract speech token" in msg:
152
  raise gr.Error("Input audio is too long — please keep it under 30 seconds.")
153
  raise gr.Error(
154
+ "This direction is not supported (the source language was detected "
155
+ "as the same as the target). Supported: zh→en/es/ja, en→zh/es/ja. "
156
+ f"({msg})"
157
  )
158
  elapsed = time.time() - t0
159
  src_text, tgt_raw, tgt_cv = info["zh"], info["tgt_raw"], info["tgt_cv"]