Skip to content

Commit 807bcd5

Browse files
LauraGPTLauraGPT
authored
feat: print transcription text in the binaries (embed vocab, in-binary detok) (#3004)
SenseVoice and Paraformer binaries now output text directly (whisper-cli style), no Python detok step. - export_{sensevoice,paraformer}_gguf.py: embed the vocab as a GGUF string array (sv.vocab = SentencePiece pieces; pf.vocab = tokens.json) so the binary is self-contained. - funasr-sensevoice.cpp: CTC ids -> text (pieces + ▁->space), drop <|...|> meta by default; --ids (raw), --keep-tags (keep meta tags). - funasr-paraformer.cpp: ids -> text (tokens.json, drop @@ continuations); --ids for raw. - both fall back to ids if the GGUF has no vocab. READMEs updated to text output. Verified: C++ text == reference detok (sentencepiece / tokens.json) on sample and the full 002 clip; ids unchanged so CER is unchanged.
1 parent 3610b75 commit 807bcd5

6 files changed

Lines changed: 63 additions & 14 deletions

File tree

‎runtime/llama.cpp/paraformer/README.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ python runtime/llama.cpp/export_paraformer_gguf.py --wtype f16 \
6060

6161
**3. Transcribe:**
6262
```bash
63-
build/bin/llama-funasr-paraformer -m paraformer.gguf -a audio.wav > ids.txt
64-
python runtime/llama.cpp/detok.py <model>/tokens.json ids.txt
63+
build/bin/llama-funasr-paraformer -m paraformer.gguf -a audio.wav # prints transcription text (--ids for raw)
6564
```
6665
Expected output:
6766
```

‎runtime/llama.cpp/paraformer/export_paraformer_gguf.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ def main():
2020
ap = argparse.ArgumentParser()
2121
ap.add_argument("--model_pt", required=True); ap.add_argument("--mvn", required=True)
2222
ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16"])
23+
ap.add_argument("--tokens", default=None, help="tokens.json (id->token); default: next to model_pt")
2324
a = ap.parse_args()
2425
sd = torch.load(a.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)
2526
w = gguf.GGUFWriter(a.out, "paraformer")
@@ -28,6 +29,13 @@ def main():
2829
w.add_uint32("pf.dec.num_blocks", 16); w.add_uint32("pf.dec.att_layer_num", 16)
2930
w.add_uint32("pf.dec.decoders3", 1); w.add_uint32("pf.dec.attention_heads", 4)
3031
w.add_uint32("pf.dec.kernel_size", 11); w.add_uint32("pf.vocab_size", 8404)
32+
import json, glob
33+
tp = a.tokens or (glob.glob(os.path.join(os.path.dirname(a.model_pt), "tokens.json")) + [None])[0]
34+
if tp and os.path.exists(tp):
35+
toks = json.load(open(tp)); w.add_array("pf.vocab", toks)
36+
print(f"embedded pf.vocab ({len(toks)} tokens) from {tp}")
37+
else:
38+
print("WARNING: tokens.json not found - gguf will have no vocab (binary falls back to ids)")
3139
w.add_float32("pf.predictor.tail_threshold", 0.45)
3240
w.add_float32("pf.predictor.threshold", 1.0)
3341
shift, scale = parse_mvn(a.mvn)

‎runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,20 +119,32 @@ static std::vector<float> run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*i
119119
int D=out->ne[0],N=out->ne[1];std::vector<float>r((size_t)D*N);ggml_backend_tensor_get(out,r.data(),0,ggml_nbytes(out));
120120
ggml_gallocr_free(ga);ggml_backend_free(be);return r;}
121121

122+
// Paraformer detok: tokens.json is BPE; join, drop "@@" continuations, "▁"(U+2581)->space.
123+
static std::string pf_trim(const std::string&s){size_t a=s.find_first_not_of(' ');if(a==std::string::npos)return "";size_t b=s.find_last_not_of(' ');return s.substr(a,b-a+1);}
124+
static std::string detok_pf(const std::vector<int>&ids,const std::vector<std::string>&vocab){
125+
std::string s; for(int id:ids){ if(id==1||id==2)continue; if(id>=0&&id<(int)vocab.size())s+=vocab[id]; }
126+
size_t p; while((p=s.find("@@"))!=std::string::npos)s.erase(p,2);
127+
const std::string lb="\xe2\x96\x81"; while((p=s.find(lb))!=std::string::npos)s.replace(p,3," ");
128+
return pf_trim(s);
129+
}
130+
122131
int main(int argc,char**argv){
123-
std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000;
132+
std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false;
124133
for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
125134
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
126135
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
127-
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}}
136+
else if(!strcmp(argv[i],"--ids"))ids_mode=true;
137+
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids]\n",argv[0]);return 1;}}
128138
if(gguf_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
129139
model m;gguf_init_params gp={false,&m.ctx_w};gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);if(!gg){fprintf(stderr,"gguf load failed\n");return 1;}
130140
auto rdi=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};
131141
auto rdf=[&](const char*k,float d){int i=gguf_find_key(gg,k);return i<0?d:gguf_get_val_f32(gg,i);};
132142
m.c.enc_blocks=rdi("pf.enc.num_blocks",50);m.c.dec_blocks=rdi("pf.dec.num_blocks",16);m.c.dec_att=rdi("pf.dec.att_layer_num",16);
133143
m.c.dec3=rdi("pf.dec.decoders3",1);m.c.vocab=rdi("pf.vocab_size",8404);m.c.tail=rdf("pf.predictor.tail_threshold",0.45f);m.c.thresh=rdf("pf.predictor.threshold",1.0f);
144+
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
134145
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);
135146
const int D=m.c.d_model,F=560,V=m.c.vocab;
147+
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
136148

137149
float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;
138150
float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j]
@@ -183,7 +195,10 @@ int main(int argc,char**argv){
183195
x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]
184196
ggml_set_output(x);
185197
logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}
186-
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}
198+
std::vector<int> seg_ids; seg_ids.reserve(N);
199+
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}seg_ids.push_back(am);}
200+
if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
201+
else { std::string t=detok_pf(seg_ids,vocab); printf("%s",t.c_str()); }
187202
};
188203

189204
int64_t t0=ggml_time_us();

‎runtime/llama.cpp/sensevoice/README.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,12 @@ python runtime/llama.cpp/export_sensevoice_gguf.py --wtype f16 \
6161

6262
**3. Transcribe:**
6363
```bash
64-
build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf -a audio.wav > ids.txt
65-
python runtime/llama.cpp/detok.py <model>/chn_jpn_yue_eng_ko_spectok.bpe.model ids.txt
64+
build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf -a audio.wav # prints transcription text
65+
# --keep-tags keeps the <|lang|>/<|emotion|>/<|event|> tags; --ids prints raw CTC ids
6666
```
6767
Expected output:
6868
```
69-
<|zh|><|NEUTRAL|><|Speech|><|woitn|>我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗
70-
[sensevoice] N=746 (q4+T742) encode 1.32s
69+
我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗
7170
```
7271
The leading `<|...|>` tags are the predicted language / emotion / event / ITN.
7372

‎runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def main():
2323
ap.add_argument("--mvn", required=True)
2424
ap.add_argument("--out", required=True)
2525
ap.add_argument("--wtype", default="f32", choices=["f32", "f16"])
26+
ap.add_argument("--spm", default=None, help="sentencepiece .bpe.model; default: next to model_pt")
2627
args = ap.parse_args()
2728

2829
sd = torch.load(args.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)
@@ -36,7 +37,17 @@ def main():
3637
w.add_uint32("sv.vocab_size", 25055)
3738
w.add_uint32("sv.blank_id", 0)
3839
# query token embed indices used at inference: [lid(auto=0), 1, 2, textnorm(woitn=15)]
39-
w.add_array("sv.query_tokens", [0, 1, 2, 15])
40+
w.add_array("sv.query_tokens", [0, 1, 2, 14]) # 14=withitn (use_itn=True), matches authoritative
41+
import glob
42+
spm_path = args.spm or (glob.glob(os.path.join(os.path.dirname(args.model_pt), "*.bpe.model")) + [None])[0]
43+
if spm_path and os.path.exists(spm_path):
44+
import sentencepiece as spm
45+
sp = spm.SentencePieceProcessor(model_file=spm_path)
46+
pieces = [sp.id_to_piece(i) for i in range(sp.get_piece_size())]
47+
w.add_array("sv.vocab", pieces)
48+
print(f"embedded sv.vocab ({len(pieces)} pieces) from {spm_path}")
49+
else:
50+
print("WARNING: *.bpe.model not found - gguf will have no vocab (binary falls back to ids)")
4051

4152
shift, scale = parse_mvn(args.mvn)
4253
w.add_tensor("cmvn.shift", shift) # (560,)

‎runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,27 @@ static void add_posenc(std::vector<float>&x,int T,int depth){
9191
x[(size_t)t*depth+i]+=(float)sin(st);x[(size_t)t*depth+depth/2+i]+=(float)cos(st);}}
9292
}
9393

94+
// SenseVoice detok: sentencepiece pieces (no byte-fallback in this vocab) -> join,
95+
// "▁"(U+2581)->space; meta tokens <|lang|>/<|emo|>/<|event|>/<|itn|> dropped unless --keep-tags.
96+
static std::string sv_trim(const std::string&s){size_t a=s.find_first_not_of(' ');if(a==std::string::npos)return "";size_t b=s.find_last_not_of(' ');return s.substr(a,b-a+1);}
97+
static std::string detok_sv(const std::vector<int>&ids,const std::vector<std::string>&vocab,bool keep_tags){
98+
std::string s; for(int id:ids){ if(id<0||id>=(int)vocab.size())continue; const std::string&p=vocab[id];
99+
if(!keep_tags && p.size()>=2 && p[0]=='<' && p[1]=='|') continue; // skip <|...|> meta
100+
s+=p; }
101+
const std::string lb="\xe2\x96\x81"; size_t pp; while((pp=s.find(lb))!=std::string::npos)s.replace(pp,3," ");
102+
return sv_trim(s);
103+
}
104+
94105
int main(int argc,char**argv){
95-
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000;
106+
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false;
96107
for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];
97108
else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];
98109
else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
99110
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
100111
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
101-
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;} }
112+
else if(!strcmp(argv[i],"--ids"))ids_mode=true;
113+
else if(!strcmp(argv[i],"--keep-tags"))keep_tags=true;
114+
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
102115
if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}
103116

104117
// load model
@@ -110,9 +123,11 @@ int main(int argc,char**argv){
110123
m.c.kernel=rd("sv.kernel_size",11); m.c.vocab=rd("sv.vocab_size",25055); m.c.blank=rd("sv.blank_id",0);
111124
int qi=gguf_find_key(gg,"sv.query_tokens"); int nq=qi<0?0:(int)gguf_get_arr_n(gg,qi);
112125
std::vector<int> qtok(nq); for(int i=0;i<nq;i++) qtok[i]=((const int32_t*)gguf_get_arr_data(gg,qi))[i];
126+
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
113127
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}
114128
gguf_free(gg);
115129
const int F=560, D=m.c.d_model, V=m.c.vocab;
130+
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
116131

117132
// NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder;
118133
// it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it
@@ -139,10 +154,12 @@ int main(int argc,char**argv){
139154
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,8);
140155
if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}
141156
std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));
142-
int prev=-1; // greedy CTC: argmax per frame -> collapse consecutive -> drop blank
157+
std::vector<int> seg_ids; int prev=-1; // greedy CTC: argmax per frame -> collapse -> drop blank
143158
for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
144159
for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}
145-
if(am!=prev && am!=m.c.blank) printf("%d ", am); prev=am; }
160+
if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; }
161+
if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
162+
else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); }
146163
ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);
147164
};
148165

0 commit comments

Comments
 (0)