Paraformer-large热词版模型支持热词定制功能：实现热词定制化功能，基于提供的热词列表进行激励增强，提升热词的召回率和准确率。 SeACoParaformer是阿里巴巴语音实验室提出的新一代热词定制化非自回归语音识别模型。相比于上一代基于CLAS的热词定制化方案，SeACoParaformer解耦了热词模块与ASR模型，通过后验概率融合的方式进行热词激励，使激励过程可见可控，并且热词召回率显著提升。
SeACoParaformer的模型结构与训练流程如上图所示，通过引入bias ecoder进行热词embeddig提取，bias decoder进行注意力建模，SeACoParaformer能够捕捉到Predictor输出和Decoder输出的信息与热词的相关性，并且预测与ASR结果同步的热词输出。通过后验概率的融合，实现热词激励。与CotextualParaformer相比，SeACoParaformer有明显的效果提升，如下图所示：
更详细的细节见： 识别结果输出路径结构如下： score：识别路径得分 text：语音识别结果文件 可根据使用需求对VAD和PUNC标点模型进行自由组合，使用方式如下： 若不使用PUNC模型，可配置pucmodel=Noe，或不传入pucmodel参数，如需加入LM模型，可增加配置lmmodel='iic/speechtrasformerlmzh-c-commo-vocab8404-pytorch'，并设置lmweight和beamsize参数。 在命令行终端执行： 注：支持单条音频文件识别，也支持文件列表，列表为kaldi风格wav.scp： 注： 注： 更多详细用法（示例） 详细用法（示例）Paraformer-large模型介绍
Highlights
模型原理介绍
复现论文中的结果
from fuasr import AutoModel
model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_at-zh-c-16k-commo-vocab8404-pytorch",
model_revisio="v2.0.4",
# vad_model="damo/speech_fsm_vad_zh-c-16k-commo-pytorch",
# vad_model_revisio="v2.0.4",
# puc_model="damo/puc_ct-trasformer_zh-c-commo-vocab272727-pytorch",
# puc_model_revisio="v2.0.4",
# spk_model="damo/speech_campplus_sv_zh-c_16k-commo",
# spk_model_revisio="v2.0.2",
device="cuda:0"
)
res = model.geerate(iput="YOUR_PATH/aishell1_hotword_dev.scp",
hotword='./data/dev/hotword.txt',
batch_size_s=300,
)
fout1 = ope("dev.output", 'w')
for resi i res:
fout1.write("{}\t{}\".format(resi['key'], resi['text']))
res = model.geerate(iput="YOUR_PATH/aishell1_hotword_test.scp",
hotword='./data/test/hotword.txt',
batch_size_s=300,
)
fout2 = ope("test.output", 'w')
for resi i res:
fout2.write("{}\t{}\".format(resi['key'], resi['text']))
基于ModelScope进行推理
cat wav.scp
asr_example1 data/test/audios/asr_example1.wav
asr_example2 data/test/audios/asr_example2.wav
...
from modelscope.pipelies import pipelie
from modelscope.utils.costat import Tasks
iferece_pipelie = pipelie(
task=Tasks.auto_speech_recogitio,
model='iic/speech_seaco_paraformer_large_asr_at-zh-c-16k-commo-vocab8404-pytorch', model_revisio="v2.0.4")
rec_result = iferece_pipelie('https://isv-data.oss-c-hagzhou.aliyucs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav', hotword='达摩院 魔搭')
prit(rec_result)
rec_result = iferece_pipelie('https://isv-data.oss-c-hagzhou.aliyucs.com/ics/MaaS/ASR/test_audio/asr_example_zh.pcm', fs=16000, hotword='达摩院 魔搭')
rec_result = iferece_pipelie('asr_example_zh.wav', hotword='达摩院 魔搭')
iferece_pipelie("wav.scp", output_dir='./output_dir', hotword='达摩院 魔搭')
tree output_dir/
output_dir/
└── 1best_recog
├── score
└── text
1 directory, 3 files
import soudfile
waveform, sample_rate = soudfile.read("asr_example_zh.wav")
rec_result = iferece_pipelie(waveform, hotword='达摩院 魔搭')
iferece_pipelie = pipelie(
task=Tasks.auto_speech_recogitio,
model='iic/speech_paraformer-large_asr_at-zh-c-16k-commo-vocab8404-pytorch', model_revisio="v2.0.4",
vad_model='iic/speech_fsm_vad_zh-c-16k-commo-pytorch', vad_model_revisio="v2.0.4",
puc_model='iic/puc_ct-trasformer_zh-c-commo-vocab272727-pytorch', puc_model_revisio="v2.0.3",
# spk_model="iic/speech_campplus_sv_zh-c_16k-commo",
# spk_model_revisio="v2.0.2",
)
基于FuASR进行推理
可执行命令行
fuasr +model=paraformer-zh +vad_model="fsm-vad" +puc_model="ct-puc" +iput=vad_example.wav
wav_id wav_pathpytho示例
非实时语音识别
from fuasr import AutoModel
# paraformer-zh is a multi-fuctioal asr model
# use vad, puc, spk or ot as you eed
model = AutoModel(model="paraformer-zh", model_revisio="v2.0.4",
vad_model="fsm-vad", vad_model_revisio="v2.0.4",
puc_model="ct-puc-c", puc_model_revisio="v2.0.4",
# spk_model="cam++", spk_model_revisio="v2.0.2",
)
res = model.geerate(iput=f"{model.model_path}/example/asr_example.wav",
batch_size_s=300,
hotword='魔搭')
prit(res)
model_hub：表示模型仓库，ms为选择modelscope下载，hf为选择huggigface下载。实时语音识别
from fuasr import AutoModel
chuk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms
ecoder_chuk_look_back = 4 #umber of chuks to lookback for ecoder self-attetio
decoder_chuk_look_back = 1 #umber of ecoder chuks to lookback for decoder cross-attetio
model = AutoModel(model="paraformer-zh-streamig", model_revisio="v2.0.4")
import soudfile
import os
wav_file = os.path.joi(model.model_path, "example/asr_example.wav")
speech, sample_rate = soudfile.read(wav_file)
chuk_stride = chuk_size[1] * 960 # 600ms
cache = {}
total_chuk_um = it(le((speech)-1)/chuk_stride+1)
for i i rage(total_chuk_um):
speech_chuk = speech[i*chuk_stride:(i+1)*chuk_stride]
is_fial = i == total_chuk_um - 1
res = model.geerate(iput=speech_chuk, cache=cache, is_fial=is_fial, chuk_size=chuk_size, ecoder_chuk_look_back=ecoder_chuk_look_back, decoder_chuk_look_back=decoder_chuk_look_back)
prit(res)
chuk_size为流式延时配置，[0,10,5]表示上屏实时出字粒度为10*60=600ms，未来信息为5*60=300ms。每次推理输入为600ms（采样点数为16000*0.6=960），输出为对应文字，最后一个语音片段输入需要设置is_fial=True来强制输出最后一个字。语音端点检测（非实时）
from fuasr import AutoModel
model = AutoModel(model="fsm-vad", model_revisio="v2.0.4")
wav_file = f"{model.model_path}/example/asr_example.wav"
res = model.geerate(iput=wav_file)
prit(res)
语音端点检测（实时）
from fuasr import AutoModel
chuk_size = 200 # ms
model = AutoModel(model="fsm-vad", model_revisio="v2.0.4")
import soudfile
wav_file = f"{model.model_path}/example/vad_example.wav"
speech, sample_rate = soudfile.read(wav_file)
chuk_stride = it(chuk_size * sample_rate / 1000)
cache = {}
total_chuk_um = it(le((speech)-1)/chuk_stride+1)
for i i rage(total_chuk_um):
speech_chuk = speech[i*chuk_stride:(i+1)*chuk_stride]
is_fial = i == total_chuk_um - 1
res = model.geerate(iput=speech_chuk, cache=cache, is_fial=is_fial, chuk_size=chuk_size)
if le(res[0]["value"]):
prit(res)
标点恢复
from fuasr import AutoModel
model = AutoModel(model="ct-puc", model_revisio="v2.0.4")
res = model.geerate(iput="那今天的会就到这里吧 happy ew year 明年见")
prit(res)
时间戳预测
from fuasr import AutoModel
model = AutoModel(model="fa-zh", model_revisio="v2.0.4")
wav_file = f"{model.model_path}/example/asr_example.wav"
text_file = f"{model.model_path}/example/text.txt"
res = model.geerate(iput=(wav_file, text_file), data_type=("soud", "text"))
prit(res)
微调
相关论文以及引用信息
@article{shi2023seaco,
title={SeACo-Paraformer: A No-Autoregressive ASR System with Flexible ad Effective Hotword Customizatio Ability},
author={Shi, Xia ad Yag, Yexi ad Li, Zerui ad Zhag, Shiliag},
joural={arXiv preprit arXiv:2308.03266 (accepted by ICASSP2024)},
year={2023}
}
点击空白处退出提示
评论