From b09cbbdb5fe246cd273d8a31fe77471b81e28cd2 Mon Sep 17 00:00:00 2001 From: Lesereingrape Date: Sat, 3 Oct 2026 16:53:44 +0800 Subject: [PATCH] fix(auto): skip blank lines when reading wav.scp and jsonl file lists A blank or whitespace-only line in a file list crashes prepare_data_iterator: the id/data branch indexes lines[0] on an empty split (IndexError), and the .jsonl branch feeds an empty string to json.loads (JSONDecodeError). Skip blank lines so a trailing or separator line no longer aborts the whole batch. --- funasr/auto/auto_model.py | 2 ++ tests/test_pcm_input_format.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index e814cba84..8f985832a 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -469,6 +469,8 @@ def prepare_data_iterator( if file_extension in filelist: # filelist: wav.scp, file.jsonl;text.txt; with open(data_in, encoding="utf-8") as fin: for line in fin: + if not line.strip(): + continue key = "rand_key_" + "".join(random.choice(chars) for _ in range(13)) if data_in.endswith(".jsonl"): # file.jsonl: json.dumps({"source": data}) lines = json.loads(line.strip()) diff --git a/tests/test_pcm_input_format.py b/tests/test_pcm_input_format.py index 94e32a192..5f4cbbdc4 100644 --- a/tests/test_pcm_input_format.py +++ b/tests/test_pcm_input_format.py @@ -97,6 +97,25 @@ def test_default_batch_behavior_is_unchanged(): assert values is data +def test_scp_filelist_skips_blank_lines(tmp_path): + list_path = tmp_path / "wav.scp" + list_path.write_text("utt1 /a.wav\n\n \t \nutt2 /b.wav\n", encoding="utf-8") + keys, values = prepare_data_iterator(str(list_path)) + assert keys == ["utt1", "utt2"] + assert values == ["/a.wav", "/b.wav"] + + +def test_jsonl_filelist_skips_blank_lines(tmp_path): + list_path = tmp_path / "data.jsonl" + list_path.write_text( + '{"key": "k1", "source": "/a.wav"}\n\n \t \n{"key": "k2", "source": "/b.wav"}\n', + encoding="utf-8", + ) + keys, values = prepare_data_iterator(str(list_path)) + assert keys == ["k1", "k2"] + assert values == ["/a.wav", "/b.wav"] + + class _RecordingModel(torch.nn.Module): def __init__(self, vad=False): super().__init__()