@@ -878,17 +878,20 @@ def test_stray_im_start_in_every_message(name, path):
878878
879879@pytest .mark .parametrize ("name,path" , _available_tokenizers ())
880880def test_unknown_quirk_raises (name , path ):
881- """If segment count != timeline length AND no content contains a literal
882- <|im_start|>, the divergence is an unknown template quirk — recovery must
883- raise rather than silently misalign (it cannot apply the im_end path, which
884- needs a stray <|im_start|> in some content to be the recoverable case).
881+ """On the legacy im_start path (sep_via_eos=False), if segment count !=
882+ timeline length AND no content contains a literal <|im_start|>, the
883+ divergence is an unknown template quirk — the recovery guard must raise
884+ rather than silently misalign (the im_end path needs a stray <|im_start|>
885+ in some content to be the recoverable case).
886+
887+ The default (sep_via_eos=True) path instead runs the im_end split, which
888+ would still mismatch and raise at the final length assert; this test pins
889+ the stricter guard on the legacy path explicitly.
885890
886891 The only clean-content way the template emits an extra segment is when it
887892 auto-injects a default <|im_start|>system block for a conversation that has
888- no system message (Qwen2.5-7B does this; Qwen3 family does not). So this test
889- only runs where that precondition holds, and is skipped elsewhere — the
890- guard itself is the behavior under test, and it can only be exercised when
891- a clean-content divergence actually exists."""
893+ no system message (Qwen2.5-7B does this; Qwen3 family does not). So this
894+ test only runs where that precondition holds, and is skipped elsewhere."""
892895 tokenizer = _load_tokenizer (path )
893896 tr = _make_tracker (tokenizer )
894897 messages = [{"role" : "user" , "content" : "hello with no system message" }]
@@ -901,7 +904,7 @@ def test_unknown_quirk_raises(name, path):
901904 assert all (_IM_START not in (m .content or "" ) for m in timeline ), \
902905 f"[{ name } ] precondition: contents must be clean"
903906 with pytest .raises (AssertionError , match = "unknown template quirk" ):
904- tr .tokenize_and_slice_timeline (timeline , [])
907+ tr .tokenize_and_slice_timeline (timeline , [], sep_via_eos = False )
905908
906909
907910@pytest .mark .parametrize ("name,path" , _available_tokenizers ())
0 commit comments