From 06f877f575fc65c1a7c19d19fb742e0747b645ff Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 03:44:16 -0700 Subject: [PATCH 01/11] feat(#397): the Catalan link joins Catalan and Polish link two surnames with `i` the way Spanish links them with `y`, and `i` was not connective vocabulary at all, so the link was never joined. `parse("Josep Carod i Rovira")` gave middle 'Carod i' with family 'Rovira' -- the family one word short of what a Catalan reader wants -- and after a family comma the link was not merely unjoined but read as a GENERATION, so `Carod i Rovira, Josep` gave family 'Carod Rovira' with suffix 'i' and the surname lost the link outright. Every release from 1.4.0 read all three that way. Adding the word alone does not fix the commonest shape, and the issue said why. `i` is a bare entry in the roman-numeral suffix vocabulary, `_is_rootname` refuses a suffix piece, and the carve-out that keeps a single letter a name word in a short name counts rootnames -- so the link shrank the very total it had to clear and needed one more name word than `y` does before it would join. So a connective counts as a name word wherever that rule counts them, whatever else the vocabulary says the word is. A word can be a connective and a generation at once, and counting it as the generation lets a connective raise the bar for its own join, which would leave the shortest linked name -- the commonest one -- as the only one that failed to link. The test is inlined rather than a call: it runs once per piece of every parse, and inlining it makes every connective name three frames cheaper. That alone would take a trailing link too, so a connective that is also generational vocabulary joins only where a name word stands on each side of it. `John Quincy Smith i` keeps suffix 'i': a connective with nothing to its right is connecting nothing, and a word of that vocabulary ending a name is the generation it also spells. The rule is about the CLASS and not the letter -- measured against a caller's own `add(conjunctions={"v"})`, which behaves identically, and against `add(conjunctions={"x"})`, a roman letter outside the suffix vocabulary, which is untouched. It is narrow by construction: it cannot see a trailing `y` or `and`, which is what leaves those readings where they were. Reaching the class has one accepted cost, pinned rather than hidden: a connective a caller ALSO makes particle vocabulary was refused the same way, so under `add(particles={"y"})` the reading of `Juan Velasquez y Garcia` moves -- onto the answer the default vocabulary already gave. `Henry i of England` moves with it, its two adjacent connectives merging into a run that now counts itself; that is the shape `Duke of Edinburgh` already had, and it is recorded rather than repaired. `i` ships in the marked subset beside `e`, a bare I initial being as common as a bare E, so a name written wholly in one case reads the letter as an initial and reports the fork rather than joining in silence -- `JOSEP CAROD I ROVIRA` and `josep carod i rovira` keep today's reading and gain a `conjunction-or-initial`. A caller with Catalan or Polish data removes the entry from the subset and gets the join back. Two names keep the old reading deliberately. `Carod i` never reaches the join loop, which needs three pieces, so it stays a name plus a trailing generation; and `Josep Carod i` is refused one gate earlier, by the three-word carve-out itself. Over every corpus name under eight configurations this moves two roles and three reports, and the reference frame count is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/rules.md | 32 +++- nameparser/_pipeline/_group.py | 38 ++++- nameparser/config/conjunctions.py | 20 ++- tests/v2/cases.py | 200 +++++++++++++++++++++++-- tests/v2/pipeline/test_classify.py | 48 +++++- tests/v2/pipeline/test_group.py | 134 +++++++++++++++++ tests/v2/test_ledger_guards.py | 18 ++- tools/differential/corpus_rules.jsonl | 5 + tools/differential/corpus_shapes.jsonl | 11 ++ 9 files changed, 471 insertions(+), 35 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index b17c6769..2b5ba436 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -517,6 +517,17 @@ P3. Rationale: connective words ("y", "of the") bind name words into The joined part is ONE name word wherever another rule counts them, so a rule taking "one name word" takes the whole join and never half of it. + A connective counts as a name word wherever this rule counts them, + whatever else the vocabulary says the word is. A word can be a + connective and a generation at once — the Catalan link is also the + roman numeral one — and counting it as the generation would let a + connective raise the bar for its own join, so the shortest linked + name, which is the commonest one, would be the only one that + failed to link. + A connective that is also generational vocabulary joins only where + a name word stands on each side of it. A connective with nothing + to its right is connecting nothing, and a word of that vocabulary + ending a name is the generation it also spells. Both questions this rule asks of a name — how many words it has, and whether it is written in one case — are asked of the name's OWN words: a maiden marker taken as one, and the words it takes @@ -544,13 +555,18 @@ P3. Rationale: connective words ("y", "of the") bind name words into "Juan & Garcia" → given="Juan & Garcia" "Mr. Jack and Jill" → family="Jack and Jill" "Mr. Jack Jill" → given="Jack" + "Josep Carod i Rovira" → family="Carod i Rovira" + "Carod i Rovira, Josep" → family="Carod i Rovira" + "Josep i Rovira" → middle="i" + "John Quincy Smith i" → suffix="i" + "Carod i" → suffix="i" Both exceptions are about the written FORM, not the word: the three-word carve-out counts letters, so a symbol connective joins at any length, and it reaches every single-letter connective the - vocabulary holds — Cyrillic и/і/й and Arabic و as well as y and - e. The initial reading counts letters too, and asks one more - question of them: a letter with no case at all (و) can be written - against nothing, so it never reads as an initial. A Cyrillic + vocabulary holds — Catalan i and Cyrillic и/і/й and Arabic و as + well as y and e. The initial reading counts letters too, and asks + one more question of them: a letter with no case at all (و) can be + written against nothing, so it never reads as an initial. A Cyrillic capital has case but is not the shape an initial is written in — Cyrillic abbreviates with a dotted letter — so in a name of more than one case it joins, the reading #267 blessed, and in a name @@ -558,9 +574,11 @@ P3. Rationale: connective words ("y", "of the") bind name words into letter. Which single letters a tradition actually wants joined differs by language, and the marked set is where that answer lives: "y" is - the commonest Hispanic compound and stays out of it, "e" is a - common bare initial and is the one entry shipped. A caller with - Portuguese data removes it; a caller with Dutch data adds "y". + the commonest Hispanic compound and stays out of it, while "e" and + Catalan "i" are both common bare initials and are the two entries + shipped. A caller with Portuguese data removes "e"; a caller with + Catalan or Polish data removes "i"; a caller with Dutch data adds + the letter "y" to it. The initial reading is visible beyond the fields, on the two derived views: parse("john e smith").initials() gives "j. e. s." and .capitalized() gives "John E Smith", where the connective diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 1798a4f4..fd9ae780 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -610,6 +610,21 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], tokens: Sequence[WorkToken]) -> bool: if len(piece) == 1 and "initial" in tokens[piece[0]].tags: return False + # rules.md#P3: "A connective counts as a name word wherever this + # rule counts them, whatever else the vocabulary says the word is" + # (#397). The order against the `initial` test above decides + # nothing, and what makes that safe lives in classify rather than + # here: for a single letter it writes `initial` or `conjunction` + # and never both, so a one-case `I`/`i` arrives with no conjunction + # tag whichever test runs first. Measured -- hoisting this arm + # above the `initial` refusal moves no field, report or initial on + # any corpus name under three name orders, and no test. The + # refusal keeps its place as the older and narrower of the two. + # INLINE rather than a call to _is_conj_piece: this runs once per + # piece of every name (frame budget). + if ("conjunction" in ptags + or (len(piece) == 1 and "conjunction" in tokens[piece[0]].tags)): + return True return not (is_title_piece(piece, ptags, tokens) or _is_prefix_piece(piece, ptags, tokens) or is_suffix_piece(piece, ptags, tokens)) @@ -804,17 +819,30 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # single-letter connective in a three-word name, which stays a # name word" (v1's Google Code issue 11 carve-out, the # "john e smith" bug). The threshold reads the ROOTNAME count, - # so a conjunction that is also suffix vocabulary raises the - # bar for itself -- #397 measures that on "i". + # and since #397 a connective counts ITSELF toward that count, + # so a connective that is also suffix vocabulary no longer + # raises the bar for its own join. k = 0 while k < len(pieces): if not conj(k): k += 1 continue text = " ".join(tokens[i].text for i in pieces[k]) - if len(text) == 1 and total < 4 and text.isalpha(): - k += 1 - continue + if len(text) == 1 and text.isalpha(): + if total < 4: + k += 1 + continue + # rules.md#P3: "A connective that is also generational + # vocabulary joins only where a name word stands on + # each side of it" (#397). About the CLASS, not the + # letter, and narrow by construction: it is reached + # only for a single-letter connective piece, and it + # cannot see a trailing `y` or `and`, which is what + # leaves those readings alone. + if (not (0 < k < len(pieces) - 1) + and is_suffix_piece(pieces[k], ptags[k], tokens)): + k += 1 + continue start = max(0, k - 1) end = min(len(pieces), k + 2) neighbor = start if start < k else end - 1 diff --git a/nameparser/config/conjunctions.py b/nameparser/config/conjunctions.py index 2ed2448c..e69d3c28 100644 --- a/nameparser/config/conjunctions.py +++ b/nameparser/config/conjunctions.py @@ -9,6 +9,13 @@ 'the', 'und', 'y', + # #397: the Catalan/Polish surname link ("Carod i Rovira", + # "Kowalski i Nowak"). Single-letter like 'y'/'и'/'e', and the one + # entry that is ALSO generational vocabulary -- 'i' is the roman + # numeral I, a bare entry of SUFFIX_WORDS -- so the carve-out + # counts it as a name word for its own sake and it joins only + # with a name word on each side (rules.md#P3). + 'i', # #269: Cyrillic (ru/uk/bg) "and": и, і, та. Ukrainian writes і and # й for the same conjunction, alternating on the surrounding # vowel/consonant for euphony ("Олесь і Олена", "Марія й Петро"), @@ -46,14 +53,17 @@ # ('Velasquez y Garcia'). Cyrillic и/і/й follow y, not e: #267 # blessed their joining and nothing here narrows it. # - # 'i' (Catalan) is NOT here because it is not conjunction vocabulary - # at all yet; it ships in this subset if #397 adds it, a bare I - # initial being as common as a bare E. + # 'i' (Catalan/Polish) ships here beside 'e' for the same reason + # (#397): a bare I initial is as common as a bare E, so a name + # written wholly in one case reads the letter as an initial and + # reports the fork rather than joining in silence. # # A caller edits the vocabulary rather than a switch: remove 'e' to - # restore joining for Portuguese data, add 'y' for a Dutch-style - # "every single letter is an initial". + # restore joining for Portuguese data, remove 'i' for Catalan or + # Polish data, add 'y' for a Dutch-style "every single letter is an + # initial". 'e', + 'i', }) """ Single-letter entries of :data:`CONJUNCTIONS` that read as an initial, diff --git a/tests/v2/cases.py b/tests/v2/cases.py index fa422ebc..80ab902f 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -2466,24 +2466,198 @@ def _check_cjk_shape_purity(self) -> None: "an accented capital is admitted -- the docstring's " "'Jean ÉCOLE' example, given the third word it needs " "to have words to spare"), - Case("catalan_i_is_not_connective_vocabulary_upper", + Case("catalan_i_in_a_one_case_name_reads_as_an_initial_upper", "JOSEP CAROD I ROVIRA", {"given": "JOSEP", "middle": "CAROD I", "family": "ROVIRA"}, - notes="pinned at TODAY's reading so #397 shows its move: 'i' " - "is not in CONJUNCTIONS at all, so the bare capital is " - "an initial by shape and this row never reaches the " - "fork. If #397 adds 'i' it ships in " - "conjunctions_ambiguous too, and this row changes", + classification="fix(#397)", + ambiguities=("conjunction-or-initial",), + notes="the row that predicted its own move, and the move is " + "the REPORT and not the reading: 'i' is connective " + "vocabulary now and ships in conjunctions_ambiguous " + "beside 'e', so a name written wholly in one case reads " + "the bare capital as an initial -- which is what it " + "already did by shape -- and now says so. Roles " + "unchanged from 1.4.0 through 2.3.0; only the " + "ambiguity is new. The id moved with the answer: it " + "read 'is_not_connective_vocabulary', which this " + "commit makes false, and nothing outside this table " + "named it", shape=1), - Case("catalan_i_is_not_connective_vocabulary_lower", + Case("catalan_i_in_a_one_case_name_reads_as_an_initial_lower", "josep carod i rovira", {"given": "josep", "middle": "carod i", "family": "rovira"}, - notes="the lowercase twin: 'i' is an ordinary name word, not " - "vocabulary, so nothing joins and nothing reports. This " - "row pins NOTHING today -- both readings are what every " - "release including 1.4.0 already gives -- and is kept " - "anyway as the other half of #397's before-picture, " - "beside its upper twin above", + classification="fix(#397)", + ambiguities=("conjunction-or-initial",), + notes="the lowercase twin, and the one that shows the fork is " + "about CASE EVIDENCE and not about capitals: a name " + "written wholly in lower case has nothing marking the " + "letter either way, so the marked set decides and the " + "letter reads as an initial. Roles unchanged at every " + "release; the report is new. Compare 'Josep Carod i " + "Rovira' below, which is written in more than one case " + "and joins", + shape=1), + # ---- #397: the Catalan/Polish link ------------------------------ + # The rule: a connective counts as a name word wherever P3's + # carve-out counts them, whatever else the vocabulary says the word + # is -- and a connective that is ALSO generational vocabulary joins + # only where a name word stands on each side of it. 'i' is the one + # shipped entry in both classes: the Catalan link and the roman + # numeral I. + Case("the_catalan_link_joins_two_surnames", + "Josep Carod i Rovira", + {"given": "Josep", "family": "Carod i Rovira"}, + classification="fix(#397)", + notes="the statement of the rule and the commonest shape. " + "Four words, so the carve-out's count has to reach four " + "-- and it does only because the link counts ITSELF, " + "'i' being suffix vocabulary that _is_rootname used to " + "refuse. 1.4.0 through 2.3.0 read middle 'Carod i', " + "family 'Rovira', which is the link left unjoined one " + "word short of what a Catalan reader wants", + shape=1), + Case("the_catalan_link_joins_with_a_middle_name_in_front", + "Josep Lluis Carod i Rovira", + {"given": "Josep", "middle": "Lluis", "family": "Carod i Rovira"}, + classification="fix(#397)", + notes="the same join with a real middle name beside it: the " + "join takes 'Carod i Rovira' and leaves 'Lluis' where " + "it was, rather than swallowing the whole tail. 1.4.0 " + "read middle 'Lluis Carod i', family 'Rovira'", + shape=1), + Case("the_catalan_link_survives_a_family_comma", + "Carod i Rovira, Josep", + {"given": "Josep", "family": "Carod i Rovira"}, + classification="fix(#397)", + notes="the sharpest of the before-pictures, and the half the " + "vocabulary alone does not fix: after a family comma " + "the link was not merely unjoined, it was read as a " + "GENERATION and the family lost it -- 1.4.0 through " + "2.3.0 gave family 'Carod Rovira' with suffix 'i'. It " + "is also the one new name here that lands in a comma " + "rule's corpus claim", + shape=2), + Case("the_link_is_still_a_name_word_in_a_three_word_name", + "Josep i Rovira", + {"given": "Josep", "middle": "i", "family": "Rovira"}, + ambiguities=(), + notes="P3's own carve-out, and the BOUNDARY: three words, so " + "the single-letter connective stays a name word and " + "nothing joins. Roles identical at 1.4.0 and at every " + "release since, which is what keeps this row parity -- " + "and the two halves of this bundle cancel on it, since " + "the readmission (#461) hands the letter back an " + "initial that the vocabulary addition would otherwise " + "have taken, so 'J. i. R.' holds across the bundle. " + "Written in more than one case, so no fork is consulted " + "and nothing reports", + shape=1), + Case("a_link_with_nothing_to_its_right_is_the_generation", + "John Quincy Smith i", + {"given": "John", "middle": "Quincy", "family": "Smith", + "suffix": "i"}, + notes="the BOUNDARY the both-sides condition exists for, and " + "the one Catalan row that reaches its is_suffix_piece " + "call at all -- every other Catalan row short-circuits " + "on the position test (measured). Four name words, so " + "the count no longer declines; drop the condition and " + "this reads family 'Smith i', losing the generation " + "(measured). A connective with nothing to its right is " + "connecting nothing. Roles unchanged at every release, " + "1.4.0 included", + shape=1), + Case("a_link_ending_a_shorter_name_is_the_generation_too", + "Josep Carod i", + {"given": "Josep", "family": "Carod", "suffix": "i"}, + notes="the same boundary one word shorter, and it is refused " + "one gate EARLIER: three name words, so the count " + "declines before the both-sides test is reached. The " + "pair with the row above is what separates the two " + "gates -- measured, delete the both-sides condition and " + "this row still passes while its four-word twin does " + "not. Roles unchanged at every release", + shape=1), + Case("the_two_word_link_keeps_the_generation_reading", + "Carod i", + {"given": "Carod", "suffix": "i"}, + classification="fix(suffix-routing)", + ambiguities=("given-or-family",), + notes="an ACCEPTED COST, pinned so the ledger rule that " + "records it has a witness in this table. Two pieces " + "never reach the join loop at all, which needs three, " + "so the name reads as a name plus a trailing " + "generation and the family is empty. 1.4.0 read last " + "'i' -- the same two-token divergence the 1.4.0 " + "ledger's fix(suffix-routing) rule describes for a " + "trailing roman numeral, and the same one 'Donald Mc' " + "pins above for the acronym half. That reading also " + "has a NOT-WANTED rule at 1.4.0 whose standing this " + "bundle converts from 'unresolved' to 'decided': the " + "cause is settled and the reading is kept", + shape=1), + Case("the_link_joins_under_a_trailing_generation_marker", + "Josep Carod i Rovira III", + {"given": "Josep", "family": "Carod i Rovira", "suffix": "III"}, + classification="fix(#397)", + notes="the suffix run is peeled before grouping, so the join " + "sees the same four pieces it sees without it and the " + "generation keeps its own slot. The contrast with " + "'John Quincy Smith i' is the whole design: a " + "generation word BEHIND the link is a suffix, a " + "generation word that IS the link with nothing behind " + "it stays a suffix too. 1.4.0 read middle 'Carod i'", + shape=1), + Case("a_bare_capital_link_in_a_mixed_case_name_is_an_initial", + "Josep Carod I Rovira", + {"given": "Josep", "middle": "Carod I", "family": "Rovira"}, + ambiguities=(), + notes="the recorded negative control for the whole " + "vocabulary addition, and the reason the `initial` test " + "stays FIRST in the rootname count: a bare Latin " + "capital in a name of more than one case is how an " + "initial is written, so classify tags it `initial` and " + "never `conjunction`, the count is unmoved and nothing " + "joins. Identical at 1.4.0 and at every release since", + shape=1), + Case("a_connective_run_absorbs_a_name_with_no_other_name_word", + "Henry i of England", + {"given": "Henry i of England"}, + classification="fix(#397)", + ambiguities=("given-or-family",), + notes="the ACCEPTED CONSEQUENCE of reaching the class, " + "recorded rather than repaired: 'i' and 'of' are " + "adjacent connectives, so they merge into one " + "connective RUN and the run -- now counting itself -- " + "joins the whole name into a single piece. Before this " + "commit the same name read given 'Henry', family 'i of " + "England'; 1.4.0 read the same. It is the shape 'Duke " + "of Edinburgh' already has, which is why the report is " + "the lone-name-word one. UNTAGGED on purpose: a " + "second-order consequence is not a shape this table " + "admits to the contract corpus", + ), + Case("a_trailing_y_is_not_in_the_both_sides_class", + "Juan Garcia Lopez y", + {"given": "Juan", "middle": "Garcia", "family": "Lopez y"}, + notes="the recorded negative control for the CLASS half of " + "the both-sides test, and it must NOT move: 'y' is " + "connective vocabulary but not generational " + "vocabulary, so the condition never asks about it and " + "today's reading stands. Drop the is_suffix_piece " + "conjunct and this row reads family 'y' with 'Lopez' " + "pushed into the middle -- measured, and no property " + "invariant in this repo catches it. Identical at 1.4.0", + shape=1), + Case("a_trailing_y_behind_a_join_is_not_in_the_class_either", + "Juan Garcia y Lopez y", + {"given": "Juan", "family": "Garcia y Lopez y"}, + notes="the same control with the letter appearing twice, one " + "joining and one trailing: the interior 'y' joins on " + "the position test and the trailing one is never asked " + "about, so the whole run stays the family. Drop the " + "class conjunct and the family collapses to 'y' with " + "middle 'Garcia y Lopez' -- measured. Identical at " + "1.4.0", shape=1), # ---- #531: the given part's trailing slot ---------------------- # The slot: after a family comma the comma has already named the diff --git a/tests/v2/pipeline/test_classify.py b/tests/v2/pipeline/test_classify.py index 7ed579aa..ed2666d8 100644 --- a/tests/v2/pipeline/test_classify.py +++ b/tests/v2/pipeline/test_classify.py @@ -34,8 +34,11 @@ # test_cyrillic_initial_outranks_the_conjunction pins is one that # really ships and the reader can check against the defaults. The # copies are local: this file never reads the shipped sets. - conjunctions=frozenset({"and", "e", "y", "й"}), - conjunctions_ambiguous=frozenset({"e"}), + # 'i' (#397) is the second marked entry and the only one that is + # ALSO in suffix_words here, which is what makes the collision cell + # below reachable. + conjunctions=frozenset({"and", "e", "i", "y", "й"}), + conjunctions_ambiguous=frozenset({"e", "i"}), bound_given_names=frozenset({"abdul"}), # "née" and the unaccented "nee" are both shipped (English writes # the French marker either way, AGENTS.md); a clause-truncation @@ -204,6 +207,47 @@ def test_emptying_the_subset_restores_joining_for_e() -> None: assert out.ambiguities == () +def test_the_catalan_link_reads_as_an_initial_in_a_one_case_name( +) -> None: + # #397's letter through #527's fork, and the reason it ships + # marked: a bare I initial is as common as a bare E, so a name + # with no case evidence reads the letter as an initial and says + # so. rules.md#P3 says the marked set is where that answer lives. + for text in ("josep carod i rovira", "JOSEP CAROD I ROVIRA"): + out = _classified(text) + letter = "i" if text.islower() else "I" + assert "initial" in _tags(out, letter), text + assert "conjunction" not in _tags(out, letter), text + kinds = [a.kind for a in out.ambiguities] + assert kinds == [AmbiguityKind.CONJUNCTION_OR_INITIAL], text + assert repr(letter) in out.ambiguities[0].detail, text + + +def test_the_catalan_link_joins_where_the_writing_speaks() -> None: + # the mixed-case half, unchanged and unreported: a lowercase + # letter among capitals is the connective and a bare capital is + # an initial, exactly as for 'e'. + lower = _classified("Josep Carod i Rovira") + assert "conjunction" in _tags(lower, "i") + assert "initial" not in _tags(lower, "i") + upper = _classified("Josep Carod I Rovira") + assert "initial" in _tags(upper, "I") + assert "conjunction" not in _tags(upper, "I") + assert lower.ambiguities == () and upper.ambiguities == () + + +def test_removing_the_link_from_the_subset_restores_joining() -> None: + # the Catalan/Polish removal recipe docs/customize.rst teaches, + # exercised: drop 'i' from the marked subset and a one-case name + # joins it again, silently, which is what a ca/pl corpus wants. + lex = dataclasses.replace( + _LEX, conjunctions_ambiguous=_LEX.conjunctions_ambiguous - {"i"}) + out = _classified_with("josep carod i rovira", lex) + assert "conjunction" in _tags(out, "i") + assert "initial" not in _tags(out, "i") + assert out.ambiguities == () + + def test_an_orphan_marker_is_inert_even_in_the_emitter() -> None: # 'e' left in conjunctions_ambiguous but removed from conjunctions # -- legal since the pair is not in _SUBSET_FIELDS (decisions.md#P3) diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index d484d631..33394066 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -5,6 +5,7 @@ import pytest +from nameparser import Parser from nameparser._lexicon import Lexicon from nameparser._pipeline import _group as _group_module from nameparser._pipeline._classify import classify @@ -1396,3 +1397,136 @@ def test_the_marker_placements_reach_both_answers() -> None: tagged[text] = bool(runs) assert sum(tagged.values()) >= 4 assert sum(not v for v in tagged.values()) >= 3 + + +# --- #397: the carve-out's count, and the both-sides condition ------ +# _LEX ships no single letter that is BOTH connective and generational +# vocabulary, so each test below builds the overlap it is about. That +# is the point of the rule -- it is keyed on the CLASS, never on the +# letter -- and a test using the shipped sets would walk the right +# branch while proving nothing about it (AGENTS.md: "Pin the decision, +# not the vocabulary"). +_LINK_LEX = _LEX.add(conjunctions={"i"}, suffix_words={"i"}) +#: the same letter as a connective that is NOT generational vocabulary +_PLAIN_LEX = _LEX.add(conjunctions={"i"}) + + +def test_a_connective_piece_counts_toward_the_carve_outs_total() -> None: + # rules.md#P3's count (#397). Four words, one of them the link, + # and the link is suffix vocabulary -- so the total reaches four + # only because a connective counts ITSELF. Without the count arm + # the total is three, the carve-out declines, and the link stays + # a name word in the middle. + out = _grouped("Josep Carod i Rovira", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Josep", "Carod i Rovira"]] + + +def test_the_count_arm_is_what_moves_it_not_the_vocabulary() -> None: + # the control that separates the two halves of commit 1: with the + # letter a connective but NOT generational vocabulary, nothing + # refused it before and the join already fired. Same output, and + # the pair is what says the count arm is about the overlap. + out = _grouped("Josep Carod i Rovira", lexicon=_PLAIN_LEX) + assert _piece_texts(out) == [["Josep", "Carod i Rovira"]] + + +def test_a_connective_with_nothing_to_its_right_does_not_join() -> None: + # the both-sides condition (#397). Four name words, so the count + # no longer declines -- what keeps the generation here is the + # position test, and dropping it reads 'Smith i' as one piece. + out = _grouped("John Quincy Smith i", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["John", "Quincy", "Smith", "i"]] + + +def test_a_connective_with_nothing_to_its_left_does_not_join() -> None: + # the other side of the same condition, and it needs four pieces + # to get past the count: a link OPENING the name has no name word + # behind it either. + out = _grouped("i Carod Rovira Puig", lexicon=_LINK_LEX) + assert _piece_texts(out)[0][0] == "i" + + +def test_the_both_sides_condition_reads_the_class_not_the_letter( +) -> None: + # the recorded negative control for the is_suffix_piece conjunct, + # and the one the property invariants CANNOT give: with the same + # letter outside the generational vocabulary the condition + # declines to ask and the trailing connective joins, exactly as a + # trailing 'y' does today. Measured -- remove that conjunct and + # this test is the one that dies. + out = _grouped("John Quincy Smith i", lexicon=_PLAIN_LEX) + assert _piece_texts(out) == [["John", "Quincy", "Smith i"]] + + +def test_a_trailing_shipped_connective_is_untouched_by_the_condition( +) -> None: + # the shipped-vocabulary half of the same control: 'y' is a + # connective and not a suffix word, so the condition never reaches + # it and 'Lopez y' stays one piece. + out = _grouped("Juan Garcia Lopez y") + assert _piece_texts(out) == [["Juan", "Garcia", "Lopez y"]] + + +def test_the_three_word_carve_out_still_declines_before_both_sides( +) -> None: + # the two gates are separate and this is what separates them: + # three name words, so the count refuses and the both-sides test + # is never reached. Delete the both-sides condition and this test + # still passes while its four-word sibling does not. + out = _grouped("Josep Carod i", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Josep", "Carod", "i"]] + + +def test_a_one_case_letter_is_an_initial_and_never_counts() -> None: + # what keeps the one-case fork's count where it was: classify + # writes `initial` or `conjunction` on a single letter and never + # both, so a letter the fork read as an initial reaches + # _is_rootname with no conjunction tag at all, the count is + # unmoved, and the name reads as it always did. The EXCLUSIVITY is + # the load-bearing part, not the order of the two tests -- measured + # (swapping them moves nothing). + lex = _LINK_LEX.add(conjunctions_ambiguous={"i"}) + out = _grouped("josep carod i rovira", lexicon=lex) + assert _piece_texts(out) == [["josep", "carod", "i", "rovira"]] + + +def test_the_class_reaches_a_callers_own_connective() -> None: + # rules.md#P3 is keyed on the class throughout: a caller who adds + # 'v' to their connectives gets the Catalan link's behavior for + # it, because 'v' is generational vocabulary the way 'i' is. + p = Parser(lexicon=Lexicon.default().add(conjunctions={"v"})) + joined = p.parse("Josep Carod v Rovira") + assert (joined.given, joined.family) == ("Josep", "Carod v Rovira") + trailing = p.parse("John Quincy Smith v") + assert (trailing.family, trailing.suffix) == ("Smith", "v") + + +def test_a_callers_non_generational_letter_is_outside_the_condition( +) -> None: + # the recorded negative control, at the reading level: 'x' is a + # roman numeral letter that is NOT suffix vocabulary, so the + # both-sides condition declines to ask and BOTH positions join, + # which is what they did before this change too. + p = Parser(lexicon=Lexicon.default().add(conjunctions={"x"})) + assert p.parse("John Quincy Smith x").family == "Smith x" + assert p.parse("Josep Carod x Rovira").family == "Carod x Rovira" + + +def test_the_count_reaches_a_connective_that_is_particle_vocabulary( +) -> None: + # the ACCEPTED CONSEQUENCE of counting a connective whatever else + # it is: _is_rootname refuses a PARTICLE piece the same way it + # refuses a generational one, so a caller who makes 'y' a particle + # too used to get a different reading from the default lexicon. + # Now it agrees with it -- the direction the rule wants. Measured + # before this change: given 'Juan', middle 'Velasquez', family + # 'y Garcia', family_base 'Garcia'. + p = Parser(lexicon=Lexicon.default().add(particles={"y"})) + out = p.parse("Juan Velasquez y Garcia") + assert (out.given, out.middle, out.family) == ( + "Juan", "", "Velasquez y Garcia") + # the cost that comes with it, pinned rather than hidden: + # family_base drops a particle wherever it stands and not only + # leading, so the joined run loses the letter here. A standing + # rules.md#R2 limit this row surfaces, not one it creates. + assert out.family_base == "Velasquez Garcia" diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 06920793..36b67f50 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -3130,8 +3130,12 @@ def _claim(rule: dict) -> _Claim: # carrying a comma. Reach, not explanation: the regex is a # comma and every family-comma row this change added matches # it. + # 2026-09-20, #397: 358 -> 359, the one new corpus name + # carrying a comma, 'Carod i Rovira, Josep'. Reach, not + # explanation, and verified name by name: the claimed set + # gained that name and lost none. "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(358, ('given', 'suffix', 'title'), '0485bb024e5a', None), + _Claim(359, ('given', 'suffix', 'title'), '37cb26979687', None), "fix(comma-family) a comma followed only by titles keeps the given/family split": _Claim(2, ('family', 'given'), "5bd9c6d96c38", None), "fix(comma-family) a comma followed only by titles keeps the given/family split, the C1 example": @@ -3181,8 +3185,10 @@ def _claim(rule: dict) -> _Claim: # comma names as the rule above and for the same reason. # 2026-09-19, #533: 343 -> 356, the same thirteen new comma # names as the rule above and for the same reason. + # 2026-09-20, #397: 358 -> 359, the same one new comma name as + # the rule above and for the same reason. "fix(comma-precomma-family) pre-comma run reads as family, not given": - _Claim(358, ('family', 'given'), '0485bb024e5a', None), + _Claim(359, ('family', 'given'), '37cb26979687', None), "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": _Claim(1, ('family', 'suffix'), "498602f3cfd0", None), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -3404,8 +3410,14 @@ def _claim(rule: dict) -> _Claim: # share a field and two corpus names -- and this rule's REACH # is unmoved by it: the regex did not change, and reach is # what this number counts. + # 2026-09-20, #397: 101 -> 102. One new corpus name holding a + # connective run, 'Juan Garcia y Lopez y' -- the negative + # control for the both-sides condition's class half, which is + # in the corpus because its row carries a shape tag. Reach + # again: the regex is unchanged and the claimed set gained + # that name alone. "fix(initials-per-word) a connective run initials each word (facade, since 2.0.0)": - _Claim(101, ('_initials',), "e91031622dca", ('DEFAULT',)), + _Claim(102, ('_initials',), "2c63e2c40d47", ('DEFAULT',)), # 2026-09-19, #533: 41 -> 43. Two new corpus names opening # with a bound-given word, 'Berg, abdul MA' and 'Berg, abdul # nee Jones MA' -- the P5 pair this change added to record diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 35cb26e8..91af5059 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -32,6 +32,8 @@ "Berg, abdul nee Jones MA" "Berg, abdul van" "Berg, abdul vd" +"Carod i" +"Carod i Rovira, Josep" "Davis Royce, Ed" "Del Toro" "Doe, Dr. nee Smith MA" @@ -124,6 +126,7 @@ "John Doctor Smith" "John Ma" "John Prof. MA" +"John Quincy Smith i" "John Smith" "John Smith 1.4" "John Smith Esq." @@ -178,6 +181,8 @@ "Jong, Piet de" "Jose E Maria Santos" "Jose e Maria Santos" +"Josep Carod i Rovira" +"Josep i Rovira" "Jr." "Juan & Garcia" "Juan McDonald" diff --git a/tools/differential/corpus_shapes.jsonl b/tools/differential/corpus_shapes.jsonl index 0f8bd789..50d325fb 100644 --- a/tools/differential/corpus_shapes.jsonl +++ b/tools/differential/corpus_shapes.jsonl @@ -1,6 +1,7 @@ {"name": "ANH DO", "shape": 1} {"name": "Bridge (1.4)", "shape": 1} {"name": "Bridge (A.B)", "shape": 1} +{"name": "Carod i", "shape": 1} {"name": "Dr. Juan de la Vega III", "shape": 1} {"name": "J.A. K.D.", "shape": 1} {"name": "J.R.R. Tolkien", "shape": 1} @@ -44,6 +45,7 @@ {"name": "John Doe JD.CPA", "shape": 1} {"name": "John Doe do", "shape": 1} {"name": "John Jack Andrew Kennedy", "shape": 1} +{"name": "John Quincy Smith i", "shape": 1} {"name": "John Smith", "shape": 1} {"name": "John Smith 1.4", "shape": 1} {"name": "John Smith B.Tech.", "shape": 1} @@ -64,7 +66,15 @@ {"name": "John van der Berg Ma", "shape": 1} {"name": "Jose E Maria Santos", "shape": 1} {"name": "Jose e Maria Santos", "shape": 1} +{"name": "Josep Carod I Rovira", "shape": 1} +{"name": "Josep Carod i", "shape": 1} +{"name": "Josep Carod i Rovira", "shape": 1} +{"name": "Josep Carod i Rovira III", "shape": 1} +{"name": "Josep Lluis Carod i Rovira", "shape": 1} +{"name": "Josep i Rovira", "shape": 1} +{"name": "Juan Garcia Lopez y", "shape": 1} {"name": "Juan Garcia Y Lopez", "shape": 1} +{"name": "Juan Garcia y Lopez y", "shape": 1} {"name": "Juan de la Vega", "shape": 1} {"name": "Lt.Gov. John Doe", "shape": 1} {"name": "Maria Kowalska z domu Nowak MA", "shape": 1} @@ -93,6 +103,7 @@ {"name": "Berg, Jane van der nee Smith DO", "shape": 2} {"name": "Berg, abdul MA", "shape": 2} {"name": "Berg, abdul nee Jones MA", "shape": 2} +{"name": "Carod i Rovira, Josep", "shape": 2} {"name": "DOE, JOHN MA", "shape": 2} {"name": "DOE, MARY JO MA", "shape": 2} {"name": "Doe, Dr. John MA", "shape": 2} From c8550b6451d07e563818bf6ddb6d3e2de2c5bf63 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 04:29:12 -0700 Subject: [PATCH 02/11] fix(#461): a connective initials where it joins nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R3 said a CONJUNCTION never initials, so a base that is one contributes nothing even then -- and the parser did not do that: `parse("Juan de y").family_base` was 'y' while `.initials()` was 'J.', so the two views of one parse disagreed about one token. The rule's scope was never settled for the given group at all: R3 said so in prose and listed four candidate answers for `John and Jane Smith`, which had initialed its connective since 1.4.0 with nothing but the differential corpora holding it. The clause is now conditional, and it is one sentence for all three groups. A connective contributes nothing where it is JOINING: a part holding another name word for it to join -- the part's working particles set aside -- is a part where the connective is doing a connective's work and is no name word of its own. A part holding nothing else is a part where it is joining nothing, and there it initials like any other name word, agreeing with the base. The question is asked of the WHOLE PART and never of a word count. `Jon Dough and` has base 'Dough and' and keeps initials 'J. D.', because the other word is a name word; a part of three is no different. That is what the given group's old blanket exemption was standing in for, and dropping the exemption is the whole of "one rule for every group". The criterion is a fact about a PART, so it is decided ONCE where the parts are settled and recorded on the token, beside the mark R2 already writes there -- the direct answer to what the first attempt at this found, since a view that re-derives a part-level fact is how these two came apart in the first place. Both views read the mark: the core's filter, and the facade's conjunction predicate, which needs no signature change to see it. The two marks now come off ONE walk instead of two generator expressions, which makes every parse six frames cheaper: 412 -> 406 for `parse` and 449 -> 443 for `HumanName` on py3.11, inside the recorded band, so no baseline moves. Over every corpus name the change moves 46 initials on both surfaces, the same 46 names with the same values -- 28 losing a letter where the connective is joining, 18 gaining one where it is not. Two of the gains are 1.4.0 parity RESTORED: `محمد و علي` and `JUAN Y GARCIA` now give the answer 1.4.0 gave. Five invariants over 170,100 generated parses hold at 0 failures, and each was mutation-checked against the branch it is stated about. Five no-parse paths keep the pre-#461 reading -- a spliced field, a v1 pickle or copy, the keyword constructor, an overridden public list property, and a widen-only override -- through the same vocabulary fallback and for the same reason #528's own accepted clause records. `capitalized()` does not follow, and that is R4's own rule rather than an oversight: a connective that initials because it joins nothing is still not written the way a name is written. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/rules.md | 70 ++++---- nameparser/_facade.py | 14 +- nameparser/_pipeline/_post_rules.py | 41 ++++- nameparser/_render.py | 57 ++++--- nameparser/_types.py | 52 ++++-- tests/test_initials.py | 9 +- tests/v2/cases.py | 102 ++++++++++- tests/v2/pipeline/test_post_rules.py | 66 ++++++- tests/v2/test_facade.py | 58 ++++++- tests/v2/test_parser.py | 66 ++++++- tests/v2/test_properties.py | 227 ++++++++++++++++++++++++- tests/v2/test_render.py | 112 ++++++++---- tools/differential/corpus_rules.jsonl | 4 + tools/differential/corpus_shapes.jsonl | 7 + 14 files changed, 774 insertions(+), 111 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index 2b5ba436..ebcc3579 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2027,18 +2027,21 @@ R3. Rationale: initials abbreviate the person's name words; titles, family word; titles, suffixes, particles and nicknames contribute nothing — except the particles of a part whose every word is one, which are not acting as particles there (R2) and - initial like any other name word. A CONJUNCTION never initials, - so a base that is one contributes nothing even then. That - carve-out is stated for the middle and base family words; the - GIVEN group is not settled here. A conjunction written among - given names does initial today, and this document does not yet - say whether it should — because two of its own rules answer - differently and neither answer has been taken: this rule counts - name words, while P3 makes a connective and its neighbours ONE - name word, so a joined given group owes one initial under P3 and - one per joined name word under the carve-out. Until that is - decided the given group's answer is pinned-but-undocumented - rather than specified, and no line below asserts it. + initial like any other name word. A connective contributes + nothing where it is joining: a part holding another name word for + it to join — the part's working particles set aside — is a part + where the connective is doing a connective's work and is no name + word of its own. A part holding nothing else is a part where it + is joining nothing, and there it initials like any other name + word, agreeing with the base. + The question is asked of the WHOLE PART, never of a word count. A + part of two words where one is the connective is still a part + where it joins — the base of "Jon Dough and" is "Dough and" and + its initials are "J. D." — and a part of three is no different. + One rule for every group. A connective written among given names + is a connective there too and contributes nothing where it joins, + so the group that was pinned-but-undocumented is now specified by + the same sentence as the other two. Which words a group contributes is one question; the ORDER they contribute in is a second, and its answer is the field's. Each group initials in the order its field reads — written order, @@ -2058,7 +2061,11 @@ R3. Rationale: initials abbreviate the person's name words; titles, A family that is ALL particles therefore contributes its words rather than nothing: they are the base (R2), so they initial. "Juan van der" → initials="J. v. d." - "Juan de y" → initials="J." + "Juan de y" → initials="J. y." + "John and Jane Smith" → initials="J. J. S." + "Duke of Edinburgh" → initials="D. E." + "Juan Velasquez y Garcia" → initials="J. V. G." + "Jon Dough and" → initials="J. D." Accepted: this rule reads a part the parser read. A field set as raw text after the parse carries no reading, and this view is handed no vocabulary to supply one — it takes a format spec and @@ -2090,23 +2097,17 @@ R3. Rationale: initials abbreviate the person's name words; titles, shapes need a field edited after the parse, or a rendering the other view does not have. decisions.md#R3 carries what all of it costs and where it is pinned. - Accepted: the unsettled given-group answer above is neither rare - nor hypothetical — 26 of the corpus names carry a conjunction - among the given names (measured 2026-09-13; recompute by parsing - the deduped corpus*.jsonl glob and keeping every name with a - GIVEN-role token tagged "conjunction"), every one of them - reachable from the default vocabulary, and it has initialed since - 1.4.0. It carries no marked deviation, for the reason that - mechanism exists: a - marker states the INTENDED value, and one name, "John and Jane - Smith", has four candidates. Today gives "J. a. J. S."; the - carve-out read as written gives "J. J. S."; P3's one-name-word - join gives just "J. S."; and 1.4.0 gave "J a J. S.". Marking it - would put an invented value in a normative document and hold - the parser to it. #461 asks the neighbouring question about the - all-particle base and does not own this one; decisions.md#R2 - carries the population and the measurements. - history: decisions.md#R3 · interacts: O3, P3, P6, R1, R2, R4 · implemented: nameparser/_render.py, nameparser/_facade.py + Accepted: the given group's answer is a 1.4.0 parity break, and + a second one in the other direction sits beside it. A connective + among given names contributed an initial from 1.4.0 until this + rule reached it, so "John and Jane Smith" gives "J. J. S." where + every release through 2.3 gave "J. a. J. S." and 1.4.0 gave the + run-together "J a J. S."; and a connective holding a part alone + now contributes one, so "محمد و علي" gives "م. و. ع." on both + views, which is 1.4.0's own answer restored. The rule is one + sentence for all three groups, and taking it is what made the + four candidate answers this group once had into one. + history: decisions.md#R3 · interacts: O3, P3, P6, R1, R2, R4 · implemented: nameparser/_render.py, nameparser/_facade.py, nameparser/_pipeline/_post_rules.py R4. Rationale: case repair is a display concern, applied only on request and never destructively. @@ -2116,9 +2117,12 @@ R4. Rationale: case repair is a display concern, applied only on convention (McDonald), not only ordinary word-by-word casing, and a part whose every word is particle vocabulary is repaired as ordinary name words, since none of them is doing a particle's - work there (R2). A CONJUNCTION keeps its lowercase even inside - such a part, being no name word in any part — the carve-out R3 - states for initials. A name already written the way repair would + work there (R2). A CONJUNCTION keeps its lowercase wherever it + stands, including inside a part whose other words the unjoined + mark has turned into ordinary name words. That is this rule's own + reading and not a borrowing from R3: a connective that initials + because it joins nothing is still not written the way a name is + written. A name already written the way repair would write it comes back unchanged, measured by repair's own conventions rather than by the bearer's. A spelling written in a single case is repaired even where its bearer meant it, because diff --git a/nameparser/_facade.py b/nameparser/_facade.py index c0642463..973a69cd 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -32,7 +32,8 @@ from nameparser._config_shim import CONSTANTS, Constants, _cached_parser from nameparser._lexicon import _normalize from nameparser._parser import Parser -from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, ParsedName, +from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, + UNJOINED_CONJUNCTION_TAG, ParsedName, Role, Token) _V2_FIELD = {"first": "given", "last": "family"} # v1 name -> v2 name @@ -521,7 +522,12 @@ def _token_is_conjunction(self, tok: Token) -> bool: self._resolve() if UNCLASSIFIED_TAG in tok.tags: return _render._reads_as_conjunction(tok.text, self._lexicon) - return "conjunction" in tok.tags + # #461: a connective with nothing in its part to join is not + # acting as one, and the parse decided that and marked the + # token -- the same mark the core's initials() reads, so the + # two views cannot disagree about it. + return ("conjunction" in tok.tags + and UNJOINED_CONJUNCTION_TAG not in tok.tags) def _split_last(self) -> tuple[list[str], list[str]]: # rules.md#R2: "a name part whose every word is particle @@ -633,7 +639,9 @@ def _process_initial(self, name_part: str, for tok in tokens) initials = [] for word, conjunction in zip(words, conjunctions, strict=True): - if not (self._is_particle(word) or conjunction) or firstname: + # #461: the conjunction filter reaches EVERY group; only + # the particle filter is exempted for the given group. + if not conjunction and (firstname or not self._is_particle(word)): initials.append(word[0]) if len(initials) > 0: return self.initials_separator.join(initials) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 326681e3..53f9ee3c 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -32,7 +32,7 @@ from nameparser._pipeline._vocab import delimiter_cores from nameparser._policy import PatronymicRule from nameparser._types import ( - FOLDED_TAG, UNJOINED_TAG, AmbiguityKind, Role, + FOLDED_TAG, UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, AmbiguityKind, Role, ) # Ported verbatim from v1 (nameparser/config/regexes.py) -- layering @@ -843,12 +843,49 @@ def post_rules(state: ParseState) -> ParseState: # exempt that role outright), so restricting this loop to MIDDLE # and FAMILY moves 0 of 4,506 parses. The GIVEN arm is marked so a # future view reading the mark gets a consistent answer. + # + # rules.md#R3: "A part holding nothing else is a part where it is + # joining nothing, and there it initials like any other name word" + # -- #461's mark, decided HERE for the same reason R2's is: the + # question is about the PART, and a view that re-derives a + # part-level fact is how initials() and family_base came apart. + # + # ONE walk decides both marks -- a plain loop, not two generator + # expressions: this runs per role on every parse and a genexp + # costs a frame of its own (frame budget). An EMPTY role needs no + # guard of its own: every arm below iterates the part, so the walk + # is inert on one, and a `continue` ahead of it survived the whole + # suite and moved neither frame count (measured 2026-09-20, + # py3.11: parse=406.00 facade=443.00 with it and without it). for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): part = _idx(tokens, role) - if part and all("particle" in tokens[i].tags for i in part): + all_particle = True + conj: list[int] = [] + others = 0 + for i in part: + tags = tokens[i].tags + if "particle" not in tags: + all_particle = False + if "conjunction" in tags: + conj.append(i) + else: + others += 1 + elif "conjunction" in tags: + conj.append(i) + if all_particle: for i in part: tokens[i] = dataclasses.replace( tokens[i], tags=tokens[i].tags | {UNJOINED_TAG}) + elif conj and not others: + # #461: nothing here for the connective to join. The `elif` + # is deliberate -- where the part is all-particle R2's mark + # already readmits every word, a word that is BOTH particle + # and connective included, which is what keeps a caller's + # `add(particles={"y"})` readings unchanged. + for i in conj: + tokens[i] = dataclasses.replace( + tokens[i], + tags=tokens[i].tags | {UNJOINED_CONJUNCTION_TAG}) _mark_suffix_entries(tokens, state) return dataclasses.replace(state, tokens=tuple(tokens), ambiguities=tuple(ambiguities)) diff --git a/nameparser/_render.py b/nameparser/_render.py index b76ef461..435dec43 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -15,7 +15,8 @@ import re from nameparser._lexicon import Lexicon, _normalize -from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, UNJOINED_TAG, +from nameparser._types import (FOLDED_TAG, UNCLASSIFIED_TAG, + UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, Ambiguity, ParsedName, Role, Token) _SPACES = re.compile(r"\s+") @@ -41,6 +42,14 @@ #: as a particle, so it is a name word of the part like the rest. #: Not STABLE_TAGS -- that also contains "initial", which must contribute. _SKIP_TAGS = frozenset({"particle", "conjunction"}) +#: The given group's own skip set (rules.md#R3, #461): a connective +#: contributes no initial in ANY group where it is joining words, so +#: the given group no longer exempts it -- "one rule for every group". +#: The particle exemption stays: a given-group particle is a name word +#: there, which is what the whole-group exemption was for. +_SKIP_TAGS_GIVEN = frozenset({"conjunction"}) +#: Either unjoined mark readmits the word it sits on. +_UNJOINED_MARKS = frozenset({UNJOINED_TAG, UNJOINED_CONJUNCTION_TAG}) # Ported verbatim from v1 (nameparser/config/regexes.py "initial", minus # the empty alternative) -- layering forbids importing the pipeline here; @@ -152,11 +161,13 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str delimiter follows each initial, separator sits between initials within a group. Each group is ordered the way its FIELD is ordered -- written order, except folded words, which initial - before the rest of the group (#408). Tokens tagged - particle/conjunction contribute no - initial in middle/family (given-name tokens always contribute), - and the unjoined mark readmits the words of an all-particle part - whichever of those tags they carry; tags come from the pipeline -- + before the rest of the group (#408). + A token tagged conjunction contributes no initial in ANY group, + and one tagged particle contributes none in middle/family + (given-group particles always contribute); either unjoined mark + readmits the word it sits on, so the words of an all-particle part + and a connective with nothing in its part to join both count; + tags come from the pipeline -- hand-built untagged tokens all contribute, and so do the words of a field spliced in by replace(), which the parse never read. This view takes NO lexicon, so it has none to fall back to for @@ -174,10 +185,10 @@ def initials(name: ParsedName, spec: str, delimiter: str, separator: str) -> str for key in _INITIALS_KEYS: role = Role(key) tokens = name.tokens_for(role) - if role is not Role.GIVEN: - tokens = tuple(t for t in tokens - if not (_SKIP_TAGS & t.tags) - or UNJOINED_TAG in t.tags) + skip = _SKIP_TAGS_GIVEN if role is Role.GIVEN else _SKIP_TAGS + tokens = tuple(t for t in tokens + if not (skip & t.tags) + or _UNJOINED_MARKS & t.tags) # mechanisms.md#FOLDED_TAG: "a rule that needs different # rendering order tags the token, and the rendering views # consult the tag" -- this is a rendering view, so it reads @@ -209,8 +220,9 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # particle's work there" -- UNJOINED_TAG is that mark (#407). # Only the PARTICLE conjunct is gated on it, and that is the rule # rather than an omission -- rules.md#R4: "A CONJUNCTION keeps its - # lowercase even inside such a part, being no name word in any - # part" -- so a conjunction keeps conjunction treatment even + # lowercase wherever it stands, including inside a part whose + # other words the unjoined mark has turned into ordinary name + # words" -- so a conjunction keeps conjunction treatment even # inside a part the mark has turned into ordinary name words. # No SHIPPED name witnesses the difference: `particles` and # `conjunctions` are disjoint in the default vocabulary and in @@ -225,15 +237,18 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # 'Anh Y Van'. That is pinned by test_repair_keeps_a_conjunction_ # lowercase_in_a_particle_part -- until which gating it passed the # whole suite. - # initials() does NOT match this carve-out, and the mismatch is - # recorded rather than fixed: #461 made it match and was backed - # out, the mark being a statement about a whole PART that #461 - # honored for some of the part's words and not for one of them, - # so what is in question is R3's "even then" clause rather than - # the code (decisions.md, under R2). Under that same lexicon - # `Anh y Van` repairs to 'Anh y Van' and initials 'A. y. V.', - # pinned by - # test_initials_readmits_a_conjunction_in_a_particle_part. + # initials() does NOT match this carve-out, and since #461 that + # is a DECIDED disagreement rather than a recorded one: a + # connective that initials because it joins nothing is still not + # written the way a name is written, which is the sentence quoted + # above and this rule's own reason rather than a borrowing from + # R3. Under that same lexicon `Anh y Van` repairs to 'Anh y Van' + # and initials 'A. y. V.' -- the two views agreeing on this row + # because R2's mark readmits the word for both -- while + # `parse("Juan de y")` repairs to 'Juan de y' and initials + # 'J. y.', where they part. Pinned by + # test_initials_readmits_a_conjunction_in_a_particle_part and + # test_repair_keeps_a_lone_connective_lowercase_where_it_initials. # That conjunct reads the TAG, not the word (#458). classify takes # the conjunction-versus-initial decision once, over the whole # token -- v1's is_conjunction excludes initials, so 'E.' in diff --git a/nameparser/_types.py b/nameparser/_types.py index 567bf079..ad035588 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -114,6 +114,19 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] #: keeping it leaves a later rule free to report the fork this decides. UNJOINED_TAG = "vocab:unjoined-particle" +#: A name part holding no word a connective could join -- once the +#: part's WORKING particles are set aside -- is a part where the +#: connective is not doing a connective's work, so it initials like an +#: ordinary name word, agreeing with `family_base` (rules.md#R3, +#: #461). The twin of UNJOINED_TAG above, computed in the same pass +#: and for the same reason: the fact is about a PART, so it is decided +#: once where the parts are settled rather than re-derived by each +#: view -- which is how `initials()` and `family_base` came apart. +#: A SEPARATE marker rather than a widening of UNJOINED_TAG, because +#: that one is also read by family_particles/family_base and by +#: _cap_word, and none of those three may move. +UNJOINED_CONJUNCTION_TAG = "vocab:unjoined-conjunction" + #: The one sanctioned view-reorder marker (namespaced = unstable API). #: Tokens cannot reorder (span order is validated), so a role fold that #: must render BEFORE the role's original tokens tags them with this; @@ -702,21 +715,32 @@ def _validated_field_strings(fields: dict[str, str]) -> dict[Role, str]: def _remarked(tokens: list[Token]) -> tuple[Token, ...]: - """UNJOINED_TAG recomputed over an edited token list. - - The mark says a particle stands ALONE in its part, which is a fact - about the part rather than the word, so an edit that re-roles - tokens invalidates it in both directions: replace()/revise() splice - a sub-parse's tokens into one field, and a particle marked alone - there can land beside a name word (stale mark) while an unmarked - one can end up alone (missing mark). Parser.revise strips - FOLDED_TAG for the same reason; this one is RECOMPUTED rather than - stripped, because absent is only correct for half the cases. + """Both unjoined marks recomputed over an edited token list. + + Each mark says a word stands in a part with nothing for it to join + -- a particle alone among particles (UNJOINED_TAG), a connective + among connectives and working particles + (UNJOINED_CONJUNCTION_TAG) -- which is a fact about the part rather + than the word, so an edit that re-roles tokens invalidates it in + both directions: replace()/revise() splice a sub-parse's tokens + into one field, and a word marked alone there can land beside a + name word (stale mark) while an unmarked one can end up alone + (missing mark). Parser.revise strips FOLDED_TAG for the same + reason; these are RECOMPUTED rather than stripped, because absent + is only correct for half the cases. + + The two predicates mirror the pipeline's single walk, including + its precedence: where the part is ALL particles the first mark + already readmits every word of it, a word that is both particle + and connective included, so the second is not written there. """ out = list(tokens) for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): part = [i for i, t in enumerate(out) if t.role is role] alone = bool(part) and all("particle" in out[i].tags for i in part) + lone_conj = (not alone) and not any( + "conjunction" not in out[i].tags and "particle" not in out[i].tags + for i in part) for i in part: tags = out[i].tags if alone and UNJOINED_TAG not in tags: @@ -724,6 +748,14 @@ def _remarked(tokens: list[Token]) -> tuple[Token, ...]: elif not alone and UNJOINED_TAG in tags: out[i] = dataclasses.replace(out[i], tags=tags - {UNJOINED_TAG}) + tags = out[i].tags + mark = lone_conj and "conjunction" in tags + if mark and UNJOINED_CONJUNCTION_TAG not in tags: + out[i] = dataclasses.replace( + out[i], tags=tags | {UNJOINED_CONJUNCTION_TAG}) + elif not mark and UNJOINED_CONJUNCTION_TAG in tags: + out[i] = dataclasses.replace(out[i], + tags=tags - {UNJOINED_CONJUNCTION_TAG}) return tuple(out) diff --git a/tests/test_initials.py b/tests/test_initials.py index 09ab620b..59c1bea8 100644 --- a/tests/test_initials.py +++ b/tests/test_initials.py @@ -283,8 +283,15 @@ def test_initials_follow_the_one_case_fork_on_both_letters(self) -> None: # carry the split and its closing. hn = HumanName("john e smith") self.m(hn.initials(), "j. e. s.", hn) + # 'maria y lopez' moved with #461, not with the one-case fork: + # its 'y' holds the middle part alone, so there is nothing + # there for it to join and it initials like any other name + # word, agreeing with the parse's own view. rules.md#R3 states + # the rule for all three groups and decisions.md#R3 records + # what it costs. The letter's TAG is unchanged -- a lowercase + # 'y' outside the marked set is still the connective. hn = HumanName("maria y lopez") - self.m(hn.initials(), "m. l.", hn) + self.m(hn.initials(), "m. y. l.", hn) # These two are among #528's four movers, and the #528 ledger # rule's own name_regex (a literal alternation, in # tools/differential/expected_since_1.4.0.toml) covers all four diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 80ab902f..b131f5da 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1237,7 +1237,13 @@ def _check_cjk_shape_purity(self) -> None: "shape 1's given-first arrangement under the default " "order, exactly as the Cyrillic twins above do -- a " "shape tag asserts the ARRANGEMENT, not a script " - "(tools/differential/shapes.py)", + "(tools/differential/shapes.py). Since #461 it is also " + "the readmission read as a PARITY RESTORATION rather " + "than a deviation: 'و' is the whole middle part, so it " + "joins nothing, initials again on both views, and both " + "give 'م. و. ع.' -- which is what 1.4.0 gave. Roles " + "unmoved throughout, so the classification does not " + "change; what moves is a view, and it moves back", shape=1), # #289: a bare ambiguous credential acronym is read by the # EVIDENCE the writing carries, and a name written in more than @@ -2659,6 +2665,100 @@ def _check_cjk_shape_purity(self) -> None: "middle 'Garcia y Lopez' -- measured. Identical at " "1.4.0", shape=1), + # ---- #461: a connective initials where it joins nothing -------- + # The rule, one sentence for all three groups: a connective + # contributes nothing where it is JOINING -- a part holding + # another name word for it to join, the part's working particles + # set aside -- and initials like any other name word where its + # part holds nothing else. Asked of the whole PART, never of a + # word count. These rows carry the ROLES, which this table's + # expectations are; the initials each one is about are named in + # its notes and asserted by rules.md#R3's example lines and by + # tests/v2/test_render.py. + Case("a_connective_among_given_names_stops_initialing", + "John and Jane Smith", + {"given": "John and Jane", "family": "Smith"}, + classification="fix(#461)", + notes="the name R3's retired paragraph said had four " + "candidate answers, and this rule picks one of them by " + "stating a rule rather than by choosing a value: the " + "given part holds name words for 'and' to join, so it " + "joins and contributes nothing. Roles unchanged at " + "every release; initials go 'J. a. J. S.' -- 1.4.0's " + "run-together 'J a J. S.' -- to 'J. J. S.'", + shape=1), + Case("a_title_run_read_as_a_name_loses_its_connective_too", + "Duke of Edinburgh", + {"given": "Duke of Edinburgh"}, + classification="fix(#461)", + ambiguities=("given-or-family",), + notes="the same rule on the shape that makes it visible " + "outside couple listings: the whole name joins into " + "one given part, 'of' has name words on both sides of " + "it, and 'D. o. E.' becomes 'D. E.'. The report is " + "unchanged and is the one this row also pins", + shape=1), + Case("a_connective_holding_a_part_alone_initials", + "Juan y Garcia", + {"given": "Juan", "middle": "y", "family": "Garcia"}, + classification="fix(#461)", + notes="the other direction, and the commonest of the three " + "shapes: P3's three-word carve-out leaves 'y' a name " + "word in the middle part BY ITSELF, so there is " + "nothing there for it to join and it initials -- 'J. " + "G.' becomes 'J. y. G.'. Roles unchanged since 1.4.0. " + "This is the half the design flagged and Derek took", + shape=1), + Case("a_lone_connective_in_the_family_initials_too", + "Juan de y", + {"given": "Juan", "family": "de y"}, + classification="fix(#461)", + notes="the row #461 was filed about, where the two views of " + "one parse disagreed about one token: family_base was " + "'y' and initials() was 'J.', so the base said the " + "word is a name word and the initials said it is not. " + "Now 'J. y.', and they agree. The part is not " + "all-particle under the default vocabulary -- 'y' is " + "no particle there -- so only the new mark readmits it", + shape=1), + Case("a_part_of_two_words_with_a_name_word_still_joins", + "Jon Dough and", + {"given": "Jon", "family": "Dough and"}, + classification="parity", + notes="the recorded negative control for the WHOLE-PART " + "question, and the row that says the criterion is not " + "a word count wearing a part's clothes: two words, one " + "of them the connective, and the other is a name word " + "for it to join -- so it joins, contributes nothing, " + "and 'J. D.' is unchanged at every release including " + "1.4.0. Read the part, never count it", + shape=1), + Case("a_working_particle_does_not_count_as_something_to_join", + "Juan Velasquez y Garcia", + {"given": "Juan", "family": "Velasquez y Garcia"}, + classification="parity", + notes="the control on the other side: the family holds two " + "name words beside the connective, so it joins and " + "'J. V. G.' is unchanged -- 1.4.0 gave the facade's " + "run-together 'J. V G.' for its own reason (grouping, " + "not membership). The clause about working particles " + "is what this row does NOT exercise and the " + "'Juan de y' row above does", + shape=1), + Case("a_joined_cyrillic_connective_stops_initialing", + "Хосе И Мария Сантос", + {"given": "Хосе И Мария", "family": "Сантос"}, + classification="fix(#461)", + notes="the non-Latin half of the losing direction, and it " + "moves the OTHER way from the Arabic " + "'caseless_connective_never_enters_the_fork' row above " + "because the part differs and not the script: here the " + "connective has given names on both sides of it, so it " + "joins. 'Х. И. М. С.' becomes 'Х. М. С.'. A Cyrillic " + "capital is not the shape an initial is written in " + "(rules.md#P3), so the letter is the connective in " + "every spelling of this name", + shape=1), # ---- #531: the given part's trailing slot ---------------------- # The slot: after a family comma the comma has already named the # family and the first word after it is the given name, so a class diff --git a/tests/v2/pipeline/test_post_rules.py b/tests/v2/pipeline/test_post_rules.py index 3f5fcab0..cb9ff6a2 100644 --- a/tests/v2/pipeline/test_post_rules.py +++ b/tests/v2/pipeline/test_post_rules.py @@ -3,7 +3,7 @@ import pytest -from nameparser import parse +from nameparser import Parser, parse from nameparser._lexicon import Lexicon from nameparser._pipeline import run from nameparser._pipeline._post_rules import suffix_entries @@ -11,8 +11,9 @@ from nameparser._policy import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, GIVEN_FIRST, PatronymicRule, Policy, Script) -from nameparser._types import (FOLDED_TAG, STABLE_TAGS, AmbiguityKind, - Role) +from nameparser._types import (FOLDED_TAG, STABLE_TAGS, + UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, + AmbiguityKind, Role) # A reduced lexicon, the convention in every pipeline stage module: a # stage test should not move when shipped vocabulary does. What it must @@ -1158,3 +1159,62 @@ def test_p6_keeps_every_non_capital_spelling_of_the_particle_member( assert name.family == family, text assert [a.kind.value for a in name.ambiguities] == \ ["particle-or-given"], text + + +def test_a_lone_connective_is_marked_as_joining_nothing() -> None: + # rules.md#R3's mark (#461), decided where the parts are settled. + r = parse("Juan de y") + y = [t for t in r.tokens if t.text == "y"][0] + assert UNJOINED_CONJUNCTION_TAG in y.tags + assert UNJOINED_TAG not in y.tags + + +def test_a_joining_connective_carries_no_mark() -> None: + # the recorded negative control: name words beside it, so it is + # doing a connective's work and the mark is not written. + r = parse("John and Jane Smith") + conj = [t for t in r.tokens if t.text == "and"][0] + assert UNJOINED_CONJUNCTION_TAG not in conj.tags + # and the two-word part, where the criterion is the PART and not + # a count + r2 = parse("Jon Dough and") + conj2 = [t for t in r2.tokens if t.text == "and"][0] + assert UNJOINED_CONJUNCTION_TAG not in conj2.tags + + +def test_an_all_particle_part_keeps_r2s_mark_and_gains_no_other( +) -> None: + # the `elif`'s precedence, and the reason it is an `elif`: where + # the part is all particles R2's mark already readmits every word, + # a word that is BOTH particle and connective included. Swapping + # the branches moves the caller's own rows. + p = Parser(lexicon=Lexicon.default().add(particles={"y"})) + r = p.parse("Juan de y") + y = [t for t in r.tokens if t.text == "y"][0] + assert UNJOINED_TAG in y.tags + assert UNJOINED_CONJUNCTION_TAG not in y.tags + assert r.initials() == "J. d. y." + + +def test_a_word_that_is_both_particle_and_connective_is_marked_too( +) -> None: + # The walk sorts the part's words into three buckets and a word of + # BOTH vocabularies belongs in the connective one, which is what + # the second arm of the sort does. The shape that needs it: a part + # that is not all-particle -- so R2's mark does not fire -- holding + # a plain connective, a working particle, and a word that is both. + # Without the arm the both-vocabulary word takes neither mark, + # drops as an ordinary family particle, and the part's own base + # disagrees with the view over it. + p = Parser(lexicon=Lexicon.default().add(particles={"y"})) + r = p.parse("Juan de y and") + assert r.family == "de y and" + assert r.initials() == "J. y. a." + tags = {t.text: t.tags for t in r.tokens if t.role is Role.FAMILY} + assert UNJOINED_CONJUNCTION_TAG in tags["y"] + assert UNJOINED_CONJUNCTION_TAG in tags["and"] + # the working particle beside them takes no mark and does not + # initial, which is what makes the criterion "nothing to join" + # rather than "nothing else here" + assert UNJOINED_CONJUNCTION_TAG not in tags["de"] + assert UNJOINED_TAG not in tags["de"] diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index a89b7b3a..dd6e15b1 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -703,9 +703,17 @@ def test_token_is_conjunction_reads_the_tag_then_the_vocabulary() -> None: # the vocabulary-and-shape test said about either. parsed = HumanName("JUAN Y GARCIA") tags = {t.text: t for t in parsed._parsed.tokens} - assert parsed._token_is_conjunction(tags["Y"]) is True + # #461: the 'Y' holds the middle part alone, so the parse marked + # it as joining nothing and the predicate reads that mark beside + # the tag. The CONTROL below is a 'Y' that IS joining, where the + # tag still answers on its own. + assert parsed._token_is_conjunction(tags["Y"]) is False assert parsed._token_is_conjunction(tags["JUAN"]) is False + joining = HumanName("JUAN GARCIA Y LOPEZ") + jtags = {t.text: t for t in joining._parsed.tokens} + assert joining._token_is_conjunction(jtags["Y"]) is True + fork = HumanName("john e smith") e = {t.text: t for t in fork._parsed.tokens}["e"] assert fork._token_is_conjunction(e) is False @@ -758,10 +766,22 @@ def test_process_initial_direct_call_keeps_the_v1_string_path() -> None: def test_process_initial_with_tokens_reads_the_parse() -> None: # The same two name parts, this time handed the tokens the parse - # built: the answer flips on both, which is #528 in one assertion. + # built: the answer flips, which is #528 in one assertion. + # #461 moved the FIRST half's value and the flip with it: the 'Y' + # of 'JUAN Y GARCIA' holds the middle part alone, so the parse + # marks it as joining nothing and this view takes its letter -- + # which is what the bare-string path above already said about the + # same text. The flip the first half pinned is re-pinned below on + # a 'Y' that IS joining, where the two paths still disagree; + # rules.md#R3 states the rule and decisions.md#R3 records it. hn = HumanName("JUAN Y GARCIA") middle = hn._list_tokens_for("middle")[0] - assert hn._process_initial("", firstname=False, tokens=middle) == "" + assert hn._process_initial("", firstname=False, tokens=middle) == "Y" + joining = HumanName("JUAN GARCIA Y LOPEZ") + joined_y = [g for g in joining._list_tokens_for("last") + if g[0].text == "Y"][0] + assert joining._process_initial("", firstname=False, + tokens=joined_y) == "" fork = HumanName("john e smith") fork_middle = fork._list_tokens_for("middle")[0] assert fork._process_initial("", firstname=False, @@ -921,11 +941,20 @@ def test_initials_freeze_the_connective_answer_at_parse_time() -> None: # next full_name assignment, not on the next initials() call. # A local Constants, never CONSTANTS: the shared singleton would # leak the removal into every later test in the process. + # #461 moved the VALUE on 'juan y garcia' and with it this name's + # ability to WITNESS the freeze: its 'y' holds the middle part + # alone, so it initials as a marked connective and would initial + # again as a plain name word, and all three readings below are now + # the same string (rules.md#R3, decisions.md#R3). The name is kept + # for its moved value and for the capitalize() precedent; the + # freeze itself is re-pinned under it on a name where the + # connective is JOINING, which is where a vocabulary edit still + # changes the answer. constants = Constants() name = HumanName("juan y garcia", constants=constants) - assert name.initials() == "j. g." + assert name.initials() == "j. y. g." constants.conjunctions.remove("y") - assert name.initials() == "j. g." # frozen at parse time + assert name.initials() == "j. y. g." # frozen at parse time # capitalize() has behaved this way all along, which is the # precedent this cost was accepted on name.capitalize() @@ -933,6 +962,14 @@ def test_initials_freeze_the_connective_answer_at_parse_time() -> None: name.full_name = "juan y garcia" # re-parse applies it assert name.initials() == "j. y. g." + joined = Constants() + joining = HumanName("juan garcia y lopez", constants=joined) + assert joining.initials() == "j. g. l." + joined.conjunctions.remove("y") + assert joining.initials() == "j. g. l." # frozen at parse time + joining.full_name = "juan garcia y lopez" # re-parse applies it + assert joining.initials() == "j. g. y. l." + def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> None: # __setstate__ is the second producer of UNCLASSIFIED_TAG tokens: a @@ -956,9 +993,18 @@ def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> Non # carry the same mark and take the same fallback. Rebuilt here from # the live parse's own fields, so the strings are identical and # only the missing parse explains the difference. + # 'JUAN Y GARCIA' no longer contrasts on INITIALS: #461 gave the + # live parse the restored answer, both views now saying 'J. Y. G.' + # because the 'Y' holds its part alone (rules.md#R3, + # decisions.md#R3). Kept for its CAPITALIZE half, which still + # differs, and replaced on the initials side by + # 'JUAN GARCIA Y LOPEZ', where the letter IS joining and the live + # parse still drops it. 'john e smith' is untouched throughout. for name, live_initials, restored_initials, live_cap, restored_cap in ( - ("JUAN Y GARCIA", "J. G.", "J. Y. G.", + ("JUAN Y GARCIA", "J. Y. G.", "J. Y. G.", "Juan y Garcia", "Juan Y Garcia"), + ("JUAN GARCIA Y LOPEZ", "J. G. L.", "J. G. Y. L.", + "Juan Garcia y Lopez", "Juan Garcia Y Lopez"), ("john e smith", "j. e. s.", "j. s.", "John E Smith", "John e Smith")): assert HumanName(name).initials() == live_initials diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 38d54698..d1c96e11 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -11,7 +11,8 @@ from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, ) -from nameparser._types import AmbiguityKind, Role, Segmentation +from nameparser._types import (UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, + AmbiguityKind, Role, Segmentation) def test_parser_defaults_and_properties() -> None: @@ -1065,6 +1066,69 @@ def test_revise_sets_a_missing_unjoined_mark() -> None: assert (revised.family, revised.family_base) == ("Do", "Do") +def test_revise_recomputes_the_connective_mark_in_both_directions( +) -> None: + # The mark says a connective stands in a part with nothing to + # join, which is a fact about the PART -- so the harvest that + # splices a sub-parse's tokens into one field invalidates it both + # ways, exactly as it does the particle mark above (rules.md#R3, + # #461). + p = Parser() + lone = p.parse("Juan de y") + assert lone.initials() == "J. y." + # STALE: the marked 'y' lands beside a name word + widened = p.revise(lone, family="y Garcia") + assert widened.initials() == "J. G." + # MISSING: a connective revised into a part of its own + narrowed = p.revise(p.parse("Juan Velasquez y Garcia"), family="y") + assert narrowed.initials() == "J. y." + # and the identity revise round-trips, which is the property the + # particle-mark tests above pin for R2 + again = p.revise(lone, family=lone.family) + assert (again.family, again.initials()) == (lone.family, + lone.initials()) + + +def test_revise_writes_a_connective_mark_the_sub_parse_did_not( +) -> None: + # The MISSING direction that role-forcing alone cannot produce, + # and the one input shape that does: the sub-parse of 'and y' + # reads 'and' as the given name and 'y' -- a particle under this + # caller's vocabulary -- as an all-particle family, so R2's mark + # is what the sub-parse writes on it. Forcing both into one field + # makes that part no longer all-particle, so R2's mark is cleared + # and #461's has to be written in its place, by the recompute + # rather than by any stage. Without it the 'y' carries no mark at + # all, drops as an ordinary family particle, and the field and + # the view disagree again. + p = Parser(lexicon=Lexicon.default().add(particles={"y"})) + revised = p.revise(p.parse("John Smith"), family="and y") + assert revised.family == "and y" + assert revised.initials() == "J. a. y." + marks = {t.text: t.tags for t in revised.tokens + if t.role is Role.FAMILY} + assert UNJOINED_CONJUNCTION_TAG in marks["y"] + assert UNJOINED_TAG not in marks["y"] + + +def test_revise_keeps_r2s_precedence_over_the_connective_mark( +) -> None: + # The recompute mirrors the pipeline walk's `elif`, and this is + # the row where the two marks would otherwise both be written: + # 'de y' under a vocabulary that makes 'y' a particle is an + # all-particle part, so R2's mark readmits every word of it -- + # the connective included -- and #461's is not written there. The + # rendered answer is the same either way, which is why this + # asserts the TAGS: the facade's own connective predicate reads + # the second mark, so writing it here would move that view alone. + p = Parser(lexicon=Lexicon.default().add(particles={"y"})) + revised = p.revise(p.parse("John Smith"), family="de y") + assert revised.initials() == "J. d. y." + y = [t for t in revised.tokens if t.text == "y"][0] + assert UNJOINED_TAG in y.tags + assert UNJOINED_CONJUNCTION_TAG not in y.tags + + def test_revise_sub_parse_structural_behavior() -> None: # the docstring's three structural promises, pinned: delimiters # never become tokens, marker words are consumed as in parsing, diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index a2c73350..e9c8f15e 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -8,6 +8,7 @@ """ import dataclasses import hashlib +import itertools import re import warnings @@ -23,7 +24,8 @@ from nameparser._pipeline import run from nameparser._pipeline._state import ParseState from nameparser._pipeline._vocab import effective_script -from nameparser._types import AmbiguityKind, Role +from nameparser._types import (UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, + AmbiguityKind, Role, Token) from .conftest import differential_corpus @@ -987,3 +989,226 @@ def test_bad_policy_field_fails_cleanly(field: str, value: object) -> None: except (ValueError, TypeError): return _quiet_parser(policy=policy).parse("Dr. John de la Vega III") + + +# --- #397/#461: the connective grid, and its five invariants -------- +# A GRID and not a name list, for the reason #531's agreement sweep +# was one: every invariant below is stated about a shape the parser +# can reach in many ways, and a list pins the ways somebody thought +# of. Generators include heads with NO name word, parts of one word, +# a part that is nothing but the connective, comma and no-comma, +# mixed/ALL-CAPS/lower, class members supplied by a CUSTOM lexicon +# (a second single letter that is also generational vocabulary, and a +# connective that is also particle vocabulary), and four policies. + + +def _connective_grid() -> list[tuple[str, Parser, str]]: + heads: tuple[list[str], ...] = ( + [], ["Josep"], ["Josep", "Lluis"], ["Dr."], ["Dr.", "Josep"]) + mids: tuple[list[str], ...] = ( + [], ["Carod"], ["de", "Carod"], ["Carod", "Rovira"]) + conns = ("i", "y", "e", "and", "&", "of", "и") + tails: tuple[list[str], ...] = ( + [], ["Rovira"], ["de", "Rovira"], ["Rovira", "Puig"]) + suffixes: tuple[list[str], ...] = ( + [], ["III"], ["Jr."], ["I"], ["MA"]) + lexicons = (("default", Lexicon.default()), + ("conj+v", Lexicon.default().add(conjunctions={"v"})), + ("part+y", Lexicon.default().add(particles={"y"}))) + policies = (("default", Policy()), + ("family-first", Policy(name_order=FAMILY_FIRST)), + ("given-last", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)), + ("strict-comma", Policy(lenient_comma_suffixes=False))) + texts: list[str] = [] + seen: set[str] = set() + for head, mid, conn, tail, suffix, comma in itertools.product( + heads, mids, conns, tails, suffixes, (False, True)): + if not (mid or tail): + continue + if comma and tail: + base = " ".join(tail) + ", " + " ".join(head + mid + [conn]) + else: + base = " ".join(head + mid + [conn] + tail) + base = (base + " " + " ".join(suffix)).strip() + for written in (base, base.upper(), base.lower()): + if written not in seen: + seen.add(written) + texts.append(written) + parsers = [(f"{ln}/{pn}", Parser(lexicon=lex, policy=pol)) + for ln, lex in lexicons for pn, pol in policies] + return [(t, p, label) for t in texts for label, p in parsers] + + +_CONNECTIVE_GRID = _connective_grid() + + +def _readmitted(tok: Token) -> bool: + return not {UNJOINED_TAG, UNJOINED_CONJUNCTION_TAG}.isdisjoint(tok.tags) + + +def _has_something_to_join(tok: Token, part: tuple[Token, ...]) -> bool: + """rules.md#R3's question, asked of the WHOLE PART. + + Working particles are set aside -- a particle the unjoined mark + has NOT readmitted is doing a particle's work and is no name word + for a connective to join. + """ + return any(other is not tok + and "conjunction" not in other.tags + and not ("particle" in other.tags + and UNJOINED_TAG not in other.tags) + for other in part) + + +def _predicted_initials(part: tuple[Token, ...], role: Role) -> list[str]: + out = [] + for tok in part: + skip = ("conjunction" in tok.tags + or ("particle" in tok.tags and role is not Role.GIVEN)) + if not skip or _readmitted(tok): + out.append(tok.text[0]) + return out + + +def test_the_connective_grid_can_fail() -> None: + """The reachability probe every grid in this file carries. + + A grid that reaches none of the shapes it is about passes + vacuously, and a comparison over a population of zero is the same + silence as a clean run. These three counts are a dated recorded + control, measured 2026-09-20 on the shipped tree. + """ + assert len(_CONNECTIVE_GRID) == 170100, len(_CONNECTIVE_GRID) + assert len({t for t, _, _ in _CONNECTIVE_GRID}) == 14175 + joined = sum(1 for text, parser, _ in _CONNECTIVE_GRID[:2000] + if any("conjunction" in tok.tags + for tok in parser.parse(text).tokens)) + assert joined > 500, joined + + +def test_a_generational_connective_joins_only_with_both_sides() -> None: + """INV1 (#397). For a single-letter connective that is ALSO + generational vocabulary: if it shares its role part with any other + word, a non-connective word stands before it AND after it. + + Mutation-checked, 2026-09-20: dropping the position test fails + this on 1575 parses, dropping the whole both-sides condition on + 504, and dropping the rootname count arm on 816. The CLASS test + is NOT covered here and has its own rows -- see + tests/v2/pipeline/test_group.py. + """ + failures = [] + for text, parser, label in _CONNECTIVE_GRID: + letters = {w for w in parser.lexicon.conjunctions + if len(w) == 1 and w in parser.lexicon.suffix_words} + name = parser.parse(text) + for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): + part = name.tokens_for(role) + if len(part) < 2: + continue + for i, tok in enumerate(part): + if ("conjunction" not in tok.tags or len(tok.text) != 1 + or tok.text.lower() not in letters): + continue + left = any("conjunction" not in t.tags for t in part[:i]) + right = any("conjunction" not in t.tags for t in part[i + 1:]) + if not (left and right): + failures.append( + f"[{label}] {text!r}: {role.value} {tok.text!r}") + assert not failures, ( + f"{len(failures)} parse(s) joined a generational connective " + f"without a name word on each side:\n" + "\n".join(failures[:10])) + + +def test_the_mark_is_exactly_the_parts_with_nothing_to_join() -> None: + """INV2 (#461). A connective carries a readmitting mark IFF its + part holds no other word that is neither a connective nor a + working particle. + + Stated over the PAIR of marks deliberately: keyed on the new mark + alone it fails 366 times under `add(particles={"y"})`, where the + part is all-particle and R2's OLD mark does the readmitting. + Mutation-checked: dropping the new mark fails this on 14,066. + """ + failures = [] + for text, parser, label in _CONNECTIVE_GRID: + name = parser.parse(text) + for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): + part = name.tokens_for(role) + for tok in part: + if "conjunction" not in tok.tags: + continue + if _has_something_to_join(tok, part) == _readmitted(tok): + failures.append( + f"[{label}] {text!r}: {role.value} {tok.text!r} " + f"joinable={_has_something_to_join(tok, part)} " + f"marked={_readmitted(tok)}") + assert not failures, ( + f"{len(failures)} connective token(s) disagree with the " + f"criterion:\n" + "\n".join(failures[:10])) + + +def test_initials_take_only_base_words_and_drop_only_joiners() -> None: + """INV3 and INV4, beside R2's existing invariant. + + INV3: every family word `initials()` contributes is a word of + `family_base`. INV4: every `family_base` word that contributes no + initial is tagged conjunction. Together they say the two views of + one parse cannot come apart again, which is the defect #461 was + filed about. + + Mutation-checked: swapping the two branches of the mark walk fails + this on 188 parses (`Carod y` under `add(particles={"y"})`, whose + base is empty while its `y` initials). + """ + failures = [] + for text, parser, label in _CONNECTIVE_GRID: + name = parser.parse(text) + family = name.tokens_for(Role.FAMILY) + base = name.family_base.split() + contributing = [t for t in family + if t.text[0] in _predicted_initials(family, + Role.FAMILY) + and (not ("conjunction" in t.tags + or "particle" in t.tags) + or _readmitted(t))] + for tok in contributing: + if tok.text not in base: + failures.append( + f"[{label}] {text!r}: INV3 {tok.text!r} initials but " + f"is not in base {base!r}") + for tok in family: + if (tok.text in base and tok not in contributing + and "conjunction" not in tok.tags): + failures.append( + f"[{label}] {text!r}: INV4 {tok.text!r} is a base " + f"word, contributes no initial, and is no connective") + assert not failures, ( + f"{len(failures)} disagreement(s) between initials() and " + f"family_base:\n" + "\n".join(failures[:10])) + + +def test_initials_emit_exactly_the_predicted_contributors() -> None: + """INV5. Per group, `initials("{role}")` emits exactly the words + the criterion predicts, in field order. + + The output-level statement of the same rule, and the one that + catches a view reading the marks correctly and then rendering + something else. Mutation-checked: restoring the given group's old + blanket exemption fails this on 27,343 parses, and dropping the + new mark from the readmitting set on 14,066. + """ + failures = [] + for text, parser, label in _CONNECTIVE_GRID: + name = parser.parse(text) + for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): + predicted = _predicted_initials(name.tokens_for(role), role) + got = name.initials( + f"{{{role.value}}}").replace(".", "").split() + if got != predicted: + failures.append( + f"[{label}] {text!r}: {role.value} {got!r} != " + f"predicted {predicted!r}") + assert not failures, ( + f"{len(failures)} group(s) emitted something other than the " + f"criterion's contributors:\n" + "\n".join(failures[:10])) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index c83ddbd6..a49f6032 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -180,23 +180,24 @@ def test_repair_keeps_a_conjunction_lowercase_in_a_particle_part() -> None: def test_initials_readmits_a_conjunction_in_a_particle_part() -> None: - """Today's answer on an OPEN question (#461), pinned as such. - - rules.md#R3 says a conjunction never initials "even then" -- even - inside the all-particle part R2 turns into ordinary name words -- - and `initials()` does not do that: the mark readmits the part's - words whichever skip tag they carry. #461 made the code match the - clause and was backed out, the clause rather than the code being - what is now in question (decisions.md, under R2). - - So this pins what a `deviates:` marker would pin if one could - hang here -- TODAY's output, strictly, so that settling #461 - fails the suite until this moves with it. It cannot be a marker: - markers hang on rules.md example lines and every line there names - an input string parsed with the DEFAULT vocabulary, over which - `particles` and `conjunctions` are disjoint and no string reaches - this shape. It is also what keeps the values quoted in prose by - decisions.md, mechanisms.md#RENDER-HONORS-THE-PARSE and + """rules.md#R3, settled (#461). + + This pinned TODAY's output on an open question so that settling it + would fail the suite until the pin moved with it. It fired, and + this is the pin moved: a connective contributes no initial where + it is JOINING, and a part holding nothing else for it to join is a + part where it initials like any other name word, agreeing with the + base. The all-particle rows below are unchanged, because R2's mark + already readmitted the word there; what moved is everything under + the default vocabulary, where a lone connective now initials too. + decisions.md#R3 records the rule and what it costs. + + Still not a `deviates:` marker and still not a rules.md example + line: markers hang on example lines and every line there names an + input parsed with the DEFAULT vocabulary, over which `particles` + and `conjunctions` are disjoint, so no string reaches the + all-particle-with-a-connective shape. It is also what keeps the + values quoted in prose by decisions.md, mechanisms.md and `_render.py` from going stale unnoticed. """ assert "y" in Lexicon.default().conjunctions, ( @@ -213,13 +214,43 @@ def test_initials_readmits_a_conjunction_in_a_particle_part() -> None: assert p.parse("Anh y Van").initials() == "A. y. V." assert p.parse("johnny y").initials() == "j. y." - # and OUTSIDE such a part the skip stands, joining or not -- - # these are what the readmission must not reach + # OUTSIDE such a part the skip stands where the connective is + # JOINING, and these are what the readmission must not reach -- + # the question is asked of the whole PART and never of a word + # count, which is what keeps a part of two words with a name word + # in it on this side of the line assert p.parse("Juan Velasquez y Garcia").initials() == "J. V. G." + assert parse("Jon Dough and").initials() == "J. D." + # Under THIS lexicon 'Juan y Garcia' is one of them, and for a + # reason worth spelling out: making 'y' a particle folds it into + # the family, so the part is 'y Garcia' and holds a name word for + # it to join. Unchanged, where the same string under the default + # vocabulary moves -- which is the row below. assert p.parse("Juan y Garcia").initials() == "J. G." - # including under the default vocabulary, where 'y' is no particle - # and the family is therefore not all-particle - assert parse("Juan de y").initials() == "J." + # and it DOES reach a part holding nothing else, under the default + # vocabulary too, where 'y' is no particle and only the new mark + # readmits it + assert parse("Juan y Garcia").initials() == "J. y. G." + assert parse("Juan de y").initials() == "J. y." + assert parse("Juan y").initials() == "J. y." + + +def test_repair_keeps_a_lone_connective_lowercase_where_it_initials( +) -> None: + """rules.md#R4's own reading, and the view split it accepts. + + A connective that initials BECAUSE it joins nothing is still not + written the way a name is written, so case repair leaves it + lowercase while initials() takes its letter. That is R4's reason + rather than a borrowing from R3 -- the two rules answer different + questions about the same token and this is the row where their + answers part. mechanisms.md#RENDER-HONORS-THE-PARSE records the + shape; decisions.md#R4 records the split. + """ + assert parse("Juan de y").initials() == "J. y." + assert parse("Juan de y").capitalized().family == "de y" + assert parse("juan y").capitalized(force=True).given == "Juan" + assert parse("juan y").capitalized(force=True).family == "y" def test_initials_order_folded_words_first_like_the_family_field() -> None: @@ -693,8 +724,12 @@ def test_facade_initials_follow_the_one_case_fork() -> None: """ assert parse("john e smith").initials() == "j. e. s." assert HumanName("john e smith").initials() == "j. e. s." - assert parse("JUAN Y GARCIA").initials() == "J. G." - assert HumanName("JUAN Y GARCIA").initials() == "J. G." + # #461 moved the VALUE and not the agreement: 'Y' holds the middle + # part alone, so it initials on both surfaces -- which is also + # 1.4.0's answer on this name, restored. The joined control below + # is where the letter still drops, on both surfaces. + assert parse("JUAN Y GARCIA").initials() == "J. Y. G." + assert HumanName("JUAN Y GARCIA").initials() == "J. Y. G." assert parse("JUAN GARCIA Y LOPEZ").initials() == "J. G. L." assert HumanName("JUAN GARCIA Y LOPEZ").initials() == "J. G. L." # The mixed-case controls, where the writing decides the letter and @@ -703,11 +738,30 @@ def test_facade_initials_follow_the_one_case_fork() -> None: assert HumanName("Juan Y. Garcia").initials() == "J. Y. G." # 'maria y lopez' is a ONE-CASE control, not a mixed-case one: written # wholly in lowercase, its 'y' is outside the marked set (rules.md#P3), - # so it stays the connective and drops on both surfaces -- unmoved, - # like the mixed-case names above, but for the vocabulary's reason - # rather than the writing's (tests/v2/test_ledger_guards.py's - # "one-case controls" wording, around line 1118). - assert HumanName("maria y lopez").initials() == "m. l." + # so it stays the CONNECTIVE on both surfaces rather than reading as + # an initial the way 'e' does (tests/v2/test_ledger_guards.py's + # "one-case controls" wording, around line 1118). What it no longer + # witnesses is the DROP: #461 gave the letter its initial back here, + # because it holds the middle part alone and so joins nothing + # (decisions.md#R3). The joining one-case control below is where + # a lowercase 'y' still drops, which is what keeps this row's point + # -- the tag, not the value -- observable. + assert HumanName("maria y lopez").initials() == "m. y. l." + assert parse("maria y lopez").initials() == "m. y. l." + assert HumanName("juan garcia y lopez").initials() == "j. g. l." + assert parse("juan garcia y lopez").initials() == "j. g. l." + # The GIVEN group, where #461 dropped the facade's own blanket + # exemption as well as the core's: a connective among given names + # is joining there like anywhere else, so it contributes nothing + # on BOTH surfaces. Without the facade half these read 'J. a. J. + # S.' and 'D. o. E.' while the core reads them as below, which is + # the disagreement the one rule exists to prevent. + assert parse("John and Jane Smith").initials() == "J. J. S." + assert HumanName("John and Jane Smith").initials() == "J. J. S." + assert parse("Duke of Edinburgh").initials() == "D. E." + assert HumanName("Duke of Edinburgh").initials() == "D. E." + assert parse("John & Jane").initials() == "J. J." + assert HumanName("John & Jane").initials() == "J. J." # The one corpus name where the two views still differ, and it is # not this rule's: the facade merges 'Ph.' + 'D.' into one list # element and renders it with no inner delimiter diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 91af5059..90e4da5d 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -63,6 +63,7 @@ "Dr. Smith née Jones" "Dr. Smith, John" "Dr. abdul salam" +"Duke of Edinburgh" "Esq. Smith" "Freiherr von Berg MA" "Freiherr von Richthofen V" @@ -162,6 +163,7 @@ "John Smith, Mr. Jr." "John Smith, PhD" "John Smith, V." +"John and Jane Smith" "John née Jones Smith MA" "John née Jones Smith Ma" "John née Jones Smith V" @@ -175,6 +177,7 @@ "John van der Berg née Jones" "John van der J. V" "John😀Smith" +"Jon Dough and" "Jones née" "Jong Anke de" "Jong, Anke de" @@ -186,6 +189,7 @@ "Jr." "Juan & Garcia" "Juan McDonald" +"Juan Velasquez y Garcia" "Juan and Garcia" "Juan de" "Juan de la Vega" diff --git a/tools/differential/corpus_shapes.jsonl b/tools/differential/corpus_shapes.jsonl index 50d325fb..e2aebd36 100644 --- a/tools/differential/corpus_shapes.jsonl +++ b/tools/differential/corpus_shapes.jsonl @@ -3,6 +3,7 @@ {"name": "Bridge (A.B)", "shape": 1} {"name": "Carod i", "shape": 1} {"name": "Dr. Juan de la Vega III", "shape": 1} +{"name": "Duke of Edinburgh", "shape": 1} {"name": "J.A. K.D.", "shape": 1} {"name": "J.R.R. Tolkien", "shape": 1} {"name": "JACK MA", "shape": 1} @@ -61,9 +62,11 @@ {"name": "John Smith Xyz.", "shape": 1} {"name": "John Smith nee Jones R.A.I.", "shape": 1} {"name": "John V. Smith", "shape": 1} +{"name": "John and Jane Smith", "shape": 1} {"name": "John de Ma", "shape": 1} {"name": "John e Smith", "shape": 1} {"name": "John van der Berg Ma", "shape": 1} +{"name": "Jon Dough and", "shape": 1} {"name": "Jose E Maria Santos", "shape": 1} {"name": "Jose e Maria Santos", "shape": 1} {"name": "Josep Carod I Rovira", "shape": 1} @@ -75,7 +78,10 @@ {"name": "Juan Garcia Lopez y", "shape": 1} {"name": "Juan Garcia Y Lopez", "shape": 1} {"name": "Juan Garcia y Lopez y", "shape": 1} +{"name": "Juan Velasquez y Garcia", "shape": 1} {"name": "Juan de la Vega", "shape": 1} +{"name": "Juan de y", "shape": 1} +{"name": "Juan y Garcia", "shape": 1} {"name": "Lt.Gov. John Doe", "shape": 1} {"name": "Maria Kowalska z domu Nowak MA", "shape": 1} {"name": "Md Abdul Karim", "shape": 1} @@ -96,6 +102,7 @@ {"name": "juan garcia y lopez", "shape": 1} {"name": "juan y garcia", "shape": 1} {"name": "ХОСЕ И МАРИЯ САНТОС", "shape": 1} +{"name": "Хосе И Мария Сантос", "shape": 1} {"name": "хосе и мария сантос", "shape": 1} {"name": "محمد و علي", "shape": 1} {"name": "Beethoven, Ludwig van", "shape": 2} From 41de3df63d36dc17eea70d3275d3d00aa77154fc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 06:30:33 -0700 Subject: [PATCH 03/11] fix(#397): review -- the link joins between NAME words, and a generation that joins nothing does not count itself The both-sides condition #397 shipped tested POSITION where it meant CLASS. `not (0 < k < len(pieces) - 1)` asks whether some piece stands on each side of the link, and some piece always does, so a generational suffix standing behind the link was swallowed into the surname in silence: `Josep Lluis Carod i III` read family 'Carod i III' with no suffix at all, where the parent and 1.4.0 both read the generation. `... i Jr.`, `... i I`, `... i V`, `... i MA`, the comma form `Rovira, Josep Carod i Jr.` and the doubled `... i i` went the same way -- 956 names of the review's 93,565-name grid, pinned by no test and by no corpus row. The case note that called the contrast "the whole design" was false of every one of them. The join now asks the NEIGHBOURS' class: a name word is what the REST of the parse reads as one. Two bounds do most of it, and both come from assign's own predicates rather than from a second reading of the vocabulary (mechanisms.md#ONE-PREDICATE-PER-QUESTION) -- `leading_ titles` for the honorifics in front, `trailing_start` for the credential run behind. `trailing_start` is what makes the bare 'V' and 'MA' cases work at all: 'V' is suffix vocabulary written like an initial, so is_suffix_piece refuses it under S2's initial veto and a class test spelled with that predicate alone would let the link take it. Inside the bounds the suffix and title tests still run, for the credential or honorific standing mid-name that neither bound reaches. The walk steps OVER a run of connectives, because a run joins as one -- `Carod i y Rovira` joins on 'Rovira', while `Juan i e` has no name word on that side at all and keeps the parent's reading. It is read once per SEGMENT, lazily, and kept as a TOKEN index: the merges below move piece indices and cannot move that one, and a name with no one-letter connective of the suffix vocabulary pays two tag lookups per piece and no call. The reference frame count is unchanged at 406/443, as are `John Smith`, `Smith, John`, `Juan Garcia y Lopez` and `John and Jane Smith`; the `i` names pay between 17 and 49 frames. An alternating-connective part stays linear from 50 to 800 words (x1.99 per doubling, as before). The frozen set is tested by a piece's FIRST TOKEN rather than with `isdisjoint` over its tokens, and the benchmark's own "and " shape is why: the piece a connective run extends GROWS with every merge, so a test over its tokens costs 1+2+...+n and took 'and ' x3200 from 21.7ms to 41.8ms -- 6.2x per 4x the input against the shape's recorded 4.1x, over `_MAX_RATIO`. The first-token test is exact and not an approximation (a frozen piece is one token and nothing merges it), and the two spellings are output-identical over the review's 93,565-name grid and over every corpus and case text under eight configurations. The COUNT now agrees with the join, which is the review's first question. A link that joins nothing is the generation it also spells, so it no longer counts itself toward P3's three-word carve-out -- `Carod y de Rovira i` returns to given 'Carod', middle 'y', family 'de Rovira', suffix 'i', where the trailing generation had been lending its own word to an unrelated 'y' two pieces away. The same frozen set keeps such a link out of the contiguous-connective run merge. Derek's Q2 decision stands: the count still reaches the CLASS, and `test_the_count_reaches_a_connective_that_is_particle_vocabulary` is green. Two new property invariants, both over a second grid carrying the shapes the first cut got wrong (the trailing numeral and bare acronym the peel takes, a maiden clause, initial-only and particle-only heads, and the suffix comma as a third comma shape; 103,040 entries, 12,880 texts, two lexicons and four policies). INV6, the OFF-SWITCH invariant: for every name, either a link JOINED between two name words -- judged on the parse with the letter out of the connectives, so an absorbed suffix cannot pass itself off as the name word on the right -- or the seven fields are identical to that parse. It fails on 2,360 parses at c8550b64 and on none here. INV1 strengthened: a word the off-switch parse reads as the SUFFIX never lands inside a joined name part. It fails on 960 parses at c8550b64, where INV1 itself -- which inspects the part a join PRODUCED, and so is satisfied by the very defect -- fails on none. Collateral, measured: over the corpus as it stood at the parent, under eight configurations, the bundle moves 2 roles / 3 reports / 46 initials, and this commit moves none of those numbers -- they are identical at c8550b64 and here, per configuration. On the review's 93,565-name grid every one of the 956 names this commit moves lands back on the parent's exact reading, none on a third; and of the 2,369 that still differ from the parent, 761 are a link joining between two name words and the other 1,608 are one-case names where the letter reads as an INITIAL, which is P3's marked-subset clause and not this one. `_process_initial`'s closing comment named a case #461 had emptied (`Vega, Santa de y` now initials 'S. y. V.'). Rewritten to what is true: over 95,119 names no PARSED group reaches that drop, and what keeps the branch alive is the paths with no parse to read -- the keyword constructor and an overridden `*_list` -- both now pinned. `_remarked`'s `lone_conj` takes the same `bool(part)` guard `alone` already had; nothing below the loop can see the difference. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/rules.md | 26 ++- nameparser/_facade.py | 28 ++- nameparser/_pipeline/_group.py | 130 +++++++++++-- nameparser/_types.py | 11 +- tests/v2/cases.py | 115 +++++++++-- tests/v2/pipeline/test_group.py | 129 ++++++++++++- tests/v2/test_facade.py | 24 +++ tests/v2/test_ledger_guards.py | 23 ++- tests/v2/test_properties.py | 258 ++++++++++++++++++++++++- tools/differential/corpus_rules.jsonl | 2 + tools/differential/corpus_shapes.jsonl | 5 + 11 files changed, 688 insertions(+), 63 deletions(-) diff --git a/docs/design/rules.md b/docs/design/rules.md index ebcc3579..fe22580b 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -518,16 +518,22 @@ P3. Rationale: connective words ("y", "of the") bind name words into them, so a rule taking "one name word" takes the whole join and never half of it. A connective counts as a name word wherever this rule counts them, - whatever else the vocabulary says the word is. A word can be a - connective and a generation at once — the Catalan link is also the - roman numeral one — and counting it as the generation would let a - connective raise the bar for its own join, so the shortest linked - name, which is the commonest one, would be the only one that - failed to link. + whatever else the vocabulary says the word is, where it is placed + to join. A word can be a connective and a generation at once — the + Catalan link is also the roman numeral one — and counting it as + the generation would let a connective raise the bar for its own + join, so the shortest linked name, which is the commonest one, + would be the only one that failed to link; while counting it as a + connective where it joins nothing lowers the bar for a different + connective's join instead. A connective that is also generational vocabulary joins only where - a name word stands on each side of it. A connective with nothing - to its right is connecting nothing, and a word of that vocabulary - ending a name is the generation it also spells. + a name word stands on each side of it — a word the rest of the + parse reads as a name word rather than as a credential or an + honorific, looked for past any run of connectives standing + between. A connective with nothing to its right is connecting + nothing, and a word of that vocabulary ending a name, or standing + before the credential a name ends with, is the generation it also + spells. Both questions this rule asks of a name — how many words it has, and whether it is written in one case — are asked of the name's OWN words: a maiden marker taken as one, and the words it takes @@ -560,6 +566,8 @@ P3. Rationale: connective words ("y", "of the") bind name words into "Josep i Rovira" → middle="i" "John Quincy Smith i" → suffix="i" "Carod i" → suffix="i" + "Josep Lluis Carod i III" → suffix="i III" + "Carod y de Rovira i" → middle="y" Both exceptions are about the written FORM, not the word: the three-word carve-out counts letters, so a symbol connective joins at any length, and it reaches every single-letter connective the diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 973a69cd..c9f219e4 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -648,10 +648,30 @@ def _process_initial(self, name_part: str, # Return '' (never empty_attribute_default, which may be None) # when a part has no initialable words. group_initials below # decides what that means: one such element among others is - # dropped; a group that yields nothing AND is wholly particles - # initials its words; and a group that yields nothing for any - # other reason -- a conjunction, or particles mixed with one -- - # is still dropped ("Vega, Santa de y" drops its middle). + # dropped (`Alex van Johnson`'s `van`); a group that yields + # nothing AND is wholly particles initials its words; and a + # group that yields nothing for any other reason is still + # dropped. + # + # That third case survives the parse path no longer, and #461 + # is why: a connective with nothing to join is readmitted, so + # a group of a PARSED name cannot reach it. Any word that is + # neither a particle nor a connective initials, so a group + # reaching this is all particles and connectives; not being + # wholly particles it holds a connective; and that + # connective's part holds nothing but particles and + # connectives for it to join, so it is readmitted and the + # group yields it. Measured 2026-09-20, zero such groups over + # 95,119 names -- every corpus and case text plus the review's + # generated grid -- where "Vega, Santa de y" was the example + # until #461 and now initials 'S. y. V.'. + # + # What still reaches it is the paths with no parse to read, + # where a connective is answered from the vocabulary and + # carries no mark: HumanName(first="Santa", middle="de y", + # last="Vega").initials() gives 'S. V.', the pre-#461 answer, + # and so does a subclass overriding middle_list with the same + # words (tests/v2/test_facade.py pins both). return "" def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index fd9ae780..29f23e49 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -630,6 +630,43 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], or is_suffix_piece(piece, ptags, tokens)) +# rules.md#P3: "a word the rest of the parse reads as a name word +# rather than as a credential or an honorific, looked for past any +# run of connectives standing between" (#397) +def _name_word_beside(k: int, step: int, lo: int, hi: int, + pieces: Sequence[Piece], ptags: Sequence[Set[str]], + tokens: Sequence[WorkToken]) -> bool: + """Whether such a word stands on the `step` side of the + connective piece at `k`. + + `lo` and `hi` bound the name's own words: assign peels the pieces + below `lo` as its leading titles and those from `hi` up as its + trailing suffix run, so a piece outside that span is a credential + or an honorific however it is spelled, and the numeral or bare + acronym the peel takes ('i V', 'i MA') is inside `hi` by + construction rather than by a second reading of the vocabulary + (mechanisms.md#ONE-PREDICATE-PER-QUESTION). + + Inside the span the suffix and title tests still run, because + neither bound reaches a credential or an honorific standing in + the MIDDLE of a name ('Josep Jr. i Rovira', 'Josep Dr. i + Rovira'): the peel walks from the end and stops at the first name + word, the title run from the front. + + The walk steps over connectives because a RUN of them joins as + one ('Carod i y Rovira'), so the word this rule is about is the + first one past the run -- and where the run runs out ('Juan i e') + there is no name word on that side at all. + """ + j = k + step + while (0 <= j < len(pieces) + and _is_conj_piece(pieces[j], ptags[j], tokens)): + j += step + return (lo <= j < hi + and not is_suffix_piece(pieces[j], ptags[j], tokens) + and not is_title_piece(pieces[j], ptags[j], tokens)) + + def _group_segment(seg: tuple[int, ...], additional: int, tokens: Sequence[WorkToken], bound_join: BoundJoin = BoundJoin.STRICT, @@ -806,12 +843,70 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), del ptags[k] if len(pieces) + additional >= 3: + # rules.md#P3: "A connective that is also generational + # vocabulary joins only where a name word stands on each side + # of it" (#397, restated by its review). `frozen` holds the + # TOKEN index of every such connective that has no name word + # on one side or the other. It is joining nothing, so it is + # the generation it also spells: it may not merge into a run, + # it may not join, and it counts toward the carve-out total + # the way the generation counted -- which is not at all, a + # suffix piece being no rootname. + # + # Asked HERE, of the pieces as classify left them, and of the + # NEIGHBOURS' class rather than of the connective's position. + # Position was the first cut and it tested the wrong thing: + # any piece on each side passed it, so a generational suffix + # standing behind the link was swallowed into the name + # ("Josep Lluis Carod i III" read family 'Carod i III'). And + # the question cannot be re-asked further down, because a + # merge answers it: in the part a join produced, the absorbed + # suffix IS the word standing on the right. + # + # A token index rather than a piece index for the same reason + # the chain's trailing run is a length from the end: the + # merges below move piece indices and cannot move this one. + # + # Nothing but a one-letter connective of the suffix vocabulary + # reaches the body, so a name that has none pays tag lookups + # and no call at all. + frozen: set[int] = set() + lo = hi = -1 + for k, piece in enumerate(pieces): + tok = tokens[piece[0]] + if (len(piece) != 1 or len(tok.text) != 1 + or "conjunction" not in tok.tags + or "vocab:suffix" not in tok.tags): + continue + if hi < 0: + lo = leading_titles(pieces, ptags, tokens) + hi = trailing_start(lo, pieces, ptags, tokens, + one_case=one_case) + if not (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens) + and _name_word_beside(k, 1, lo, hi, pieces, ptags, + tokens)): + frozen.add(piece[0]) total = sum(_is_rootname(p, t, tokens) - for p, t in zip(pieces, ptags)) + additional + for p, t in zip(pieces, ptags) + if p[0] not in frozen) + additional # contiguous conjunction runs merge first (v1: "of the") + # + # `pieces[k][0] in frozen` and not `frozen.isdisjoint(...)`: + # the piece this loop extends GROWS with every merge, so a + # test over its tokens costs 1+2+...+n and the stage goes + # quadratic in the length of a connective run -- measured, + # 'and ' x3200 took 41.8ms against 21.7ms, 6.2x per 4x input + # where the shape reads 4.1x, and tests/v2/test_benchmark.py's + # "and " shape is the guard that caught it. Reading the first + # token alone is exact rather than an approximation: a frozen + # piece is one token, nothing merges it (this branch declines, + # and the join below skips it), so a piece holding a frozen + # token IS that token. k = 0 while k < len(pieces) - 1: - if conj(k) and conj(k + 1): + if (conj(k) and conj(k + 1) + and pieces[k][0] not in frozen + and pieces[k + 1][0] not in frozen): merge(k, k + 2, add={"conjunction"}) else: k += 1 @@ -819,30 +914,23 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # single-letter connective in a three-word name, which stays a # name word" (v1's Google Code issue 11 carve-out, the # "john e smith" bug). The threshold reads the ROOTNAME count, - # and since #397 a connective counts ITSELF toward that count, - # so a connective that is also suffix vocabulary no longer - # raises the bar for its own join. + # and since #397 a connective counts ITSELF toward that count + # WHERE IT IS JOINING, so a connective that is also suffix + # vocabulary no longer raises the bar for its own join and no + # longer lowers it for an unrelated one ("Carod y Rovira i" + # counted the trailing generation and let the `y` join). k = 0 while k < len(pieces): - if not conj(k): + # first token again, and here it is exact for the second + # reason as well: the piece a join produces is left BEHIND + # `k`, so no merged piece is ever tested twice. + if not conj(k) or pieces[k][0] in frozen: k += 1 continue text = " ".join(tokens[i].text for i in pieces[k]) - if len(text) == 1 and text.isalpha(): - if total < 4: - k += 1 - continue - # rules.md#P3: "A connective that is also generational - # vocabulary joins only where a name word stands on - # each side of it" (#397). About the CLASS, not the - # letter, and narrow by construction: it is reached - # only for a single-letter connective piece, and it - # cannot see a trailing `y` or `and`, which is what - # leaves those readings alone. - if (not (0 < k < len(pieces) - 1) - and is_suffix_piece(pieces[k], ptags[k], tokens)): - k += 1 - continue + if len(text) == 1 and text.isalpha() and total < 4: + k += 1 + continue start = max(0, k - 1) end = min(len(pieces), k + 2) neighbor = start if start < k else end - 1 diff --git a/nameparser/_types.py b/nameparser/_types.py index ad035588..da868c60 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -738,7 +738,16 @@ def _remarked(tokens: list[Token]) -> tuple[Token, ...]: for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): part = [i for i, t in enumerate(out) if t.role is role] alone = bool(part) and all("particle" in out[i].tags for i in part) - lone_conj = (not alone) and not any( + # `bool(part)` on both, and DEFENSIVE and measured inert on + # this one (2026-09-20, #397 review question 4): an EMPTY part + # satisfies `all` and `not any` alike, so without the guard + # this reads True there and says a mark is due on a part with + # no tokens to put it on. Nothing can see the difference -- + # `lone_conj` is read only inside the `for i in part` below, + # which has no members. Control: drop it and the whole suite + # is green, which is why it is stated as a reader's guard + # rather than pinned by a test. + lone_conj = bool(part) and not alone and not any( "conjunction" not in out[i].tags and "particle" not in out[i].tags for i in part) for i in part: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b131f5da..276c5918 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -2562,15 +2562,95 @@ def _check_cjk_shape_purity(self) -> None: "John Quincy Smith i", {"given": "John", "middle": "Quincy", "family": "Smith", "suffix": "i"}, - notes="the BOUNDARY the both-sides condition exists for, and " - "the one Catalan row that reaches its is_suffix_piece " - "call at all -- every other Catalan row short-circuits " - "on the position test (measured). Four name words, so " - "the count no longer declines; drop the condition and " - "this reads family 'Smith i', losing the generation " - "(measured). A connective with nothing to its right is " - "connecting nothing. Roles unchanged at every release, " - "1.4.0 included", + notes="the BOUNDARY the both-sides condition exists for, in " + "its simplest shape: there is no piece at all to the " + "right of the link. Four name words, so the count no " + "longer declines; drop the condition and this reads " + "family 'Smith i', losing the generation (measured). A " + "connective with nothing to its right is connecting " + "nothing. Roles unchanged at every release, 1.4.0 " + "included", + shape=1), + Case("a_link_before_a_generation_is_the_generation_too", + "Josep Lluis Carod i III", + {"given": "Josep", "middle": "Lluis", "family": "Carod", + "suffix": "i III"}, + classification="fix(#436/#437)", + notes="the other half of the same boundary, and the one the " + "first cut got wrong: there IS a piece to the right, " + "so a condition asking about POSITION passed it and " + "the family read 'Carod i III' -- a generational " + "suffix silently swallowed into the surname, on 130 " + "names of the review's grid and pinned by nothing. The " + "question is the neighbour's CLASS: what the rest of " + "the parse reads that word as. The four ROLES are " + "1.4.0's and every release's; what the classification " + "records is the suffix STRING, which 1.4.0 wrote " + "'i, III' because it joined suffix entries with a " + "comma and R1 derives them from the writer's commas " + "now", + shape=1), + Case("the_neighbour_test_reads_the_peel_and_not_the_vocabulary", + "Josep Lluis Carod i V", + {"given": "Josep", "middle": "Lluis", "family": "Carod", + "suffix": "i V"}, + classification="fix(suffix-routing)", + ambiguities=("suffix-or-name",), + notes="why the condition asks assign's TRAILING-RUN walk " + "rather than the suffix-piece test: a bare 'V' is " + "suffix vocabulary written like an initial, so " + "is_suffix_piece refuses it (rules.md#S2's initial " + "veto) and a class test spelled with that predicate " + "alone would let the link swallow it. The peel takes " + "it, so trailing_start puts it outside the name's own " + "words and the link stays the generation -- carrying " + "the fork report the swallowed reading had lost. 'MA' " + "is the acronym half of the same shape. Roles " + "unchanged from 2.0.0 through 2.3.0; 1.4.0 read middle " + "'Lluis Carod', last 'i', suffix 'V', which is the " + "two-token divergence 'Carod i' pins above and the " + "same fix(suffix-routing) rule describes", + shape=1), + Case("the_link_joins_nothing_across_a_family_comma_either", + "Rovira, Josep Carod i Jr.", + {"given": "Josep", "middle": "Carod", "family": "Rovira", + "suffix": "i Jr."}, + classification="fix(#436/#437)", + notes="the comma form of the row above, and it is a " + "different code path rather than a second member of " + "one: the link and the generation sit in segment 1, " + "where the name words are counted without the family " + "the comma already fixed. The first cut read middle " + "'Carod i Jr.'. Roles are 1.4.0's here too, and the " + "classification is the same suffix-string comma: " + "1.4.0 wrote 'i, Jr.'", + shape=2), + Case("a_trailing_link_does_not_count_itself_for_another_join", + "Carod y de Rovira i", + {"given": "Carod", "middle": "y", "family": "de Rovira", + "suffix": "i"}, + notes="the COUNT half of the same rule, and the reason it " + "cannot be stated for the join alone. The trailing 'i' " + "joins nothing, so it is the generation -- and a " + "generation is no name word, so it must not raise the " + "three-word carve-out's total either. Counting it did: " + "the total reached four, and an unrelated 'y' two " + "pieces away joined on the strength of it, reading " + "given 'Carod y de' with family 'Rovira'. Roles " + "unchanged at every release", + shape=1), + Case("a_particle_on_the_right_is_a_name_side_neighbour", + "Josep Carod i de Rovira", + {"given": "Josep", "middle": "Carod i de", "family": "Rovira"}, + classification="fix(#397)", + notes="the boundary on the other side of the class test: a " + "PARTICLE beside the link is a name word, not a " + "credential, so the link joins and the particle chain " + "then takes the surname. 1.4.0 through 2.3.0 read " + "middle 'Carod i' with family 'de Rovira' -- the link " + "unjoined, which is the whole defect #397 is about. " + "Contrast 'Josep Lluis Jr. i Rovira', where the " + "neighbour is a credential and the reading stands", shape=1), Case("a_link_ending_a_shorter_name_is_the_generation_too", "Josep Carod i", @@ -2605,13 +2685,16 @@ def _check_cjk_shape_purity(self) -> None: "Josep Carod i Rovira III", {"given": "Josep", "family": "Carod i Rovira", "suffix": "III"}, classification="fix(#397)", - notes="the suffix run is peeled before grouping, so the join " - "sees the same four pieces it sees without it and the " - "generation keeps its own slot. The contrast with " - "'John Quincy Smith i' is the whole design: a " - "generation word BEHIND the link is a suffix, a " - "generation word that IS the link with nothing behind " - "it stays a suffix too. 1.4.0 read middle 'Carod i'", + notes="the CONTROL for the three rows above: a generation " + "standing behind a name word leaves a name word on " + "each side of the link, so the link joins and the " + "generation keeps its own slot. Move it one word left " + "-- 'Josep Lluis Carod i III' -- and the link has a " + "credential on its right instead, joins nothing, and " + "is the generation it also spells. That pair is the " + "whole design, and the pair is why the condition asks " + "the NEIGHBOUR's class rather than the link's own. " + "1.4.0 read middle 'Carod i'", shape=1), Case("a_bare_capital_link_in_a_mixed_case_name_is_an_initial", "Josep Carod I Rovira", diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index 33394066..39c350a2 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -1433,11 +1433,126 @@ def test_the_count_arm_is_what_moves_it_not_the_vocabulary() -> None: def test_a_connective_with_nothing_to_its_right_does_not_join() -> None: # the both-sides condition (#397). Four name words, so the count # no longer declines -- what keeps the generation here is the - # position test, and dropping it reads 'Smith i' as one piece. + # condition, and dropping it reads 'Smith i' as one piece. The + # simplest shape of it: there is no piece to the right at all. out = _grouped("John Quincy Smith i", lexicon=_LINK_LEX) assert _piece_texts(out) == [["John", "Quincy", "Smith", "i"]] +# The condition asks about the NEIGHBOUR's class, and every arm of +# that question has a LINK/PLAIN pair below: with the letter outside +# the generational vocabulary nothing is asked and the link joins, so +# each pair says which arm is doing the work rather than that some +# arm is (#397 review -- the first cut asked about the link's +# POSITION, which every one of these shapes satisfies). +def test_a_suffix_word_on_the_right_is_no_name_word() -> None: + out = _grouped("Josep Lluis Carod i Jr.", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Josep", "Lluis", "Carod", "i", "Jr."]] + plain = _grouped("Josep Lluis Carod i Jr.", lexicon=_PLAIN_LEX) + assert _piece_texts(plain) == [["Josep", "Lluis", "Carod i Jr."]] + + +def test_the_right_hand_test_reads_the_peel_not_the_suffix_piece( +) -> None: + # the arm that forces trailing_start rather than is_suffix_piece: + # a ONE-CHARACTER suffix word is initial-shaped, so the + # suffix-piece test refuses it (rules.md#S2's initial veto) while + # assign's trailing peel takes it. Spelled with the piece test + # alone, the link would swallow the generation here. + lex = _LINK_LEX.add(suffix_words={"v"}) + out = _grouped("Josep Lluis Carod i V", lexicon=lex) + assert _piece_texts(out) == [["Josep", "Lluis", "Carod", "i", "V"]] + plain = _grouped("Josep Lluis Carod i V", + lexicon=_PLAIN_LEX.add(suffix_words={"v"})) + assert _piece_texts(plain) == [["Josep", "Lluis", "Carod i V"]] + + +def test_a_title_on_the_right_is_no_name_word() -> None: + out = _grouped("Josep Lluis Carod i Mr.", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Josep", "Lluis", "Carod", "i", "Mr."]] + plain = _grouped("Josep Lluis Carod i Mr.", lexicon=_PLAIN_LEX) + assert _piece_texts(plain) == [["Josep", "Lluis", "Carod i Mr."]] + + +def test_a_leading_title_on_the_left_is_no_name_word() -> None: + # the link opens the NAME even though a piece stands before it: + # assign peels the title run off the front. + out = _grouped("Mr. i Rovira Puig Vila", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Mr.", "i", "Rovira", "Puig", "Vila"]] + plain = _grouped("Mr. i Rovira Puig Vila", lexicon=_PLAIN_LEX) + assert _piece_texts(plain) == [["Mr. i Rovira", "Puig", "Vila"]] + + +def test_an_unlisted_leading_abbreviation_is_no_name_word_either( +) -> None: + # and THIS is the `lo` bound's own row, the listed spelling above + # being caught by the title-piece test as well. rules.md#H2 reads + # an unlisted abbreviation opening a name as a title by SHAPE, so + # the vocabulary says nothing about 'Xyz.' and only the bound + # assign's own title run draws keeps the link from joining it. + # Measured 2026-09-20: replace `lo` with 0 and this is the one + # test in the suite that dies. + out = _grouped("Xyz. i Rovira Puig Vila", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Xyz.", "i", "Rovira", "Puig", "Vila"]] + plain = _grouped("Xyz. i Rovira Puig Vila", lexicon=_PLAIN_LEX) + assert _piece_texts(plain) == [["Xyz. i Rovira", "Puig", "Vila"]] + + +def test_a_credential_or_honorific_mid_name_is_no_name_word_either( +) -> None: + # what the two bounds do NOT reach, and why the piece tests stay + # inside them: neither the title run nor the trailing peel walks + # into the middle of a name. + suffix = _grouped("Josep Lluis Jr. i Rovira", lexicon=_LINK_LEX) + assert _piece_texts(suffix) == [ + ["Josep", "Lluis", "Jr.", "i", "Rovira"]] + title = _grouped("Josep Lluis Mr. i Rovira", lexicon=_LINK_LEX) + assert _piece_texts(title) == [ + ["Josep", "Lluis", "Mr.", "i", "Rovira"]] + assert _piece_texts(_grouped("Josep Lluis Jr. i Rovira", + lexicon=_PLAIN_LEX)) == [ + ["Josep", "Lluis", "Jr. i Rovira"]] + assert _piece_texts(_grouped("Josep Lluis Mr. i Rovira", + lexicon=_PLAIN_LEX)) == [ + ["Josep", "Lluis", "Mr. i Rovira"]] + + +def test_the_walk_looks_past_a_run_of_connectives() -> None: + # a RUN joins as one, so the word the condition is about is the + # first one past the run, not the connective beside the link. + out = _grouped("Carod i y Rovira", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Carod i y Rovira"]] + + +def test_where_the_run_runs_out_there_is_no_name_word() -> None: + # the same walk reaching the end of the pieces: a link behind + # nothing but connectives is joining nothing, and the run may not + # absorb it either. + out = _grouped("Juan i y", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Juan", "i", "y"]] + assert _piece_texts(_grouped("Juan i y", lexicon=_PLAIN_LEX)) == [ + ["Juan i y"]] + doubled = _grouped("Josep Lluis Carod i i", lexicon=_LINK_LEX) + assert _piece_texts(doubled) == [ + ["Josep", "Lluis", "Carod", "i", "i"]] + assert _piece_texts(_grouped("Josep Lluis Carod i i", + lexicon=_PLAIN_LEX)) == [ + ["Josep", "Lluis", "Carod i i"]] + + +def test_a_link_that_joins_nothing_does_not_count_for_another_join( +) -> None: + # the COUNT half agreeing with the join (#397 review). The + # trailing link joins nothing, so it is the generation -- and a + # generation is no rootname, so the total stays at three and the + # unrelated 'y' two pieces away keeps the carve-out. Counting it + # gave the total four and joined the 'y'. + out = _grouped("Carod y Rovira i", lexicon=_LINK_LEX) + assert _piece_texts(out) == [["Carod", "y", "Rovira", "i"]] + plain = _grouped("Carod y Rovira i", lexicon=_PLAIN_LEX) + assert _piece_texts(plain) == [["Carod y Rovira i"]] + + def test_a_connective_with_nothing_to_its_left_does_not_join() -> None: # the other side of the same condition, and it needs four pieces # to get past the count: a link OPENING the name has no name word @@ -1448,12 +1563,12 @@ def test_a_connective_with_nothing_to_its_left_does_not_join() -> None: def test_the_both_sides_condition_reads_the_class_not_the_letter( ) -> None: - # the recorded negative control for the is_suffix_piece conjunct, - # and the one the property invariants CANNOT give: with the same - # letter outside the generational vocabulary the condition - # declines to ask and the trailing connective joins, exactly as a - # trailing 'y' does today. Measured -- remove that conjunct and - # this test is the one that dies. + # the recorded negative control for the class test the freeze + # walk opens with, and the one the property invariants CANNOT + # give: with the same letter outside the generational vocabulary + # the condition declines to ask and the trailing connective + # joins, exactly as a trailing 'y' does today. Measured -- remove + # the "vocab:suffix" arm and this test is the one that dies. out = _grouped("John Quincy Smith i", lexicon=_PLAIN_LEX) assert _piece_texts(out) == [["John", "Quincy", "Smith i"]] diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index dd6e15b1..8b027f46 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -916,6 +916,30 @@ def last_list(self) -> list[str]: assert SubLastOnly("john e smith").initials() == "j. e. Z." +def test_a_group_of_particles_and_a_connective_drops_only_unparsed( +) -> None: + # `_process_initial`'s closing comment, pinned (#397 review). The + # "group yields nothing for a reason other than being wholly + # particles" branch is the one #461 emptied on the PARSE path: the + # middle here is a working particle plus a connective with nothing + # to join, so the connective is readmitted and initials. Measured + # 2026-09-20 over every corpus and case text plus the review's + # generated grid, 95,119 names: zero parsed groups reach the drop. + assert HumanName("Vega, Santa de y").initials() == "S. y. V." + # and the paths with no parse to read, where a connective is + # answered from the vocabulary and carries no mark, which keep the + # pre-#461 answer and so keep the branch alive + built = HumanName(first="Santa", middle="de y", last="Vega") + assert built.initials() == "S. V." + + class Sub(HumanName): + @property + def middle_list(self) -> list[str]: + return ["de y"] + + assert Sub("Vega, Santa de y").initials() == "S. V." + + def test_initials_of_a_spliced_field_ask_the_vocabulary() -> None: # A field assigned after the parse is raw text: ParsedName.replace() # stamps UNCLASSIFIED_TAG on it, which says the words were read by diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 36b67f50..7e6c80ec 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -3134,8 +3134,12 @@ def _claim(rule: dict) -> _Claim: # carrying a comma, 'Carod i Rovira, Josep'. Reach, not # explanation, and verified name by name: the claimed set # gained that name and lost none. + # 2026-09-20, #397 review: 359 -> 360, the one new comma name + # the review's rows add, 'Rovira, Josep Carod i Jr.' -- the + # comma form of the swallowed generation. Reach again, and + # verified name by name. "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(359, ('given', 'suffix', 'title'), '37cb26979687', None), + _Claim(360, ('given', 'suffix', 'title'), 'f0e6eed75c6a', None), "fix(comma-family) a comma followed only by titles keeps the given/family split": _Claim(2, ('family', 'given'), "5bd9c6d96c38", None), "fix(comma-family) a comma followed only by titles keeps the given/family split, the C1 example": @@ -3187,8 +3191,10 @@ def _claim(rule: dict) -> _Claim: # names as the rule above and for the same reason. # 2026-09-20, #397: 358 -> 359, the same one new comma name as # the rule above and for the same reason. + # 2026-09-20, #397 review: 359 -> 360, the same one new comma + # name as the rule above and for the same reason. "fix(comma-precomma-family) pre-comma run reads as family, not given": - _Claim(359, ('family', 'given'), '37cb26979687', None), + _Claim(360, ('family', 'given'), 'f0e6eed75c6a', None), "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": _Claim(1, ('family', 'suffix'), "498602f3cfd0", None), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -3416,8 +3422,13 @@ def _claim(rule: dict) -> _Claim: # in the corpus because its row carries a shape tag. Reach # again: the regex is unchanged and the claimed set gained # that name alone. + # 2026-09-20, #397 review: 102 -> 103. One new corpus name + # holding a connective run, 'Carod y de Rovira i' -- the row + # pinning that a trailing link does not count itself for an + # unrelated 'y', which is in the corpus because its row + # carries a shape tag. Reach again, verified name by name. "fix(initials-per-word) a connective run initials each word (facade, since 2.0.0)": - _Claim(102, ('_initials',), "2c63e2c40d47", ('DEFAULT',)), + _Claim(103, ('_initials',), "05db047609d0", ('DEFAULT',)), # 2026-09-19, #533: 41 -> 43. Two new corpus names opening # with a bound-given word, 'Berg, abdul MA' and 'Berg, abdul # nee Jones MA' -- the P5 pair this change added to record @@ -3427,8 +3438,12 @@ def _claim(rule: dict) -> _Claim: # 2026-09-18: 109 -> 110. One new corpus name, # 'john van der berg ma' -- rules.md#P2's one-case contrast, # and a particle chain like every other member. + # 2026-09-20, #397 review: 111 -> 112. The same one new name + # as the connective-run rule above, 'Carod y de Rovira i', + # whose 'de Rovira' is a particle chain; two rules reach one + # name and neither widened. Verified name by name. "fix(initials-per-word) a particle chain inside a name part initials each word (facade, since 2.0.0)": - _Claim(111, ('_initials',), '3729c1e3152d', ('DEFAULT',)), + _Claim(112, ('_initials',), 'b3b3b696a56e', ('DEFAULT',)), "fix(initials-per-word) the Ph. D. merge initials each word (facade, since 2.0.0)": _Claim(18, ('_initials',), "f67d8ebddd56", ('DEFAULT',)), # The 2.3 title-run bundle's five rules, last in every diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index e9c8f15e..017e2b35 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -25,7 +25,7 @@ from nameparser._pipeline._state import ParseState from nameparser._pipeline._vocab import effective_script from nameparser._types import (UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, - AmbiguityKind, Role, Token) + AmbiguityKind, ParsedName, Role, Token) from .conftest import differential_corpus @@ -1212,3 +1212,259 @@ def test_initials_emit_exactly_the_predicted_contributors() -> None: assert not failures, ( f"{len(failures)} group(s) emitted something other than the " f"criterion's contributors:\n" + "\n".join(failures[:10])) + + +# --- #397 review: the OFF-SWITCH grid, and its two invariants ------- +# A second grid, kept apart from the one above rather than folded +# into it, and the reason is the cost: both invariants below parse +# every entry TWICE -- once as configured and once with the class +# letter out of the connective vocabulary -- so the shapes that +# earned a place here are the ones the first cut of #397 got wrong, +# not every shape the file already covers. What this one adds over +# `_connective_grid`: the trailing numeral and bare acronym assign's +# peel takes ('V', 'i', beside the 'III'/'Jr.'/'I'/'MA' both carry), +# a maiden clause, a head that is nothing but an initial or a +# particle, and the SUFFIX comma as a third comma shape. + + +def _off_switch_grid() -> list[tuple[str, Parser, str]]: + heads: tuple[list[str], ...] = ( + [], ["Josep"], ["Josep", "Lluis"], ["Dr."], ["J."], ["van"], + ["Josep", "Lluis", "Marti"]) + mids: tuple[list[str], ...] = ([], ["Carod"], ["de", "Carod"]) + conns = ("i", "y", "e", "and") + tails: tuple[list[str], ...] = ([], ["Rovira"], ["de", "Rovira"]) + suffixes: tuple[list[str], ...] = ( + [], ["III"], ["Jr."], ["I"], ["V"], ["MA"], ["i"], ["nee", "Puig"]) + lexicons = (("default", Lexicon.default()), + ("conj+v", Lexicon.default().add(conjunctions={"v"}))) + policies = (("default", Policy()), + ("family-first", Policy(name_order=FAMILY_FIRST)), + ("given-last", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)), + ("strict-comma", Policy(lenient_comma_suffixes=False))) + texts: list[str] = [] + seen: set[str] = set() + for head, mid, conn, tail, suffix, comma in itertools.product( + heads, mids, conns, tails, suffixes, (0, 1, 2)): + if not (mid or tail): + continue + if comma == 1 and not tail: + continue + if comma == 2 and not suffix: + continue + front = head + mid + [conn] + if comma == 1: + base = " ".join(tail) + ", " + " ".join(front + suffix) + elif comma == 2: + base = " ".join(front + tail) + ", " + " ".join(suffix) + else: + base = " ".join(front + tail + suffix) + for written in (base, base.upper(), base.lower()): + if written not in seen: + seen.add(written) + texts.append(written) + parsers = [(f"{ln}/{pn}", Parser(lexicon=lex, policy=pol)) + for ln, lex in lexicons for pn, pol in policies] + return [(t, p, label) for t in texts for label, p in parsers] + + +_OFF_SWITCH_GRID = _off_switch_grid() +_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) +_OFF_PARSERS: dict[tuple[int, frozenset[str]], Parser] = {} + + +def _class_letters(lexicon: Lexicon) -> frozenset[str]: + """The class rules.md#P3's both-sides condition is about: a + one-letter connective that is ALSO generational vocabulary.""" + return frozenset(w for w in lexicon.conjunctions + if len(w) == 1 and w in lexicon.suffix_words) + + +def _off_switch(parser: Parser, letters: frozenset[str]) -> Parser: + """The same parser with those letters out of the connectives -- + the parent-equivalent reading, where nothing can have joined. + + Keyed by `id`, which is safe here and nowhere else: the grid above + holds every parser for the module's lifetime, so no id is reused. + A `Parser` is not hashable (its `Lexicon` carries a mappingproxy). + """ + key = (id(parser), letters) + if key not in _OFF_PARSERS: + _OFF_PARSERS[key] = Parser( + lexicon=parser.lexicon.remove(conjunctions=set(letters)), + policy=parser.policy) + return _OFF_PARSERS[key] + + +def _present(text: str, letters: frozenset[str]) -> frozenset[str]: + words = {w.strip(".,").lower() for w in text.split()} + return frozenset(letters & words) + + +_Placed = tuple[Token, tuple[int, int]] + + +def _placed(name: ParsedName) -> list[_Placed]: + """The parse's tokens with their spans, spliced ones dropped.""" + return [(tok, tok.span) for tok in name.tokens if tok.span is not None] + + +def _off_roles(off: ParsedName) -> dict[tuple[int, int], Role]: + return {span: tok.role for tok, span in _placed(off)} + + +def _name_word_beside(toks: list[_Placed], i: int, step: int, + off_role: dict[tuple[int, int], Role], + original: str) -> bool: + """Whether a name word stands on the `step` side of toks[i], + judged by the OFF-SWITCH parse's roles -- which is what makes + this a PRE-JOIN reading: in that parse the class letter is no + connective, so no join has moved anything. A comma between ends + the walk (it is another segment), and connectives are stepped + over because a run of them joins as one.""" + j = i + while True: + k = j + step + if not 0 <= k < len(toks): + return False + left, right = (j, k) if step > 0 else (k, j) + if "," in original[toks[left][1][1]:toks[right][1][0]]: + return False + j = k + if "conjunction" in toks[j][0].tags: + continue + return off_role.get(toks[j][1]) in _NAME_ROLES + + +def _initial_spans(name: ParsedName, + letters: frozenset[str]) -> set[tuple[int, int]]: + return {span for tok, span in _placed(name) + if "initial" in tok.tags and tok.text.lower() in letters} + + +def _initial_reading_moved(on: ParsedName, off: ParsedName, + letters: frozenset[str]) -> bool: + """Whether the two parses disagree about a class letter being an + INITIAL -- rules.md#P3's marked-subset clause rather than its + both-sides one, and the one thing an off-switch comparison cannot + hold fixed: taking the word out of the connectives decides that + question too, in either direction. A marked letter in a one-case + name reads as an initial only while it IS connective vocabulary; + a bare capital the caller's own connectives claim stops reading + as one.""" + return _initial_spans(on, letters) != _initial_spans(off, letters) + + +def _link_joins_between_name_words(on: ParsedName, off: ParsedName, + letters: frozenset[str]) -> bool: + off_role = _off_roles(off) + toks = _placed(on) + for i, (tok, _span) in enumerate(toks): + if ("conjunction" not in tok.tags + or tok.text.lower() not in letters): + continue + if (_name_word_beside(toks, i, -1, off_role, on.original) + and _name_word_beside(toks, i, 1, off_role, on.original)): + return True + return False + + +def test_the_off_switch_grid_can_fail() -> None: + """The reachability probe, the shape every grid in this file + carries. Dated recorded control, measured 2026-09-20.""" + assert len(_OFF_SWITCH_GRID) == 103040, len(_OFF_SWITCH_GRID) + assert len({t for t, _, _ in _OFF_SWITCH_GRID}) == 12880 + reached = sum(1 for text, parser, _ in _OFF_SWITCH_GRID[:4000] + if _present(text, _class_letters(parser.lexicon))) + assert reached > 1000, reached + + +def test_a_link_that_joins_nothing_changes_no_field() -> None: + """INV6 (#397 review), and the strongest thing this rule can be + asked: turning the class letter OFF is the parent's reading, so + a letter that joins nothing must leave every field where the + parent left it. + + Two exemptions, both narrow and both P3's own OTHER clauses. A + letter that JOINED between name words is the rule working, judged + on the off-switch parse's classes so an absorbed suffix cannot + pass itself off as the name word on the right. And a letter the + two parses disagree about being an INITIAL is the marked-subset + clause, which the switch decides along with the join and so + cannot hold fixed. + + Mutation-checked, 2026-09-20: this fails on 2,360 parses at + c8550b64, the commit the review was written against. + """ + failures = [] + for text, parser, label in _OFF_SWITCH_GRID: + letters = _present(text, _class_letters(parser.lexicon)) + if not letters: + continue + on = parser.parse(text) + off = _off_switch(parser, letters).parse(text) + # the SEVEN FIELDS, and not comparison_key: a parse carries + # more than its fields, and what the off-switch legitimately + # moves besides them is the conjunction-or-initial report + if on.as_dict() == off.as_dict(): + continue + if _link_joins_between_name_words(on, off, letters): + continue + if _initial_reading_moved(on, off, letters): + continue + failures.append(f"[{label}] {text!r}: {on.as_dict()} != " + f"off-switch {off.as_dict()}") + assert not failures, ( + f"{len(failures)} parse(s) moved a field with no link joining " + f"anything:\n" + "\n".join(failures[:10])) + + +def test_a_trailing_credential_never_joins_into_a_name_part() -> None: + """INV1 strengthened (#397 review). INV1 above inspects the part + a join PRODUCED, where an absorbed credential is itself the name + word standing on the right, so it is satisfied by the very defect + it is about -- `Josep Lluis Carod i III` passes it. This asks the + off-switch parse instead: a word THAT reading puts in the suffix + never lands inside a joined name part. + + It carries INV6's second exemption and not its first: a letter + the two parses disagree about being an INITIAL moved for the + marked-subset clause's reasons, not this one. The JOIN exemption + is deliberately absent -- a link joining elsewhere in the name + never licenses a credential joining here. + + Mutation-checked, 2026-09-20: this fails on 960 parses at + c8550b64, where INV1 fails on none of them. + """ + failures = [] + for text, parser, label in _OFF_SWITCH_GRID: + letters = _present(text, _class_letters(parser.lexicon)) + if not letters: + continue + on = parser.parse(text) + off = _off_switch(parser, letters).parse(text) + if _initial_reading_moved(on, off, letters): + continue + off_role = _off_roles(off) + for role in _NAME_ROLES: + part = on.tokens_for(role) + if len(part) < 2: + continue + for tok in part: + # not the LINK itself: it is generational vocabulary + # by definition of the class, so the off-switch parse + # reads it as the suffix in every name it ends. This + # rule is about the OTHER word -- the credential a + # link must not take with it. + # `span is None` is a typing guard and nothing else: + # every token of a PARSER-produced name carries one, + # and this grid holds no spliced parse. + if "conjunction" in tok.tags or tok.span is None: + continue + if off_role.get(tok.span) is Role.SUFFIX: + failures.append( + f"[{label}] {text!r}: {tok.text!r} reads as the " + f"suffix and joined into {role.value}") + assert not failures, ( + f"{len(failures)} credential(s) joined into a name part:\n" + + "\n".join(failures[:10])) diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 90e4da5d..99facfb5 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -34,6 +34,7 @@ "Berg, abdul vd" "Carod i" "Carod i Rovira, Josep" +"Carod y de Rovira i" "Davis Royce, Ed" "Del Toro" "Doe, Dr. nee Smith MA" @@ -185,6 +186,7 @@ "Jose E Maria Santos" "Jose e Maria Santos" "Josep Carod i Rovira" +"Josep Lluis Carod i III" "Josep i Rovira" "Jr." "Juan & Garcia" diff --git a/tools/differential/corpus_shapes.jsonl b/tools/differential/corpus_shapes.jsonl index e2aebd36..09cdca72 100644 --- a/tools/differential/corpus_shapes.jsonl +++ b/tools/differential/corpus_shapes.jsonl @@ -2,6 +2,7 @@ {"name": "Bridge (1.4)", "shape": 1} {"name": "Bridge (A.B)", "shape": 1} {"name": "Carod i", "shape": 1} +{"name": "Carod y de Rovira i", "shape": 1} {"name": "Dr. Juan de la Vega III", "shape": 1} {"name": "Duke of Edinburgh", "shape": 1} {"name": "J.A. K.D.", "shape": 1} @@ -73,7 +74,10 @@ {"name": "Josep Carod i", "shape": 1} {"name": "Josep Carod i Rovira", "shape": 1} {"name": "Josep Carod i Rovira III", "shape": 1} +{"name": "Josep Carod i de Rovira", "shape": 1} +{"name": "Josep Lluis Carod i III", "shape": 1} {"name": "Josep Lluis Carod i Rovira", "shape": 1} +{"name": "Josep Lluis Carod i V", "shape": 1} {"name": "Josep i Rovira", "shape": 1} {"name": "Juan Garcia Lopez y", "shape": 1} {"name": "Juan Garcia Y Lopez", "shape": 1} @@ -166,6 +170,7 @@ {"name": "Kennedy, John (Jack)", "shape": 2} {"name": "NASCIMENTO, EDSON ARANTES DO", "shape": 2} {"name": "Nascimento, Edson Arantes do", "shape": 2} +{"name": "Rovira, Josep Carod i Jr.", "shape": 2} {"name": "Royce, Ed", "shape": 2} {"name": "SMITH, JOHN DO", "shape": 2} {"name": "Salem, Abdul Rahman Ahmed", "shape": 2} From f4c79f9c3f9e7c50c6f1b19640594ec57b65e59b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 07:00:35 -0700 Subject: [PATCH 04/11] test(#397/#461): record the ledger diffs at every baseline Five ledgers, and the gate exits 0 at all of them with `unexplained: 0` and no radar diff left unclassified beyond the seven the bundle never touched. The Catalan join gets a ROLE rule and, from 2.0.0, a report rule of its own: `_ambiguities` cannot diff below 2.0, so the three one-case spellings that gain a connective-or-initial are unclassifiable at 1.4.0 and need no rule there. Their ROLES do not move, and that is the marked subset working rather than an accident of these spellings -- the lower-case forms where the letter used to be the generation do move fields, and no corpus carries one. The five names the join moves are new to the CONTRACT tier as well as to these files: they entered the corpus as rules.md example lines and case rows, having been radar-tier names the contract did not answer for. The initials movers split by CAUSE rather than by baseline: a connective holding its part alone gains a letter, a connective with a name word beside it loses one, and the two get separate rules because they are separate readings of one sentence and a rule listing `_initials` may list nothing else. WHICH HALF OF A DIFF BELONGS TO THIS CHANGE was measured name by name against the parent as well as against each wheel, and it decided where several names sit. At 1.4.0 the joining half was already diffing on the 2.0.0 per-word grouping, so its nineteen names stay on that rule and the rule gains a dated paragraph naming both halves -- its claim that every name it explains agrees with 1.4.0 letter for letter stopped being true. The readmission half had no 1.4.0 diff at all: fifteen names agreed with the wheel before this bundle and diverge from it now, which is a parity break stated in its own rule rather than absorbed by an older one. Eleven of the fifteen are reached by the grouping rule's alternation too, so each is pinned in _RECORDED_DIFFS and _CROSS_RULE_WINNERS with the argument for the winner beside it. Three names are new to the corpus and move NOTHING: 'Josep Lluis Carod i III', 'Josep Lluis Carod i V' and 'Rovira, Josep Carod i Jr.' parse byte-identically at the parent and here. Their diffs are older divergences seeing the gate for the first time, and they go under the rules that caused them -- the space-separated post-nominal run at four baselines, and a trailing two-word generation run peeling whole at 1.4.0. TWO rules retire, and both are this bundle restoring 1.4.0 parity. The 1.4.0 entry recording that a recognized non-Latin connective contributes no initial explained one name, and that name now gives 1.4.0's own answer on both surfaces. The 2.3.0 entry recording that a bare capital connective in an all-upper name stops initialing explained one name too, and that name stops diffing. Both are deleted rather than declared dormant: their reasons are wrong now, not merely unreached. The second of the two also leaves the 1.4.0 facade rule's alternation, which narrows from four names to three. The 1.4.0 rule for the two-word link is REWRITTEN IN PLACE rather than deleted, which is the opposite of what its own comment predicted. It said to delete it when #397 landed; #397 landed, and the name it explains is unchanged -- a connective with nothing to its right is connecting nothing, which is the boundary the new rule states. Its standing moves from NOT WANTED and unresolved to the decided boundary of a landed rule, and its _CROSS_RULE_WINNERS pin and _MUST_NOT_MATCH probes follow the new title. And one name keeps its rule while FLIPPING SURFACE. At 2.0.0, 2.1.0 and 2.2.0 the wheels give facade 'J. G.' and core 'J. Y. G.' for 'JUAN Y GARCIA'. Until now only the core moved; now only the facade does, and the core agrees with the baseline word for word. One pseudo-field covers both, so the rule stands and its dated paragraph -- which said the opposite -- is amended in all three files. Co-Authored-By: Claude Opus 5 (1M context) --- tests/v2/test_ledger_guards.py | 447 +++++++++++++++++-- tools/differential/compare.py | 35 ++ tools/differential/expected_since_1.4.0.toml | 282 +++++++++--- tools/differential/expected_since_2.0.0.toml | 207 ++++++++- tools/differential/expected_since_2.1.0.toml | 207 ++++++++- tools/differential/expected_since_2.2.0.toml | 211 ++++++++- tools/differential/expected_since_2.3.0.toml | 221 ++++++++- 7 files changed, 1502 insertions(+), 108 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 7e6c80ec..d51a4173 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -880,12 +880,57 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # has no third token for the regex to require. "fix(N3) a nickname-led name with a trailing suffix keeps the suffix in `suffix`": ("'Smitty' Jones", "Jones Jr.", "'Smitty' Dr. Jones"), - # #451's remaining NOT-WANTED rule, literal-anchored to one corpus - # name: a three-token name with a rootname before or after is a - # different diff shape (or, for 'Carod i Rovira' and 'Lluis Carod - # i', no diff at all -- #397's still-open enhancement, not a - # 1.4-to-2.x regression). - "fix(#397)": ("Carod i Rovira", "Josep Carod i Rovira", "Lluis Carod i"), + # #451's remaining literal-anchored rule, now stating a DECIDED + # boundary rather than an open enhancement: a three-token name + # with a rootname before or after is a different diff shape + # ('Carod i Rovira' and 'Josep Carod i Rovira' now join and have + # fix(#397) rules of their own), and 'Lluis Carod i' is the same + # both-sides boundary one word longer, diffing nothing. The probe + # is unchanged: none of the three may be claimed by a regex + # anchored to the two-word shape. + # + # KEYED on the retitled issue rather than on the bare 'fix(#397)' + # it carried until 2026-09-20. The keys are `issue` SUBSTRINGS and + # there are three fix(#397) rules now; the bare key would apply + # this list to the join rule below, which claims 'Josep Carod i + # Rovira' on purpose. + "fix(#397) accepted": + ("Carod i Rovira", "Josep Carod i Rovira", "Lluis Carod i"), + # The join rule's own wall, and every probe is a name the join + # must NOT reach for a different reason: 'Josep i Rovira' is three + # words and under rules.md#P3's carve-out; 'Carod i' and 'John + # Quincy Smith i' have no name word to the link's right, the + # accepted cost the rule above states; 'Josep Lluis Carod i III' + # and 'Josep Lluis Carod i V' end in a generation run and are + # claimed by fix(#436/#437) and the suffix-routing rule beside it. + # A rule keyed on the SHAPE -- a single letter between two name + # words -- reaches all five. + "fix(#397) the Catalan/Polish link joins": + ("Josep i Rovira", "Carod i", "John Quincy Smith i", + "Josep Lluis Carod i III", "Josep Lluis Carod i V", + "Lluis Carod i"), + # The one-case report rule: mixed-case spellings are where the + # writing decides the letter, and the join moved their ROLES + # instead, so a regex reaching one would absorb the join's own + # regression. 'JUAN GARCIA Y LOPEZ' is the unmarked letter in the + # same one-case position, which emits no report at all. + "fix(#397) the Catalan link reports": + ("Josep Carod i Rovira", "Carod i Rovira, Josep", + "Josep Lluis Carod i Rovira", "JUAN GARCIA Y LOPEZ"), + # #461's two halves, each probed with the other half's names and + # with the boundary the rule is keyed on. The criterion is the + # PART and never a word count: 'Jon Dough and' has base 'Dough + # and' and keeps 'J. D.', 'Juan Velasquez y Garcia' keeps 'J. V. + # G.', and 'Juan Garcia Lopez y' is the trailing spelling of the + # same -- none of the three moves, measured 2026-09-20 against the + # parent 46651750, so no #461 rule may claim any of them. + "fix(#461) a connective holding its part alone": + ("Jon Dough and", "Juan Velasquez y Garcia", + "Juan Garcia Lopez y", "John and Jane Smith", + "Duke of Edinburgh"), + "fix(#461) a connective with a name word beside it": + ("Jon Dough and", "Juan Velasquez y Garcia", + "Juan Garcia Lopez y", "Juan de y", "johnny y"), # #342's rule is a literal alternation of five names, so a # widening reaching only names the corpora lack would leave # _CORPUS_CLAIMS unmoved. These probes are the wall, and every one @@ -1185,13 +1230,13 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: "fix(#383/#479) a marked connective letter in a one-case name is reported": ("John e Smith", "John E Smith", "john e. smith", "john e jones, III", "johnny y", "der, y van"), - # The view-only rule's boundary: the mixed-case spelling where the - # capital Y is an initial and stays one, the all-lower spelling - # that already read this way, and the sibling name whose ROLES move - # (the first rule above claims that one). - "fix(#383/#479) a bare capital connective in an all-upper name stops initialing": - ("Juan Y Garcia", "juan y garcia", "JUAN GARCIA Y LOPEZ", - "juan q. xavier velasquez y garcia iii"), + # `fix(#383/#479) a bare capital connective in an all-upper name + # stops initialing` carried a probe tuple here until 2026-09-20. + # The rule is DELETED from expected_since_2.3.0.toml -- #461 gave + # 'JUAN Y GARCIA' its letter back and the name stopped diffing + # there -- and a key matching no rule empties its own pin + # silently, which the roster's own assertion refuses. The + # boundaries it pinned are now the #461 rules' below. # #528's rule is a literal alternation of four names, so # _CORPUS_CLAIMS cannot see a widening that reaches only names the # corpora lack -- these probes are the wall, and the key is the @@ -1214,16 +1259,13 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: ("John E Smith", "Juan Y. Garcia", "Juan y Garcia", "Scott E. Werner", "Jose E Maria Santos", "maria y lopez", "Ph. D., John", "john e jones, III", "juan y garcia"), - # The third feat(#269) rule, and the only one keyed on a derived - # view. Its boundary is the other two: the prefix chain and the - # Cyrillic pair are #269 recognitions as well, and both move ROLES, - # so an alternation that grew to reach them would be taking names - # off the rules that describe what actually happened to them. - "feat(#269) a recognized non-Latin connective contributes no initial": - ("محمد بن سلمان", - "ХОСЕ И МАРИЯ САНТОС", - "хосе и мария сантос", - "محمد و علي السيد"), + # The third feat(#269) rule, the only one keyed on a derived view, + # carried a probe tuple here until 2026-09-20. It is DELETED from + # expected_since_1.4.0.toml -- #461 gave 'محمد و علي' its 'و' back + # and the name gives 1.4.0's own answer again -- and a key matching + # no rule empties its own pin silently. The name still diffs at the + # four 2.x baselines, where the #461 readmission rule carries it + # and _MUST_NOT_MATCH pins that rule's boundary instead. # #289/#516's three rules are literal-anchored alternations, so # _CORPUS_CLAIMS cannot see a widening that reaches only names the # corpora lack -- these probes are the wall, and the keys are the @@ -2485,8 +2527,14 @@ class _LatinCopy(NamedTuple): # list them. Scoped to that reason: facade initials that move # against 1.4.0 for OTHER reasons -- 'e and e' and 'juan garcia y # lopez', on the 2.0.0 per-word grouping -- are no part of the six. - frozenset({"john e smith", "john e jones", "jones, john e", - "JUAN Y GARCIA"}), + # + # 2026-09-20, #461: 'JUAN Y GARCIA' LEFT the alternation and the + # set is three. Its 'Y' holds the middle part alone, so it + # initials again on both surfaces and gives the 1.4.0 wheel's + # string; the name has no diff left at that baseline for this + # rule to explain. Nothing about the other three moved, and the + # set is still a list of names for the reason above. + frozenset({"john e smith", "john e jones", "jones, john e"}), # #533's rules, one corpus name per alternative -- lists of names, # not copies of any wordlist. What selects every one of them is a # SLOT the vocabulary participates in only at one end: a member of @@ -2559,6 +2607,114 @@ class _LatinCopy(NamedTuple): "Jane Doe nee Smith MA PhD", "Jane Doe nee Smith Ma JD"}), frozenset({"JOHN NEE JONES SMITH MA PHD", "John n[ée]e Jones Smith Ma"}), + # 2026-09-20, #397: the #436/#437 rule's set grows at both the + # 1.4.0 ledger and the three 2.x ones, and the two stay separate + # for the reason the block above gives -- one comma-written + # generation run enters at 1.4.0 ('Josep Lluis Carod i III') and + # three at the 2.x baselines, 'Josep Lluis Carod i V' and 'Rovira, + # Josep Carod i Jr.' moving ROLES at 1.4.0 and so belonging to + # other rules there. Still a list of names and still about the + # SEPARATOR: the new members carry the roman numerals i/III/V and + # a jr, so a member copying SUFFIX_WORDS would reach every + # generation-bearing name in the corpora, most of which do not + # move. + frozenset({"JOHN DOE PHD MD", "John Doe MD PhD", + "John Smith MD PhD", "John Smith Mc V", + "Josep Lluis Carod i III", + "Kenneth Clarke QC MP", "Smith, John PhD I\\.", + "The Rt Hon Kenneth Clarke QC MP, HMG", + "Washington Jr\\. MD, Franklin", "abdul Smith Jr Ma", + "abdul Smith Jr V"}), + frozenset({"JOHN DOE PHD MD", "Jane Doe nee Smith PhD MA", + "John Doe MD PhD", + "John Smith MD PhD", "John Smith Mc V", + "Josep Lluis Carod i III", "Josep Lluis Carod i V", + "Kenneth Clarke QC MP", "Rovira, Josep Carod i Jr\\.", + "Smith, John PhD I\\.", + "The Rt Hon Kenneth Clarke QC MP, HMG", + "Washington Jr\\. MD, Franklin", "abdul Smith Jr Ma", + "abdul Smith Jr V"}), + # #397's join and its one-case report, one corpus name per + # alternative -- lists of names, not copies of any wordlist. The + # join's subject is a SHAPE the vocabulary participates in at one + # end only: a connective standing between two name words, where + # the connective is also generational vocabulary. A member copying + # CONJUNCTIONS or CONJUNCTIONS_AMBIGUOUS would reach every + # connective-bearing name in the corpora, and most of them -- + # 'Josep i Rovira' under the three-word carve-out, 'Carod i' and + # 'John Quincy Smith i' with nothing to the link's right -- do not + # move, which is the whole of #397's accepted cost. The report + # rule's three are the one-case spellings of the same names, and + # what selects THEM is the writing rather than any word. + # _MUST_NOT_MATCH carries both walls. The join set is identical in + # all five ledgers; the report set in the four 2.x ones, there + # being no `_ambiguities` to diff at 1.4.0. + frozenset({"Carod i Rovira, Josep", "Josep Carod i Rovira", + "Josep Carod i Rovira III", "Josep Carod i de Rovira", + "Josep Lluis Carod i Rovira"}), + frozenset({"JOSEP CAROD I ROVIRA", "JOSEP LLUIS CAROD I ROVIRA", + "josep carod i rovira"}), + # #461's two halves, one corpus name per alternative -- lists of + # names, not copies of any wordlist. Four sets, because each half + # holds a different population at different baselines and each + # ledger's own comment says which: the readmission half gains + # 'محمد و علي' from 2.0.0 (1.4.0 parity is RESTORED there, so it + # has no 1.4.0 member) and 'Juan y Garcia née Jones' from 2.2.0 + # (below that the maiden clause moves its roles), and the joining + # half gains nine names at 2.3.0 that the #449 report masks at + # every earlier baseline. + # + # What selects every member is a PART -- whether the connective + # has a name word beside it to join -- which no wordlist can + # spell and no regex over the raw string can state. A member + # copying CONJUNCTIONS would reach both halves at once, they + # being the same vocabulary read in two positions, and would also + # reach 'Jon Dough and', 'Juan Velasquez y Garcia' and 'Juan + # Garcia Lopez y', which do not move at all. The non-Latin + # members are spelled literally rather than as script spans for + # the same reason: the sentence is about a part, in any script, + # so a span would declare a scope the rule does not have + # (_SPAN_BEARING_RULES records that neither rule declares one). + frozenset({"Carod y de Rovira i", "Fritz Freiherr und von Bar", + "Garcia y Lopez", "John e Smith", "John e Smith III", + "John e Smith, III", "John y Jane", "Jose e Maria", + "Juan de y", "Juan y Garcia", "Lt\\.Gov\\. juan e garcia", + "Smith, John e, III, Jr", "john e jones, III", + "johnny y", "juan y garcia"}), + frozenset({"Carod y de Rovira i", "Fritz Freiherr und von Bar", + "Garcia y Lopez", "John e Smith", "John e Smith III", + "John e Smith, III", "John y Jane", "Jose e Maria", + "Juan de y", "Juan y Garcia", "Lt\\.Gov\\. juan e garcia", + "Smith, John e, III, Jr", "john e jones, III", + "johnny y", "juan y garcia", "محمد و علي"}), + frozenset({"Carod y de Rovira i", "Fritz Freiherr und von Bar", + "Garcia y Lopez", "John e Smith", "John e Smith III", + "John e Smith, III", "John y Jane", "Jose e Maria", + "Juan de y", "Juan y Garcia", "Juan y Garcia née Jones", + "Lt\\.Gov\\. juan e garcia", + "Smith, John e, III, Jr", "john e jones, III", + "johnny y", "juan y garcia", "محمد و علي"}), + frozenset({"Assoc Dean of Chemistry Robert Johns", + "Dean of Chemistry Robert Johns", + "John and Jane Aznar y Lopez", "John and Jane Smith", + "Jose e Maria Santos", "Juan y Eva Garcia", + "Mr\\. and Mrs\\. John and Jane Smith", + "Rob And Beth Edmunds", "and Jon Dough", + "the and Jon Dough", "ХОСЕ И МАРИЯ САНТОС", + "Хосе И Мария Сантос", "хосе и мария сантос"}), + frozenset({"Assoc Dean of Chemistry Robert Johns", + "Dean of Chemistry", "Dean of Chemistry Robert Johns", + "Duke of Edinburgh", "Duke of Wellington", + "John & Jane", "John and Jane Aznar y Lopez", + "John and Jane Smith", "John of the Doe", + "Jose e Maria Santos", "Juan & Garcia", + "Juan and Garcia", "Juan y Eva Garcia", + "Mr\\. and Mrs\\. John and Jane Smith", + "Rob And Beth Edmunds", "and Jon Dough", + "part1 of The part2 of the part3 and part4", + "part1 of and The part2 of the part3 And part4", + "the and Jon Dough", "ХОСЕ И МАРИЯ САНТОС", + "Хосе И Мария Сантос", "хосе и мария сантос"}), }) def _unjustified_reach(name_regex: str, members: set[str]) -> list[str]: @@ -2978,8 +3134,13 @@ def _claim(rule: dict) -> _Claim: # Ten corpus names, `suffix` alone: the rule moves the # SEPARATOR and no role, so a widening that took a role would # change the row here before it reached the gate. + # 2026-09-20, #397: 10 -> 11, one new comma-written + # generation run ('Josep Lluis Carod i III'), which #397 + # leaves byte-identical -- measured against the parent + # 46651750 -- and which arrived with that change's own case + # rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(10, ('suffix',), "30f5314a2662", None), + _Claim(11, ('suffix',), "eaece748b3c9", None), # #346's alternation. Four corpus names, `family` and # `given` together: the fold moves both roles at once, so a # widening taking one alone would change the roles here @@ -3195,7 +3356,10 @@ def _claim(rule: dict) -> _Claim: # name as the rule above and for the same reason. "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(360, ('family', 'given'), 'f0e6eed75c6a', None), - "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost": + # 2026-09-20, #397: retitled in place, reach and digest + # unchanged -- the rule keeps 'Carod i', which the landing + # leaves byte-identical. + "fix(#397) accepted: a two-word Catalan link has no name word to its right and stays the generation": _Claim(1, ('family', 'suffix'), "498602f3cfd0", None), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": _Claim(0, ('suffix',), "e3b0c44298fc", None), @@ -3389,8 +3553,11 @@ def _claim(rule: dict) -> _Claim: # nothing had classified because no corpus name had exercised # it. Literal and caseless, so a second name here means the # alternation grew. - "feat(#269) a recognized non-Latin connective contributes no initial": - _Claim(1, ('_initials',), "770ce7374f32", ('DEFAULT',)), + # `feat(#269) a recognized non-Latin connective contributes + # no initial` had a row here until 2026-09-20 (reach 1, + # 'محمد و علي'). #461 gave that name its 'و' back on both + # surfaces, which is 1.4.0's own answer, so the rule stopped + # explaining anything and is deleted from the ledger. # #528's literal name list, added 2026-09-13 and sitting ahead # of the three vocabulary rules below because it shares their # `_initials` field, and because the first of them -- the @@ -3398,8 +3565,13 @@ def _claim(rule: dict) -> _Claim: # Four corpus names, `_initials` alone: the roles move on none, # which is what leaves the derived view as the whole diff. A # fifth name here means the alternation grew. + # 2026-09-20, #461: 4 -> 3. 'JUAN Y GARCIA' left the + # alternation -- its 'Y' holds its part alone and initials + # again on both surfaces, giving the 1.4.0 wheel's own string + # -- so the rule has nothing left to say about it. The other + # three are unmoved. "fix(#528) the facade's initials follow the parse's connective tags": - _Claim(4, ('_initials',), "7cb6b2f5779e", ('DEFAULT',)), + _Claim(3, ('_initials',), "3b5e8a0e39bb", ('DEFAULT',)), # 96 -> 97 on 2026-09-08: 'Prince of Wales Jr' joined the # rules corpus with the 2.3 title-run bundle -- a parity # row, kept as the boundary the peel floor declines -- and @@ -3626,14 +3798,46 @@ def _claim(rule: dict) -> _Claim: _Claim(5, ('family', 'given', 'maiden', 'middle', 'suffix'), 'ae9d39617af1', None), "fix(#335/#533) a marker-led bracketed clause is the maiden name whatever its last word is": _Claim(3, ('maiden', 'nickname'), 'cc1045ecdc05', None), + # 2026-09-20, #397 and #461's three new rules at this + # baseline, all literal alternations of exactly the names each + # explains -- so a count that moves means the alternation + # grew, and _MUST_NOT_MATCH carries the walls a reach this + # small cannot be. + # + # The join reaches 5 and declares the union of its two diff + # shapes: {family, suffix} for the comma spelling, whose link + # was a GENERATION in v1's `suffix`, and {family, middle} for + # the four written straight. The suffix-routing rule reaches + # its 1 and declares the same three roles for a diff #397 does + # not cause at all -- a trailing two-word generation run that + # v2 has peeled whole since 2.0. #461's readmission rule + # reaches 15 on `_initials` alone: fifteen names whose + # connective holds its part alone, every one of which agreed + # with the 1.4.0 wheel before this bundle. It has no joining + # twin here, the joining half having diffed at this baseline + # since 2.0 on the per-word grouping rule. + "fix(#397) the Catalan/Polish link joins two surnames": + _Claim(5, ('family', 'middle', 'suffix'), "0c31a48b1867", + ('DEFAULT',)), + "fix(suffix-routing) a two-word trailing generation run peels whole, and the name word before it is the family": + _Claim(1, ('family', 'middle', 'suffix'), "5ecc6ba1109e", + ('DEFAULT',)), + "fix(#461) a connective holding its part alone contributes an initial": + _Claim(15, ('_initials',), "4d436d1ebeca", ('DEFAULT',)), }, "expected_since_2.0.0.toml": { # #436/#437's Latin alternation, first in every ledger. # Ten corpus names, `suffix` alone: the rule moves the # SEPARATOR and no role, so a widening that took a role would # change the row here before it reached the gate. + # 2026-09-20, #397: 11 -> 14, three new comma-written + # generation runs ('Josep Lluis Carod i III', 'Josep Lluis + # Carod i V', 'Rovira, Josep Carod i Jr.'), each of which #397 + # leaves byte-identical -- measured against the parent + # 46651750 -- and each of which arrived with that change's own + # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(11, ('suffix',), "e665eae5c5df", None), + _Claim(14, ('suffix',), "4f5e63e709ae", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -4104,6 +4308,27 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('_ambiguities', 'family', 'maiden', 'middle'), 'fa3fe4878b47', None), "fix(#411/#533) the bound-given pair after a comma, and the clause it keeps its credential in": _Claim(1, ('_ambiguities', 'given', 'maiden', 'middle'), '431d3dd24c14', None), + # 2026-09-20, #397 and #461's four new rules, all literal + # alternations of exactly the names each explains -- a count + # that moves means the alternation grew, and _MUST_NOT_MATCH + # carries the walls a reach this small cannot be. + # + # The join reaches 5 and declares the union of its two diff + # shapes; the one-case report rule reaches 3 on + # `_ambiguities` alone, no role moving on any of them. #461's + # two halves are 16 and 13 on `_initials` alone, and they + # are two rules and not one because a rule listing `_initials` + # may list nothing else, so the two readings of one sentence + # cannot share a row either. + "fix(#397) the Catalan/Polish link joins two surnames": + _Claim(5, ('family', 'middle', 'suffix'), "0c31a48b1867", + ('DEFAULT',)), + "fix(#397) the Catalan link reports a connective-or-initial in a one-case name": + _Claim(3, ('_ambiguities',), "e43f9595ac02", ('DEFAULT',)), + "fix(#461) a connective holding its part alone contributes an initial": + _Claim(16, ('_initials',), "075dc34f9e95", ('DEFAULT',)), + "fix(#461) a connective with a name word beside it stops contributing an initial": + _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), }, # The 2.3 cycle's first rule, and a facade-only render fix: every # role is identical, so `_initials` alone. Reach and digest as in @@ -4114,8 +4339,14 @@ def _claim(rule: dict) -> _Claim: # Ten corpus names, `suffix` alone: the rule moves the # SEPARATOR and no role, so a widening that took a role would # change the row here before it reached the gate. + # 2026-09-20, #397: 11 -> 14, three new comma-written + # generation runs ('Josep Lluis Carod i III', 'Josep Lluis + # Carod i V', 'Rovira, Josep Carod i Jr.'), each of which #397 + # leaves byte-identical -- measured against the parent + # 46651750 -- and each of which arrived with that change's own + # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(11, ('suffix',), "e665eae5c5df", None), + _Claim(14, ('suffix',), "4f5e63e709ae", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -4357,14 +4588,46 @@ def _claim(rule: dict) -> _Claim: # through. "fix(#533) the maiden clause ends at the credential in a native-script name too": _Claim(1, ('_ambiguities', 'maiden', 'suffix'), "6bab87214ddf", None), + # 2026-09-20, #397 and #461's four new rules, all literal + # alternations of exactly the names each explains -- a count + # that moves means the alternation grew, and _MUST_NOT_MATCH + # carries the walls a reach this small cannot be. + # + # The join reaches 5 and declares the union of its two diff + # shapes; the one-case report rule reaches 3 on + # `_ambiguities` alone, no role moving on any of them. #461's + # two halves are 17 and 13 on `_initials` alone, and they + # are two rules and not one because a rule listing `_initials` + # may list nothing else, so the two readings of one sentence + # cannot share a row either. + # + # The readmission half reaches one more name here than at + # 2.0.0/2.1.0 -- 'Juan y Garcia née Jones', whose roles move + # against the older wheels and so keep `_initials` out of its + # diff there. + "fix(#397) the Catalan/Polish link joins two surnames": + _Claim(5, ('family', 'middle', 'suffix'), "0c31a48b1867", + ('DEFAULT',)), + "fix(#397) the Catalan link reports a connective-or-initial in a one-case name": + _Claim(3, ('_ambiguities',), "e43f9595ac02", ('DEFAULT',)), + "fix(#461) a connective holding its part alone contributes an initial": + _Claim(17, ('_initials',), "797473971e75", ('DEFAULT',)), + "fix(#461) a connective with a name word beside it stops contributing an initial": + _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), }, "expected_since_2.1.0.toml": { # #436/#437's Latin alternation, first in every ledger. # Ten corpus names, `suffix` alone: the rule moves the # SEPARATOR and no role, so a widening that took a role would # change the row here before it reached the gate. + # 2026-09-20, #397: 11 -> 14, three new comma-written + # generation runs ('Josep Lluis Carod i III', 'Josep Lluis + # Carod i V', 'Rovira, Josep Carod i Jr.'), each of which #397 + # leaves byte-identical -- measured against the parent + # 46651750 -- and each of which arrived with that change's own + # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(11, ('suffix',), "e665eae5c5df", None), + _Claim(14, ('suffix',), "4f5e63e709ae", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -4806,6 +5069,27 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('_ambiguities', 'family', 'maiden', 'middle'), 'fa3fe4878b47', None), "fix(#411/#533) the bound-given pair after a comma, and the clause it keeps its credential in": _Claim(1, ('_ambiguities', 'given', 'maiden', 'middle'), '431d3dd24c14', None), + # 2026-09-20, #397 and #461's four new rules, all literal + # alternations of exactly the names each explains -- a count + # that moves means the alternation grew, and _MUST_NOT_MATCH + # carries the walls a reach this small cannot be. + # + # The join reaches 5 and declares the union of its two diff + # shapes; the one-case report rule reaches 3 on + # `_ambiguities` alone, no role moving on any of them. #461's + # two halves are 16 and 13 on `_initials` alone, and they + # are two rules and not one because a rule listing `_initials` + # may list nothing else, so the two readings of one sentence + # cannot share a row either. + "fix(#397) the Catalan/Polish link joins two surnames": + _Claim(5, ('family', 'middle', 'suffix'), "0c31a48b1867", + ('DEFAULT',)), + "fix(#397) the Catalan link reports a connective-or-initial in a one-case name": + _Claim(3, ('_ambiguities',), "e43f9595ac02", ('DEFAULT',)), + "fix(#461) a connective holding its part alone contributes an initial": + _Claim(16, ('_initials',), "075dc34f9e95", ('DEFAULT',)), + "fix(#461) a connective with a name word beside it stops contributing an initial": + _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), }, "expected_since_2.3.0.toml": { # #383/#479's three rules, the first this ledger carries. The @@ -4828,8 +5112,12 @@ def _claim(rule: dict) -> _Claim: "ec00806a06f0", ('DEFAULT',)), "fix(#383/#479) a marked connective letter in a one-case name is reported": _Claim(7, ('_ambiguities',), "2eb6eff33836", ('DEFAULT',)), - "fix(#383/#479) a bare capital connective in an all-upper name stops initialing": - _Claim(1, ('_initials',), "7ff29af96914", ('DEFAULT',)), + # `fix(#383/#479) a bare capital connective in an all-upper + # name stops initialing` had a row here until 2026-09-20 + # (reach 1, 'JUAN Y GARCIA'). #461 gave that name its 'Y' + # back on both surfaces, which is what 2.3.0 itself gives, so + # the rule stopped explaining anything and is deleted from the + # ledger. # #289's alternation, appended with #516's and the report # rule's below it. Literal-anchored to the movers a written # case contrast decides, `orders` DEFAULT: the class is a @@ -4928,6 +5216,32 @@ def _claim(rule: dict) -> _Claim: # through. "fix(#533) the maiden clause ends at the credential in a native-script name too": _Claim(1, ('_ambiguities', 'maiden', 'suffix'), "6bab87214ddf", None), + # 2026-09-20, #397 and #461's four new rules, all literal + # alternations of exactly the names each explains -- a count + # that moves means the alternation grew, and _MUST_NOT_MATCH + # carries the walls a reach this small cannot be. + # + # The join reaches 5 and declares the union of its two diff + # shapes; the one-case report rule reaches 3 on + # `_ambiguities` alone, no role moving on any of them. #461's + # two halves are 17 and 22 on `_initials` alone, and they + # are two rules and not one because a rule listing `_initials` + # may list nothing else, so the two readings of one sentence + # cannot share a row either. + # + # Both halves are at their widest here: the readmission + # half holds 'Juan y Garcia née Jones' as at 2.2.0, and the + # joining half holds nine names the #449 report masks at every + # earlier baseline (compare.py's roles-identical guard, #484). + "fix(#397) the Catalan/Polish link joins two surnames": + _Claim(5, ('family', 'middle', 'suffix'), "0c31a48b1867", + ('DEFAULT',)), + "fix(#397) the Catalan link reports a connective-or-initial in a one-case name": + _Claim(3, ('_ambiguities',), "e43f9595ac02", ('DEFAULT',)), + "fix(#461) a connective holding its part alone contributes an initial": + _Claim(17, ('_initials',), "797473971e75", ('DEFAULT',)), + "fix(#461) a connective with a name word beside it stops contributing an initial": + _Claim(22, ('_initials',), "e73827447b4e", ('DEFAULT',)), }, } @@ -5093,9 +5407,14 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: # because a later edit that moves either rule, or widens the # numeral rule's `fields`, would take one silently -- exactly the # absorption #451 exists to end. + # 2026-09-20, #397: the rule kept the name and was RETITLED in + # place -- its standing moved from NOT WANTED and unresolved + # to the decided boundary of a landed rule -- so this pin + # follows the new issue string. 'Carod i' itself is unchanged + # by that landing, measured against the parent 46651750. "Carod i": - "fix(#397) NOT WANTED: a trailing Catalan/Polish linking " - "'i' is read as a generation marker and the family is lost", + "fix(#397) accepted: a two-word Catalan link has no name " + "word to its right and stays the generation", # #289/#516's one. '田中 太郎, MA' is this arc's own case row # (radar tier): two whitespace tokens before the comma make it # two NAME words by rules.md#C1's count, so the suffix reading @@ -5158,6 +5477,60 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: "john e jones": "fix(#528) the facade's initials follow the parse's " "connective tags", + # The eleven equal-`fields` contests #461 opened on + # 2026-09-20, the same shape as #528's pair above and against + # the same rule: both carry `fields = ["_initials"]`, so + # neither is narrower, `precedes_narrower` has nothing to + # declare, and classify() hands the name to whichever is + # written first. The #461 rule is written first and WINS each + # of them, because 1.4.0 rendered every one of these with the + # granularity the tree does -- the per-word grouping moved + # nothing on them, the letter COUNT did (measured 2026-09-20 + # from the 1.4.0 wheel and from the parent 46651750: all + # eleven agreed with 1.4.0 before this bundle). 'Carod y de + # Rovira i' loses to the #461 rule on the particle-chain rule + # as well, and one row pins the name once. + # + # The other four names the #461 rule explains are NOT + # contested, each kept out by a different half of the + # connective alternation: 'Juan de y' and 'johnny y' end on + # their letter and the alternation wants whitespace after it; + # 'Smith, John e, III, Jr' has a comma there instead; and + # 'Fritz Freiherr und von Bar' carries 'und', which is in no + # alternation. None has a row. + "Carod y de Rovira i": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "Garcia y Lopez": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "John e Smith": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "John e Smith III": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "John e Smith, III": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "John y Jane": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "Jose e Maria": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "Juan y Garcia": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "Lt.Gov. juan e garcia": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "john e jones, III": + "fix(#461) a connective holding its part alone contributes " + "an initial", + "juan y garcia": + "fix(#461) a connective holding its part alone contributes " + "an initial", # the glued/spaced boundary. 'Andersonさん' and '김민준씨' left # suffix-routing for a rule that names them; '김민준 씨.' is # spaced and stays on the spaced rule, which #372 taught to diff --git a/tools/differential/compare.py b/tools/differential/compare.py index be19aff7..68692eb1 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -1820,6 +1820,41 @@ class _ShapeMismatch(NamedTuple): # measured. The winners are pinned in _CROSS_RULE_WINNERS. "john e smith": ("_initials",), "john e jones": ("_initials",), + # #461's eleven, adjudicated 2026-09-20, below #528's pair and + # contested against the same rule for the same structural + # reason: `fix(#461) a connective holding its part alone + # contributes an initial` and `fix(initials-per-word) a + # connective run initials each word` both carry + # `fields = ["_initials"]`, so neither is narrower, + # `precedes_narrower` has no narrower rule to name, and file + # order is the whole decision. 'Carod y de Rovira i' carries + # TWO losers -- the particle-chain per-word rule reaches it + # through 'de Rovira' -- and one row covers both, as this + # dict's header says a row does. + # + # The ARGUMENT for the winner, and it is the same on all + # eleven: 1.4.0 rendered each of these names with the SAME + # granularity the tree does ('J. G.', never 'J G.'), so the + # per-word grouping moved nothing on them; what moved is the + # letter count, the connective holding its part alone + # contributing an initial again. Measured 2026-09-20 against + # the 1.4.0 wheel and against the parent 46651750: every one + # of the eleven AGREED with 1.4.0 before this bundle and + # diverges from it now, which no rule about a 2.0.0 rendering + # change can be describing. Deleting or narrowing the #461 + # rule hands them back to the grouping rule silently, which is + # the handover these rows exist to catch. + "Carod y de Rovira i": ("_initials",), + "Garcia y Lopez": ("_initials",), + "John e Smith": ("_initials",), + "John e Smith III": ("_initials",), + "John e Smith, III": ("_initials",), + "John y Jane": ("_initials",), + "Jose e Maria": ("_initials",), + "Juan y Garcia": ("_initials",), + "Lt.Gov. juan e garcia": ("_initials",), + "john e jones, III": ("_initials",), + "juan y garcia": ("_initials",), # #289/#516's one, adjudicated 2026-09-18, below both cohorts # above for the same provenance reason. '田中 太郎, MA' entered # the corpus with this arc's own case row (radar tier, a diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 12901d6a..2f5a7781 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -81,7 +81,16 @@ # never claim. [[change]] issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas" -name_regex = "^(?:JOHN DOE PHD MD|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397: one name joins the alternation and the rule does +# not change. 'Josep Lluis Carod i III' is a two-word generation run +# this baseline renders 'i, III' and the tree renders 'i III' -- the +# whole of its diff here. #397 moves NOTHING on it: measured against +# the parent 46651750 the parse is byte-identical before and after +# this bundle (given 'Josep', middle 'Lluis', family 'Carod', suffix +# 'i III'), the link having no name word to its right. What grew is +# the CORPUS -- the name arrived with #397's own case rows -- and the +# run rendering is what the diff is about. +name_regex = "^(?:JOHN DOE PHD MD|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # #346: swami, guru, baba and lama moved from the TITLES-only block @@ -1259,18 +1268,25 @@ name_regex = "^(?:Ahmad Jayadi, CHA|Aishwarya Rai|John Smith RAI|John Smith, RAI fields = ["family", "given", "middle", "suffix"] [[change]] -issue = "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as a generation marker and the family is lost" +issue = "fix(#397) accepted: a two-word Catalan link has no name word to its right and stays the generation" # 'Carod i': the tree reads given 'Carod', family '', suffix 'i'. 'i' # is a bare entry in SUFFIX_WORDS (nameparser/config/suffixes.py -- # the Roman-numeral generation markers i/ii/iii/iv/v, not -# SUFFIX_ACRONYMS), so a two-word name ending in it reads as a -# name plus a trailing generation rather than a two-word family link. -# #397 (OPEN, milestone v2.2, enhancement) reports that Catalan and -# Polish link two surnames with 'i' functioning as a conjunction, the -# way Spanish already links them with 'y' ("Juan Garcia y Lopez" reads -# family 'Garcia y Lopez'); nameparser's CONJUNCTIONS vocabulary does -# not include 'i', so nothing there joins the link, and SUFFIX_WORDS' -# numeral reading is what actually claims the trailing word instead. +# SUFFIX_ACRONYMS), so a two-word name ending in it reads as a name +# plus a trailing generation rather than a two-word family link. +# +# DECIDED, not unresolved, and that is what changed on 2026-09-20: +# #397 landed, 'i' IS connective vocabulary now, and the Catalan link +# joins wherever a name word stands on each side of it -- 'Josep Carod +# i Rovira' and 'Carod i Rovira, Josep' both read family 'Carod i +# Rovira' and have a rule of their own below. This name is the +# boundary that rule states: a connective with nothing to its right is +# connecting nothing, and a word of that vocabulary ending a name is +# the generation it also spells. Two pieces do not reach the join loop +# at all, which needs three, so the reading is refused one gate +# earlier still. rules.md#P3 carries both sentences and this name is +# one of its example lines. decisions.md#P3 records the count decision +# and the both-sides scope behind them. # # Whether 1.4 itself read the trailing word as `family` is outside # what this ledger's worker can check directly @@ -1279,37 +1295,83 @@ issue = "fix(#397) NOT WANTED: a trailing Catalan/Polish linking 'i' is read as # baseline moves exactly {family, suffix}, which is consistent with # that reading and is what the fields below claim. # -# Classified because the cause is known and recorded, NOT because the -# reading is wanted -- the same standing fix(#342) above claims, from -# decisions.md#P5's 2026-08-22 #369 entry. A [[never]] exclusion was -# considered and rejected for the same reason as fix(#342) above: an -# excluded name reports UNEXPLAINED and exits non-zero, and #397's own -# "What needs deciding" section leaves open both whether 'i' joins -# CONJUNCTIONS at all and how the join's rootname carve-out should -# treat a word that is both a conjunction and suffix vocabulary -- -# unresolved, so the fix is not yet in hand. -# -# DELETE THIS RULE when #397 lands. The gate then reports it as -# 'EXPLAINED NOTHING', which is the reminder. +# Classified because the cause is known and RECORDED AS WANTED, which +# is a stronger standing than the fix(#342) rule above: that one is +# NOT WANTED and unresolved, this one is the stated boundary of a +# landed rule. A [[never]] exclusion is still wrong for it -- an +# excluded name reports UNEXPLAINED and exits non-zero. # # Literal-anchored to the one corpus name that reaches this shape, for # fix(#342) above's reason: a rule keyed on "two-word name, trailing # single letter" would pre-excuse every generation-marker regression, -# not just this one. Measured, the four longer corpus names in the -# same family produce no diff at this baseline at all, so none of them -# reaches classify() and none needs a rule here: 'Carod i Rovira', -# 'Josep Carod i Rovira' and 'Josep Lluis Carod i Rovira' read 'i' into -# `middle` with `family` already the tail word ('Rovira'), and 'Lluis -# Carod i' reads it into `suffix` with `family` already populated -# ('Carod') -- unlike 'Carod i' above, none of the four has an empty -# `family`, and all four read this way at every baseline this ledger's -# worker can reach. That is #397's open enhancement (the family link -# is still one word short of what a Catalan reader wants), not a -# 1.4-to-2.x regression, so it is out of scope for a ledger that only -# records diffs FROM 1.4.0. +# not just this one. The three longer names in the same family are +# each accounted for elsewhere now -- 'Carod i Rovira' and 'Josep +# Carod i Rovira' by the fix(#397) join rule below, and 'Lluis Carod +# i' by the same both-sides boundary as this name, one word longer and +# therefore diffing nothing at all. name_regex = "(?i)^carod\\s+i$" fields = ["family", "suffix"] +[[change]] +issue = "fix(#397) the Catalan/Polish link joins two surnames" +# 'i' is connective vocabulary since #397, and a connective counts as +# a name word toward rules.md#P3's three-word carve-out whatever else +# the vocabulary says it is -- which matters here because 'i' is also +# the roman numeral. So a four-word name reaches the threshold and the +# link joins. 1.4.0 read 'Josep Carod i Rovira' as middle 'Carod i' +# with family 'Rovira', and 'Carod i Rovira, Josep' -- where the link +# stands after a family comma -- as family 'Carod Rovira' with the +# link taken as a GENERATION in `suffix`, which is why the two move +# different field sets and are grouped here rather than split. The +# three longer spellings move with the first: 'Josep Carod i Rovira +# III' and 'Josep Lluis Carod i Rovira' the same {family, middle}, and +# 'Josep Carod i de Rovira' the same pair with the link joining PAST a +# particle (middle 'Carod i' -> 'Carod i de', family 'de Rovira' -> +# 'Rovira'). Five names, `fields` the union of their two shapes. +# +# All five entered this corpus with #397's own rules.md#P3 example +# lines and case rows; before that they were radar-tier names the +# contract did not answer for, which is why a rule appears at this +# baseline only now. decisions.md#P3. +# +# Literal-anchored to the five. The shape is "a single letter between +# two name words", which reaches every corpus name carrying a bare +# initial and would pre-excuse the opposite reading -- #397's whole +# accepted cost is that some such letters must NOT join. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the wall: +# 'Josep i Rovira' (three words, under the carve-out), 'Carod i' (the +# rule above), 'John Quincy Smith i' (the link with nothing to its +# right) and 'Josep Lluis Carod i III' (a generation run, the +# fix(#436/#437) rule at the head of this file). +name_regex = "^(?:Carod i Rovira, Josep|Josep Carod i Rovira|Josep Carod i Rovira III|Josep Carod i de Rovira|Josep Lluis Carod i Rovira)$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(suffix-routing) a two-word trailing generation run peels whole, and the name word before it is the family" +# 'Josep Lluis Carod i V': v1 read middle 'Lluis Carod', last 'i', +# suffix 'V' -- it peeled the LAST word of the trailing run and left +# the first to the family -- where 2.x peels the run whole and reads +# middle 'Lluis', family 'Carod', suffix 'i V'. The same shape +# 'John Smith Mc V' carries on the fix(#424) rule further down, whose +# sentence is about a PARTICLE opening the run; here both words are +# generation vocabulary, so it needs a rule rather than that one's +# widening. +# +# NOT this bundle's doing, and measured rather than argued: against +# the parent 46651750 the parse is byte-identical before and after +# #397 and #461 (given 'Josep', middle 'Lluis', family 'Carod', suffix +# 'i V'), the link having a generation marker and not a name word to +# its right. The name is new to the CORPUS -- it arrived with #397's +# case rows as exactly that boundary -- so a divergence v2 has carried +# since 2.0 is classified here for the first time. +# +# Literal, one name: the shape is "a trailing run of two suffix words" +# and would stand ready to explain every future peel regression. +name_regex = "^Josep Lluis Carod i V$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + [[change]] issue = "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole" # Only fires when a custom suffix delimiter is configured (Policy / @@ -3053,33 +3115,22 @@ name_regex = "(?i)^de los santos$" fields = ["_initials"] orders = ["DEFAULT"] -[[change]] -issue = "feat(#269) a recognized non-Latin connective contributes no initial" -# 'محمد و علي': #269 (2.0) put the non-Latin connectives into -# CONJUNCTIONS, and `و` has been read as one ever since -- the roles -# agree with 1.4.0 word for word (given 'محمد', middle 'و', family -# 'علي'), because 1.4.0 already put the lone letter in the middle; -# what moved is that a connective contributes no initial -# (rules.md#R3), so 'م. و. ع.' became 'م. ع.'. That is the same -# recognition the two `feat(#269)` ROLE rules above classify, read -# from the derived view, and it needs a rule of its own because a -# rule listing `_initials` may list nothing else (#484). -# -# It is being classified now, three minors late, because the name -# only just entered a corpus: it is a rules.md#P3 example (the -# caseless control -- a script with one case carries no case evidence -# and never reaches the #383/#479 fork) and arrived with that rule's -# case-table rows on 2026-09-13. Nothing about it moves in 2.4; -# measured the same day, its parse and both its initials views are -# byte-identical before and after the fork. -# -# Literal, one name: the shape -- "a name whose only connective is a -# caseless one" -- reaches every Arabic and Cyrillic corpus name the -# moment one carries a `و` or an `и`, and the two Cyrillic names that -# do move here move their ROLES and belong to the rule above. -name_regex = "^محمد و علي$" -fields = ["_initials"] -orders = ["DEFAULT"] +# `feat(#269) a recognized non-Latin connective contributes no +# initial` STOOD HERE until 2026-09-20 and is DELETED, not declared +# dormant. It covered '^محمد و علي$' on `fields = ["_initials"]`, +# saying that `و` was recognized as a connective in 2.0 and so stopped +# contributing a letter: 'م. و. ع.' became 'م. ع.'. #461 asks the +# question of the PART instead, and `و` is the whole middle part with +# no name word beside it to join, so it initials again -- the tree +# gives 'م. و. ع.' from both surfaces, which is 1.4.0's own answer, +# measured from the wheel. The name stops diffing at this baseline +# and the rule reported EXPLAINED NOTHING. Its REASON is wrong now, +# not merely unreached, so a `dormant = "..."` declaration would +# assert something false; the ledger's own convention +# (tools/differential/README.md, the dormancy tiers) is deletion. +# It still diffs at the four 2.x baselines, where 2.0 through 2.3 all +# dropped the letter, and the fix(#461) readmission rule in each of +# those ledgers carries it. decisions.md#R3. [[change]] issue = "fix(#528) the facade's initials follow the parse's connective tags" @@ -3119,7 +3170,90 @@ issue = "fix(#528) the facade's initials follow the parse's connective tags" # (measured 2026-09-13 at this baseline: both sat in its classified # list). Why this rule takes them instead: the block header above. # _CROSS_RULE_WINNERS pins it and _RECORDED_DIFFS carries the shapes. -name_regex = "^(?:john e smith|john e jones|jones, john e|JUAN Y GARCIA)$" +# +# 2026-09-20, #461: 'JUAN Y GARCIA' LEAVES the alternation, and the +# name is the one of the four this rule can no longer say anything +# about. Its 'Y' holds the middle part alone, so under #461 it +# initials again on both surfaces: the tree gives 'J. Y. G.' from +# HumanName.initials() and 'J. Y. G.' from parse(...).initials(), +# which is word for word what the 1.4.0 wheel gives, so the name has +# no diff left at this baseline to explain (measured 2026-09-20 from +# the wheel and from the tree). Dropping it from the regex rather +# than leaving it there is the narrowing this file asks for -- a +# member reaching a name that cannot diff is a standing claim on +# whatever that name diffs NEXT. The other three still move for this +# rule's own reason, the facade reading the parse's tags, and are +# untouched. The name keeps a rule at 2.0.0/2.1.0/2.2.0, where +# fix(#462) explains it and its surface flipped; at 2.3.0 the rule +# written for it is deleted for this reason. decisions.md#R3. +name_regex = "^(?:john e smith|john e jones|jones, john e)$" +fields = ["_initials"] +orders = ["DEFAULT"] + +# --------------------------------------------------------------- +# #461: A CONNECTIVE CONTRIBUTES NOTHING WHERE IT IS JOINING, AND +# INITIALS LIKE ANY OTHER NAME WORD WHERE ITS PART HOLDS NOTHING +# ELSE FOR IT TO JOIN (rules.md#R3, decisions.md#R3). +# +# ONE rule at this baseline and two at every 2.x one, and the +# asymmetry is measured rather than tidy. The half that LOSES a +# letter -- a connective with name words beside it -- was already +# diffing here before this change, on the 2.0.0 per-word GROUPING +# the four fix(initials-per-word) rules below describe, so its names +# stay on those rules and the first of them carries a dated +# paragraph saying which half of its diff is now which. The half +# that GAINS one had no 1.4.0 diff at all: every name below agreed +# with the wheel before this bundle and diverges from it now, which +# is a parity break this file has to state in its own words rather +# than let an older rule absorb. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#461) a connective holding its part alone contributes an initial" +# Fifteen names, each one a part whose only word is the connective, +# so there is nothing there for it to join and it initials as any +# other name word does -- which is what `family_base` and +# `given_base` have said about the same token since 2.2 +# (rules.md#R2/#R3). 1.4.0 dropped the letter and so did every +# release since; the tree keeps it. Measured from the 1.4.0 wheel +# and from the tree on 2026-09-20, name by name: 'Juan de y' 'J.' -> +# 'J. y.'; 'johnny y' 'j.' -> 'j. y.'; 'Garcia y Lopez' 'G. L.' -> +# 'G. y. L.'; 'Juan y Garcia' 'J. G.' -> 'J. y. G.' and 'juan y +# garcia' the lowercase twin; 'John y Jane' 'J. J.' -> 'J. y. J.'; +# 'Carod y de Rovira i' 'C. R.' -> 'C. y. R.'; 'John e Smith' 'J. S.' +# -> 'J. e. S.', with 'John e Smith III', 'John e Smith, III' and +# 'Smith, John e, III, Jr' giving the same pair; 'john e jones, III' +# 'j. j.' -> 'j. e. j.'; 'Jose e Maria' 'J. M.' -> 'J. e. M.'; +# 'Lt.Gov. juan e garcia' 'j. g.' -> 'j. e. g.'; and 'Fritz Freiherr +# und von Bar' 'F.' -> 'F. u.', the German spelling of the same +# shape. +# +# NOT THE PER-WORD GROUPING BELOW, and that is the reason this rule +# exists at this baseline rather than the names being left to it. +# 1.4.0 rendered every one of these with the same granularity the +# tree does -- 'J. G.', not 'J G.' -- so nothing about the grouping +# moved; what moved is the LETTER COUNT. Each of the eleven whose +# spelling carries a lowercase ' y ' or ' e ' is reached by that +# rule's alternation all the same, and would be absorbed by it +# silently, which is what _RECORDED_DIFFS and _CROSS_RULE_WINNERS +# adjudicate: equal `fields`, so neither rule is the narrower and +# file order is the whole decision, and this rule is written first. +# +# `Fritz Freiherr und von Bar`, 'johnny y' and 'Smith, John e, III, +# Jr' reach no such contest -- the alternation below carries no +# 'und', and it wants whitespace AFTER the letter, which a letter at +# end of string and a letter before a comma never offer -- so the +# three have no pin. +# +# Literal-anchored to the fifteen. The shape is "a part holding one +# connective and nothing else", which no regex over the raw string +# can state -- the reading is the parse's -- and the nearest +# writable shape, "a name carrying a lone cased letter", reaches +# every bare-initial name in the corpora. _MUST_NOT_MATCH carries +# the wall: 'Jon Dough and' (base 'Dough and', a connective with a +# name word beside it, keeping 'J. D.'), 'Juan Velasquez y Garcia' +# (the joined run, keeping 'J. V. G.') and 'Juan Garcia Lopez y'. +name_regex = "^(?:Carod y de Rovira i|Fritz Freiherr und von Bar|Garcia y Lopez|John e Smith|John e Smith III|John e Smith, III|John y Jane|Jose e Maria|Juan de y|Juan y Garcia|Lt\\.Gov\\. juan e garcia|Smith, John e, III, Jr|john e jones, III|johnny y|juan y garcia)$" fields = ["_initials"] orders = ["DEFAULT"] @@ -3133,7 +3267,29 @@ issue = "fix(initials-per-word) a connective run initials each word (facade, sin # The parse is identical; this is the facade's rendering, changed in # 2.0.0 and recorded first in 2.3.0's release log (decisions.md, # "the initials view"). Strip periods and spaces and every name here -# agrees with 1.4.0 letter for letter. +# agreed with 1.4.0 letter for letter until 2026-09-20. +# +# 2026-09-20, #461: THAT SENTENCE NO LONGER HOLDS FOR EVERY NAME, and +# the ones it stopped holding for have TWO causes in one diff. A +# connective with a name word beside it now contributes no initial at +# all, so the group it used to render per-word is simply gone: 'John +# and Jane Smith' goes 'J a J. S.' at the wheel to 'J. J. S.' in the +# tree, where the grouping half is 'J a J. S.' -> 'J. a. J. S.' and +# #461's half is the drop of the 'a'. Measured against the parent +# 46651750 on 2026-09-20, the names this rule explains where BOTH +# halves are now in the diff are: 'Assoc Dean of Chemistry Robert +# Johns', 'Dean of Chemistry', 'Dean of Chemistry Robert Johns', +# 'Duke of Edinburgh', 'Duke of Wellington', 'John & Jane', 'John and +# Jane Aznar y Lopez', 'John and Jane Smith', 'John of the Doe', 'Jose +# e Maria Santos', 'Juan & Garcia', 'Juan and Garcia', 'Juan y Eva +# Garcia', 'Mr. and Mrs. John and Jane Smith', 'Rob And Beth Edmunds', +# 'and Jon Dough', 'the and Jon Dough' and the two 'part1 of ...' +# strings. Every one of them ALREADY diffed here before this bundle, +# and on this rule's own sentence, which is why they stay on it rather +# than moving to the fix(#461) rule above -- that rule is for the +# names whose 1.4.0 diff this change CREATED. Both halves are stated +# rather than one absorbing the other, the treatment fix(#462) and +# fix(#383/#479) already get in these files. decisions.md#R3. # # CASE-SENSITIVE, the whole alternation, deliberately -- and it is the # SINGLE LETTERS the case decides, because a capital `E`/`Y` is an diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 768d1761..4a28af15 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -62,7 +62,18 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # the unambiguous PhD already ended the walk, so the clause gave # the MA up before this change as it does after it. What grew is # the CORPUS, and the run rendering is what the diff is about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397: three names join the alternation and the rule +# does not change. 'Josep Lluis Carod i III' ('i, III' -> 'i III'), +# 'Josep Lluis Carod i V' ('i, V' -> 'i V') and 'Rovira, Josep Carod +# i Jr.' ('i, Jr.' -> 'i Jr.') are two-word generation runs this +# baseline renders with a comma and the tree renders with a space -- +# the whole of each one's diff here. #397 moves NOTHING on any of +# them: measured against the parent 46651750 all three parse +# byte-identically before and after this bundle, the link having a +# generation marker and not a name word to its right. What grew is +# the CORPUS -- the three arrived with #397's own case rows -- and +# the run rendering is what the diffs are about. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -1866,6 +1877,26 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # ledger's comment says so. # This is the ONE full copy: the 2.1.0 and 2.2.0 ledgers and the # three _CORPUS_CLAIMS entries point here rather than restating it. +# +# 2026-09-20, #461: THE SURFACE FLIPPED AND THE RULE DID NOT. At +# this baseline the wheel gives facade 'J. G.' and core 'J. Y. G.' +# for 'JUAN Y GARCIA' (measured 2026-09-20 from the 2.0.0 wheel). +# Before this change only the CORE moved, 'J. Y. G.' -> 'J. G.', a +# connective contributing no initial; after it only the FACADE moves, +# 'J. G.' -> 'J. Y. G.', and the core agrees with the baseline word +# for word -- the 'Y' holds its part alone, so it initials again on +# both surfaces and the tree gives 'J. Y. G.' from each. `_initials` +# is one pseudo-field, so this rule still explains the name; what +# changed is WHICH HALF of the disagreement it is explaining, which +# is why the paragraph above reads backwards without this one. +# rules.md#R3, decisions.md#R3. +# +# The name stops diffing at 1.4.0 and at 2.3.0 under the same move, +# and the rule written for it in each of those ledgers goes: the +# 2.3.0 `fix(#383/#479) a bare capital connective in an all-upper +# name stops initialing` is deleted outright, and 1.4.0's +# `fix(#528) ...` keeps its other three names and drops this one from +# its alternation. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" fields = ["_initials"] orders = ["DEFAULT"] @@ -2983,3 +3014,177 @@ issue = "fix(#424) the maiden walk stops before the trailing numeral, with the f # front of the gate for the first time at a baseline that can see it. name_regex = "^Jane Doe nee Smith V$" fields = ["_ambiguities", "maiden", "suffix"] + +# --------------------------------------------------------------- +# #397 + #461: THE CATALAN LINK JOINS, AND A CONNECTIVE INITIALS +# WHERE IT JOINS NOTHING (2026-09-20). Four rules: the join's ROLES, +# the REPORT a one-case spelling makes instead of joining, and +# #461's two halves, which are one sentence read in its two +# directions and need two rules because a rule listing `_initials` +# may list nothing else (#484). +# +# LAST in the file, and that is narrow-first rather than a +# preference: the join rule declares {family, middle, suffix} and +# every rule above it that reaches one of its five names declares a +# strict subset, so an earlier position is an order-decided contest +# the run refuses (#382). +# +# rules.md#P3 states the join and its two boundaries; rules.md#R3 +# states what a connective contributes. decisions.md#P3 carries the +# count decision and the both-sides scope, decisions.md#R3 the +# criterion and its measured blast radius. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) the Catalan/Polish link joins two surnames" +# 'i' is connective vocabulary since #397, and a connective counts as +# a name word toward rules.md#P3's three-word carve-out whatever else +# the vocabulary says it is -- which matters here because 'i' is also +# the roman numeral. So a four-word name reaches the threshold and the +# link joins. This baseline read 'Josep Carod i Rovira' as middle +# 'Carod i' with family 'Rovira', and 'Carod i Rovira, Josep' -- where +# the link stands after a family comma -- as family 'Carod Rovira' +# with the link taken as a GENERATION in `suffix`, which is why the +# two move different field sets and are grouped here rather than +# split. The three longer spellings move with the first: 'Josep Carod +# i Rovira III' and 'Josep Lluis Carod i Rovira' the same +# {family, middle}, and 'Josep Carod i de Rovira' the same pair with +# the link joining PAST a particle (middle 'Carod i' -> 'Carod i de', +# family 'de Rovira' -> 'Rovira'). Five names, `fields` the union of +# their two shapes. +# +# All five entered this corpus with #397's own rules.md#P3 example +# lines and case rows; before that they were radar-tier names the +# contract did not answer for, which is why a rule appears at this +# baseline only now. +# +# Literal-anchored to the five. The shape is "a single letter between +# two name words", which reaches every corpus name carrying a bare +# initial and would pre-excuse the opposite reading -- #397's whole +# accepted cost is that some such letters must NOT join. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the wall: +# 'Josep i Rovira' (three words, under the carve-out), 'Carod i' and +# 'John Quincy Smith i' (the link with nothing to its right) and the +# two 'Josep Lluis Carod i ' spellings, whose trailing +# generation run is the fix(#436/#437) rule's at the head of this +# file. +name_regex = "^(?:Carod i Rovira, Josep|Josep Carod i Rovira|Josep Carod i Rovira III|Josep Carod i de Rovira|Josep Lluis Carod i Rovira)$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#397) the Catalan link reports a connective-or-initial in a one-case name" +# 'JOSEP CAROD I ROVIRA', 'JOSEP LLUIS CAROD I ROVIRA' and 'josep +# carod i rovira'. 'i' joined `conjunctions_ambiguous` with #397, so +# in a name written wholly in one case -- where the writing says +# nothing either way -- the letter reads as an INITIAL and the call is +# reported, exactly as 'e' has read since #383/#479. #527's mechanism, +# a new letter: rules.md#P3's marked subset. +# +# ROLES DO NOT MOVE on any of the three, and that is the point of the +# marked subset rather than an accident of these spellings: reading +# the letter as an initial leaves it the middle-name word this +# baseline already had, so the whole diff is the report. Measured +# against the parent 46651750 on 2026-09-20, the same three names' +# parses are byte-identical before and after the bundle and only +# `ambiguities` moved. The lower-case spellings where the letter WAS +# the generation before -- 'rovira, i', 'john smith i jr', 'maier, amy +# i, jr.', 'josep de carod i rovira' -- do move fields under the same +# marked-subset reading, and no corpus carries one of them, so this +# rule is about the report alone. +# +# No rule at 1.4.0 and that is structural: `_ambiguities` is a v2 +# field and cannot diff against a v1 worker at all. +# +# Literal, three names. _MUST_NOT_MATCH carries the mixed-case +# spellings, where the writing decides the letter and the JOIN is what +# moved, and 'JUAN GARCIA Y LOPEZ', the unmarked letter in the same +# one-case position, which reports nothing. +name_regex = "^(?:JOSEP CAROD I ROVIRA|JOSEP LLUIS CAROD I ROVIRA|josep carod i rovira)$" +fields = ["_ambiguities"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective holding its part alone contributes an initial" +# 16 names, each one a part whose only word is the connective, +# so there is nothing there for it to join and it initials as any +# other name word does -- which is what `family_base` and `given_base` +# have said about the same token since 2.2 (rules.md#R2/#R3). This +# baseline drops the letter; the tree keeps it. Measured from the +# 2.0.0 wheel and from the tree on 2026-09-20: 'Juan de y' 'J.' -> +# 'J. y.'; 'johnny y' 'j.' -> 'j. y.'; 'Garcia y Lopez' 'G. L.' -> +# 'G. y. L.'; 'Juan y Garcia' 'J. G.' -> 'J. y. G.' and 'juan y +# garcia' the lowercase twin; 'John y Jane' 'J. J.' -> 'J. y. J.'; +# 'Carod y de Rovira i' 'C. R.' -> 'C. y. R.'; 'John e Smith' 'J. S.' +# -> 'J. e. S.', with 'John e Smith III', 'John e Smith, III' and +# 'Smith, John e, III, Jr' giving the same pair; 'john e jones, III' +# 'j. j.' -> 'j. e. j.'; 'Jose e Maria' 'J. M.' -> 'J. e. M.'; +# 'Lt.Gov. juan e garcia' 'j. g.' -> 'j. e. g.'; and 'Fritz Freiherr +# und von Bar' 'F.' -> 'F. u.', the German spelling of the same shape. +# BOTH SURFACES move together and to the same string on every one of +# them, the two views reading one mark (#528, decisions.md#R3). +# 'محمد و علي' is the readmission RESTORING 1.4.0 parity rather than +# breaking it: 'م. ع.' here, 'م. و. ع.' in the tree, which is the +# string the 1.4.0 wheel gives. The 1.4.0 ledger's +# `feat(#269) a recognized non-Latin connective contributes no +# initial` explained that name and is DELETED in the same commit as +# this rule, the name having stopped diffing there. +# +# ONE CAUSE, measured, not two. Every name here agreed with this +# baseline's wheel BEFORE the bundle -- the parent 46651750 gives the +# wheel's string on each -- so none of the diff belongs to #383/#479's +# one-case fork or to any older change; the whole of it is #461. +# +# Literal-anchored. The shape is "a part holding one connective and +# nothing else", which no regex over the raw string can state -- the +# reading is the parse's -- and the nearest writable shape, "a name +# carrying a lone cased letter", reaches every bare-initial name in +# the corpora. _MUST_NOT_MATCH carries the wall: 'Jon Dough and' (base +# 'Dough and', a connective with a name word beside it, keeping +# 'J. D.'), 'Juan Velasquez y Garcia' (the joined run, keeping +# 'J. V. G.') and 'Juan Garcia Lopez y'. +name_regex = "^(?:Carod y de Rovira i|Fritz Freiherr und von Bar|Garcia y Lopez|John e Smith|John e Smith III|John e Smith, III|John y Jane|Jose e Maria|Juan de y|Juan y Garcia|Lt\\.Gov\\. juan e garcia|Smith, John e, III, Jr|john e jones, III|johnny y|juan y garcia|محمد و علي)$" +fields = ["_initials"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective with a name word beside it stops contributing an initial" +# 13 names, the other direction of the same sentence: where the +# part holds a name word for the connective to join, the connective is +# doing a connective's work and contributes nothing. This baseline +# initialed it; the tree does not. 'John and Jane Smith' 'J. a. J. S.' +# -> 'J. J. S.'; 'Juan y Eva Garcia' 'J. y. E. G.' -> 'J. E. G.'; +# 'Jose e Maria Santos' 'J. e. M. S.' -> 'J. M. S.'; 'and Jon Dough' +# 'a. J. D.' -> 'J. D.' and 'the and Jon Dough' 't. a. J. D.' -> +# 'J. D.'; 'Rob And Beth Edmunds' 'R. A. B. E.' -> 'R. B. E.'; the two +# 'Dean of Chemistry Robert Johns' spellings 'D. o. C. R. J.' -> +# 'D. C. R. J.'; 'John and Jane Aznar y Lopez' 'J. a. J. A. L.' -> +# 'J. J. A. L.'; 'Mr. and Mrs. John and Jane Smith' 'J. a. J. S.' -> +# 'J. J. S.'; and the three Cyrillic spellings of 'Хосе И Мария +# Сантос' 'Х. И. М. С.' -> 'Х. М. С.', lower case and upper case +# alike. Both surfaces move together and to the same string. +# +# ONE CAUSE, measured. Each of these names agreed with this +# baseline's wheel before the bundle -- the parent 46651750 gives the +# wheel's string on every one, the Cyrillic three included, whose 'и' +# has been decided vocabulary since 2.0 and reaches no case fork. So +# none of the diff is #383/#479's and the whole of it is #461. At +# 1.4.0 these names diff too, but they diffed there BEFORE this +# bundle, on the 2.0.0 per-word GROUPING, and that ledger keeps them +# on `fix(initials-per-word) a connective run initials each word` +# with a dated paragraph naming both halves. +# +# CYRILLIC SPELLED OUT, not classed. A `\p{Cyrillic}`-style span +# would declare a script this rule is not about -- the sentence is +# about a PART, in any script -- and _SPAN_BEARING_RULES in +# tests/v2/test_ledger_guards.py records that this rule declares no +# span. +# +# Literal-anchored, for the GAIN rule's reason, and with the same +# wall: 'Jon Dough and', 'Juan Velasquez y Garcia' and 'Juan Garcia +# Lopez y' are _MUST_NOT_MATCH probes here too -- the criterion is the +# PART and never a word count, which is exactly what keeps those three +# fixed. +name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" +fields = ["_initials"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 8370d597..783d6fee 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -86,7 +86,18 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # the unambiguous PhD already ended the walk, so the clause gave # the MA up before this change as it does after it. What grew is # the CORPUS, and the run rendering is what the diff is about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397: three names join the alternation and the rule +# does not change. 'Josep Lluis Carod i III' ('i, III' -> 'i III'), +# 'Josep Lluis Carod i V' ('i, V' -> 'i V') and 'Rovira, Josep Carod +# i Jr.' ('i, Jr.' -> 'i Jr.') are two-word generation runs this +# baseline renders with a comma and the tree renders with a space -- +# the whole of each one's diff here. #397 moves NOTHING on any of +# them: measured against the parent 46651750 all three parse +# byte-identically before and after this bundle, the link having a +# generation marker and not a name word to its right. What grew is +# the CORPUS -- the three arrived with #397's own case rows -- and +# the run rendering is what the diffs are about. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -1769,6 +1780,26 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # remains and this rule still explains it, one `_initials` move. # expected_since_2.0.0.toml's copy of this rule carries the account; # it is the same at all three baselines. +# +# 2026-09-20, #461: THE SURFACE FLIPPED AND THE RULE DID NOT. At +# this baseline the wheel gives facade 'J. G.' and core 'J. Y. G.' +# for 'JUAN Y GARCIA' (measured 2026-09-20 from the 2.1.0 wheel). +# Before this change only the CORE moved, 'J. Y. G.' -> 'J. G.', a +# connective contributing no initial; after it only the FACADE moves, +# 'J. G.' -> 'J. Y. G.', and the core agrees with the baseline word +# for word -- the 'Y' holds its part alone, so it initials again on +# both surfaces and the tree gives 'J. Y. G.' from each. `_initials` +# is one pseudo-field, so this rule still explains the name; what +# changed is WHICH HALF of the disagreement it is explaining, which +# is why the paragraph above reads backwards without this one. +# rules.md#R3, decisions.md#R3. +# +# The name stops diffing at 1.4.0 and at 2.3.0 under the same move, +# and the rule written for it in each of those ledgers goes: the +# 2.3.0 `fix(#383/#479) a bare capital connective in an all-upper +# name stops initialing` is deleted outright, and 1.4.0's +# `fix(#528) ...` keeps its other three names and drops this one from +# its alternation. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" fields = ["_initials"] orders = ["DEFAULT"] @@ -2894,3 +2925,177 @@ issue = "fix(#533) the maiden clause ends at the credential in a native-script n # larger diff from those baselines. name_regex = "^田中 太郎 旧姓 佐藤 MA$" fields = ["_ambiguities", "maiden", "suffix"] + +# --------------------------------------------------------------- +# #397 + #461: THE CATALAN LINK JOINS, AND A CONNECTIVE INITIALS +# WHERE IT JOINS NOTHING (2026-09-20). Four rules: the join's ROLES, +# the REPORT a one-case spelling makes instead of joining, and +# #461's two halves, which are one sentence read in its two +# directions and need two rules because a rule listing `_initials` +# may list nothing else (#484). +# +# LAST in the file, and that is narrow-first rather than a +# preference: the join rule declares {family, middle, suffix} and +# every rule above it that reaches one of its five names declares a +# strict subset, so an earlier position is an order-decided contest +# the run refuses (#382). +# +# rules.md#P3 states the join and its two boundaries; rules.md#R3 +# states what a connective contributes. decisions.md#P3 carries the +# count decision and the both-sides scope, decisions.md#R3 the +# criterion and its measured blast radius. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) the Catalan/Polish link joins two surnames" +# 'i' is connective vocabulary since #397, and a connective counts as +# a name word toward rules.md#P3's three-word carve-out whatever else +# the vocabulary says it is -- which matters here because 'i' is also +# the roman numeral. So a four-word name reaches the threshold and the +# link joins. This baseline read 'Josep Carod i Rovira' as middle +# 'Carod i' with family 'Rovira', and 'Carod i Rovira, Josep' -- where +# the link stands after a family comma -- as family 'Carod Rovira' +# with the link taken as a GENERATION in `suffix`, which is why the +# two move different field sets and are grouped here rather than +# split. The three longer spellings move with the first: 'Josep Carod +# i Rovira III' and 'Josep Lluis Carod i Rovira' the same +# {family, middle}, and 'Josep Carod i de Rovira' the same pair with +# the link joining PAST a particle (middle 'Carod i' -> 'Carod i de', +# family 'de Rovira' -> 'Rovira'). Five names, `fields` the union of +# their two shapes. +# +# All five entered this corpus with #397's own rules.md#P3 example +# lines and case rows; before that they were radar-tier names the +# contract did not answer for, which is why a rule appears at this +# baseline only now. +# +# Literal-anchored to the five. The shape is "a single letter between +# two name words", which reaches every corpus name carrying a bare +# initial and would pre-excuse the opposite reading -- #397's whole +# accepted cost is that some such letters must NOT join. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the wall: +# 'Josep i Rovira' (three words, under the carve-out), 'Carod i' and +# 'John Quincy Smith i' (the link with nothing to its right) and the +# two 'Josep Lluis Carod i ' spellings, whose trailing +# generation run is the fix(#436/#437) rule's at the head of this +# file. +name_regex = "^(?:Carod i Rovira, Josep|Josep Carod i Rovira|Josep Carod i Rovira III|Josep Carod i de Rovira|Josep Lluis Carod i Rovira)$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#397) the Catalan link reports a connective-or-initial in a one-case name" +# 'JOSEP CAROD I ROVIRA', 'JOSEP LLUIS CAROD I ROVIRA' and 'josep +# carod i rovira'. 'i' joined `conjunctions_ambiguous` with #397, so +# in a name written wholly in one case -- where the writing says +# nothing either way -- the letter reads as an INITIAL and the call is +# reported, exactly as 'e' has read since #383/#479. #527's mechanism, +# a new letter: rules.md#P3's marked subset. +# +# ROLES DO NOT MOVE on any of the three, and that is the point of the +# marked subset rather than an accident of these spellings: reading +# the letter as an initial leaves it the middle-name word this +# baseline already had, so the whole diff is the report. Measured +# against the parent 46651750 on 2026-09-20, the same three names' +# parses are byte-identical before and after the bundle and only +# `ambiguities` moved. The lower-case spellings where the letter WAS +# the generation before -- 'rovira, i', 'john smith i jr', 'maier, amy +# i, jr.', 'josep de carod i rovira' -- do move fields under the same +# marked-subset reading, and no corpus carries one of them, so this +# rule is about the report alone. +# +# No rule at 1.4.0 and that is structural: `_ambiguities` is a v2 +# field and cannot diff against a v1 worker at all. +# +# Literal, three names. _MUST_NOT_MATCH carries the mixed-case +# spellings, where the writing decides the letter and the JOIN is what +# moved, and 'JUAN GARCIA Y LOPEZ', the unmarked letter in the same +# one-case position, which reports nothing. +name_regex = "^(?:JOSEP CAROD I ROVIRA|JOSEP LLUIS CAROD I ROVIRA|josep carod i rovira)$" +fields = ["_ambiguities"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective holding its part alone contributes an initial" +# 16 names, each one a part whose only word is the connective, +# so there is nothing there for it to join and it initials as any +# other name word does -- which is what `family_base` and `given_base` +# have said about the same token since 2.2 (rules.md#R2/#R3). This +# baseline drops the letter; the tree keeps it. Measured from the +# 2.1.0 wheel and from the tree on 2026-09-20: 'Juan de y' 'J.' -> +# 'J. y.'; 'johnny y' 'j.' -> 'j. y.'; 'Garcia y Lopez' 'G. L.' -> +# 'G. y. L.'; 'Juan y Garcia' 'J. G.' -> 'J. y. G.' and 'juan y +# garcia' the lowercase twin; 'John y Jane' 'J. J.' -> 'J. y. J.'; +# 'Carod y de Rovira i' 'C. R.' -> 'C. y. R.'; 'John e Smith' 'J. S.' +# -> 'J. e. S.', with 'John e Smith III', 'John e Smith, III' and +# 'Smith, John e, III, Jr' giving the same pair; 'john e jones, III' +# 'j. j.' -> 'j. e. j.'; 'Jose e Maria' 'J. M.' -> 'J. e. M.'; +# 'Lt.Gov. juan e garcia' 'j. g.' -> 'j. e. g.'; and 'Fritz Freiherr +# und von Bar' 'F.' -> 'F. u.', the German spelling of the same shape. +# BOTH SURFACES move together and to the same string on every one of +# them, the two views reading one mark (#528, decisions.md#R3). +# 'محمد و علي' is the readmission RESTORING 1.4.0 parity rather than +# breaking it: 'م. ع.' here, 'م. و. ع.' in the tree, which is the +# string the 1.4.0 wheel gives. The 1.4.0 ledger's +# `feat(#269) a recognized non-Latin connective contributes no +# initial` explained that name and is DELETED in the same commit as +# this rule, the name having stopped diffing there. +# +# ONE CAUSE, measured, not two. Every name here agreed with this +# baseline's wheel BEFORE the bundle -- the parent 46651750 gives the +# wheel's string on each -- so none of the diff belongs to #383/#479's +# one-case fork or to any older change; the whole of it is #461. +# +# Literal-anchored. The shape is "a part holding one connective and +# nothing else", which no regex over the raw string can state -- the +# reading is the parse's -- and the nearest writable shape, "a name +# carrying a lone cased letter", reaches every bare-initial name in +# the corpora. _MUST_NOT_MATCH carries the wall: 'Jon Dough and' (base +# 'Dough and', a connective with a name word beside it, keeping +# 'J. D.'), 'Juan Velasquez y Garcia' (the joined run, keeping +# 'J. V. G.') and 'Juan Garcia Lopez y'. +name_regex = "^(?:Carod y de Rovira i|Fritz Freiherr und von Bar|Garcia y Lopez|John e Smith|John e Smith III|John e Smith, III|John y Jane|Jose e Maria|Juan de y|Juan y Garcia|Lt\\.Gov\\. juan e garcia|Smith, John e, III, Jr|john e jones, III|johnny y|juan y garcia|محمد و علي)$" +fields = ["_initials"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective with a name word beside it stops contributing an initial" +# 13 names, the other direction of the same sentence: where the +# part holds a name word for the connective to join, the connective is +# doing a connective's work and contributes nothing. This baseline +# initialed it; the tree does not. 'John and Jane Smith' 'J. a. J. S.' +# -> 'J. J. S.'; 'Juan y Eva Garcia' 'J. y. E. G.' -> 'J. E. G.'; +# 'Jose e Maria Santos' 'J. e. M. S.' -> 'J. M. S.'; 'and Jon Dough' +# 'a. J. D.' -> 'J. D.' and 'the and Jon Dough' 't. a. J. D.' -> +# 'J. D.'; 'Rob And Beth Edmunds' 'R. A. B. E.' -> 'R. B. E.'; the two +# 'Dean of Chemistry Robert Johns' spellings 'D. o. C. R. J.' -> +# 'D. C. R. J.'; 'John and Jane Aznar y Lopez' 'J. a. J. A. L.' -> +# 'J. J. A. L.'; 'Mr. and Mrs. John and Jane Smith' 'J. a. J. S.' -> +# 'J. J. S.'; and the three Cyrillic spellings of 'Хосе И Мария +# Сантос' 'Х. И. М. С.' -> 'Х. М. С.', lower case and upper case +# alike. Both surfaces move together and to the same string. +# +# ONE CAUSE, measured. Each of these names agreed with this +# baseline's wheel before the bundle -- the parent 46651750 gives the +# wheel's string on every one, the Cyrillic three included, whose 'и' +# has been decided vocabulary since 2.0 and reaches no case fork. So +# none of the diff is #383/#479's and the whole of it is #461. At +# 1.4.0 these names diff too, but they diffed there BEFORE this +# bundle, on the 2.0.0 per-word GROUPING, and that ledger keeps them +# on `fix(initials-per-word) a connective run initials each word` +# with a dated paragraph naming both halves. +# +# CYRILLIC SPELLED OUT, not classed. A `\p{Cyrillic}`-style span +# would declare a script this rule is not about -- the sentence is +# about a PART, in any script -- and _SPAN_BEARING_RULES in +# tests/v2/test_ledger_guards.py records that this rule declares no +# span. +# +# Literal-anchored, for the GAIN rule's reason, and with the same +# wall: 'Jon Dough and', 'Juan Velasquez y Garcia' and 'Juan Garcia +# Lopez y' are _MUST_NOT_MATCH probes here too -- the criterion is the +# PART and never a word count, which is exactly what keeps those three +# fixed. +name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" +fields = ["_initials"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index d74d9755..8d7506d5 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -75,7 +75,18 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # the unambiguous PhD already ended the walk, so the clause gave # the MA up before this change as it does after it. What grew is # the CORPUS, and the run rendering is what the diff is about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Kenneth Clarke QC MP|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397: three names join the alternation and the rule +# does not change. 'Josep Lluis Carod i III' ('i, III' -> 'i III'), +# 'Josep Lluis Carod i V' ('i, V' -> 'i V') and 'Rovira, Josep Carod +# i Jr.' ('i, Jr.' -> 'i Jr.') are two-word generation runs this +# baseline renders with a comma and the tree renders with a space -- +# the whole of each one's diff here. #397 moves NOTHING on any of +# them: measured against the parent 46651750 all three parse +# byte-identically before and after this bundle, the link having a +# generation marker and not a name word to its right. What grew is +# the CORPUS -- the three arrived with #397's own case rows -- and +# the run rendering is what the diffs are about. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -373,6 +384,26 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # remains and this rule still explains it, one `_initials` move. # expected_since_2.0.0.toml's copy of this rule carries the account; # it is the same at all three baselines. +# +# 2026-09-20, #461: THE SURFACE FLIPPED AND THE RULE DID NOT. At +# this baseline the wheel gives facade 'J. G.' and core 'J. Y. G.' +# for 'JUAN Y GARCIA' (measured 2026-09-20 from the 2.2.0 wheel). +# Before this change only the CORE moved, 'J. Y. G.' -> 'J. G.', a +# connective contributing no initial; after it only the FACADE moves, +# 'J. G.' -> 'J. Y. G.', and the core agrees with the baseline word +# for word -- the 'Y' holds its part alone, so it initials again on +# both surfaces and the tree gives 'J. Y. G.' from each. `_initials` +# is one pseudo-field, so this rule still explains the name; what +# changed is WHICH HALF of the disagreement it is explaining, which +# is why the paragraph above reads backwards without this one. +# rules.md#R3, decisions.md#R3. +# +# The name stops diffing at 1.4.0 and at 2.3.0 under the same move, +# and the rule written for it in each of those ledgers goes: the +# 2.3.0 `fix(#383/#479) a bare capital connective in an all-upper +# name stops initialing` is deleted outright, and 1.4.0's +# `fix(#528) ...` keeps its other three names and drops this one from +# its alternation. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" fields = ["_initials"] orders = ["DEFAULT"] @@ -1349,3 +1380,181 @@ issue = "fix(#533) the maiden clause ends at the credential in a native-script n # larger diff from those baselines. name_regex = "^田中 太郎 旧姓 佐藤 MA$" fields = ["_ambiguities", "maiden", "suffix"] + +# --------------------------------------------------------------- +# #397 + #461: THE CATALAN LINK JOINS, AND A CONNECTIVE INITIALS +# WHERE IT JOINS NOTHING (2026-09-20). Four rules: the join's ROLES, +# the REPORT a one-case spelling makes instead of joining, and +# #461's two halves, which are one sentence read in its two +# directions and need two rules because a rule listing `_initials` +# may list nothing else (#484). +# +# LAST in the file, and that is narrow-first rather than a +# preference: the join rule declares {family, middle, suffix} and +# every rule above it that reaches one of its five names declares a +# strict subset, so an earlier position is an order-decided contest +# the run refuses (#382). +# +# rules.md#P3 states the join and its two boundaries; rules.md#R3 +# states what a connective contributes. decisions.md#P3 carries the +# count decision and the both-sides scope, decisions.md#R3 the +# criterion and its measured blast radius. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) the Catalan/Polish link joins two surnames" +# 'i' is connective vocabulary since #397, and a connective counts as +# a name word toward rules.md#P3's three-word carve-out whatever else +# the vocabulary says it is -- which matters here because 'i' is also +# the roman numeral. So a four-word name reaches the threshold and the +# link joins. This baseline read 'Josep Carod i Rovira' as middle +# 'Carod i' with family 'Rovira', and 'Carod i Rovira, Josep' -- where +# the link stands after a family comma -- as family 'Carod Rovira' +# with the link taken as a GENERATION in `suffix`, which is why the +# two move different field sets and are grouped here rather than +# split. The three longer spellings move with the first: 'Josep Carod +# i Rovira III' and 'Josep Lluis Carod i Rovira' the same +# {family, middle}, and 'Josep Carod i de Rovira' the same pair with +# the link joining PAST a particle (middle 'Carod i' -> 'Carod i de', +# family 'de Rovira' -> 'Rovira'). Five names, `fields` the union of +# their two shapes. +# +# All five entered this corpus with #397's own rules.md#P3 example +# lines and case rows; before that they were radar-tier names the +# contract did not answer for, which is why a rule appears at this +# baseline only now. +# +# Literal-anchored to the five. The shape is "a single letter between +# two name words", which reaches every corpus name carrying a bare +# initial and would pre-excuse the opposite reading -- #397's whole +# accepted cost is that some such letters must NOT join. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the wall: +# 'Josep i Rovira' (three words, under the carve-out), 'Carod i' and +# 'John Quincy Smith i' (the link with nothing to its right) and the +# two 'Josep Lluis Carod i ' spellings, whose trailing +# generation run is the fix(#436/#437) rule's at the head of this +# file. +name_regex = "^(?:Carod i Rovira, Josep|Josep Carod i Rovira|Josep Carod i Rovira III|Josep Carod i de Rovira|Josep Lluis Carod i Rovira)$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#397) the Catalan link reports a connective-or-initial in a one-case name" +# 'JOSEP CAROD I ROVIRA', 'JOSEP LLUIS CAROD I ROVIRA' and 'josep +# carod i rovira'. 'i' joined `conjunctions_ambiguous` with #397, so +# in a name written wholly in one case -- where the writing says +# nothing either way -- the letter reads as an INITIAL and the call is +# reported, exactly as 'e' has read since #383/#479. #527's mechanism, +# a new letter: rules.md#P3's marked subset. +# +# ROLES DO NOT MOVE on any of the three, and that is the point of the +# marked subset rather than an accident of these spellings: reading +# the letter as an initial leaves it the middle-name word this +# baseline already had, so the whole diff is the report. Measured +# against the parent 46651750 on 2026-09-20, the same three names' +# parses are byte-identical before and after the bundle and only +# `ambiguities` moved. The lower-case spellings where the letter WAS +# the generation before -- 'rovira, i', 'john smith i jr', 'maier, amy +# i, jr.', 'josep de carod i rovira' -- do move fields under the same +# marked-subset reading, and no corpus carries one of them, so this +# rule is about the report alone. +# +# No rule at 1.4.0 and that is structural: `_ambiguities` is a v2 +# field and cannot diff against a v1 worker at all. +# +# Literal, three names. _MUST_NOT_MATCH carries the mixed-case +# spellings, where the writing decides the letter and the JOIN is what +# moved, and 'JUAN GARCIA Y LOPEZ', the unmarked letter in the same +# one-case position, which reports nothing. +name_regex = "^(?:JOSEP CAROD I ROVIRA|JOSEP LLUIS CAROD I ROVIRA|josep carod i rovira)$" +fields = ["_ambiguities"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective holding its part alone contributes an initial" +# 17 names, each one a part whose only word is the connective, +# so there is nothing there for it to join and it initials as any +# other name word does -- which is what `family_base` and `given_base` +# have said about the same token since 2.2 (rules.md#R2/#R3). This +# baseline drops the letter; the tree keeps it. Measured from the +# 2.2.0 wheel and from the tree on 2026-09-20: 'Juan de y' 'J.' -> +# 'J. y.'; 'johnny y' 'j.' -> 'j. y.'; 'Garcia y Lopez' 'G. L.' -> +# 'G. y. L.'; 'Juan y Garcia' 'J. G.' -> 'J. y. G.' and 'juan y +# garcia' the lowercase twin; 'John y Jane' 'J. J.' -> 'J. y. J.'; +# 'Carod y de Rovira i' 'C. R.' -> 'C. y. R.'; 'John e Smith' 'J. S.' +# -> 'J. e. S.', with 'John e Smith III', 'John e Smith, III' and +# 'Smith, John e, III, Jr' giving the same pair; 'john e jones, III' +# 'j. j.' -> 'j. e. j.'; 'Jose e Maria' 'J. M.' -> 'J. e. M.'; +# 'Lt.Gov. juan e garcia' 'j. g.' -> 'j. e. g.'; and 'Fritz Freiherr +# und von Bar' 'F.' -> 'F. u.', the German spelling of the same shape. +# BOTH SURFACES move together and to the same string on every one of +# them, the two views reading one mark (#528, decisions.md#R3). +# 'محمد و علي' is the readmission RESTORING 1.4.0 parity rather than +# breaking it: 'م. ع.' here, 'م. و. ع.' in the tree, which is the +# string the 1.4.0 wheel gives. The 1.4.0 ledger's +# `feat(#269) a recognized non-Latin connective contributes no +# initial` explained that name and is DELETED in the same commit as +# this rule, the name having stopped diffing there. +# 'Juan y Garcia née Jones' is in this list at 2.2.0 and 2.3.0 and at +# no earlier baseline: below 2.2 the maiden clause moves ROLES on it, +# so `_initials` never enters its diff and `fix(#418)` explains what +# happened (compare.py's roles-identical guard, #484). +# +# ONE CAUSE, measured, not two. Every name here agreed with this +# baseline's wheel BEFORE the bundle -- the parent 46651750 gives the +# wheel's string on each -- so none of the diff belongs to #383/#479's +# one-case fork or to any older change; the whole of it is #461. +# +# Literal-anchored. The shape is "a part holding one connective and +# nothing else", which no regex over the raw string can state -- the +# reading is the parse's -- and the nearest writable shape, "a name +# carrying a lone cased letter", reaches every bare-initial name in +# the corpora. _MUST_NOT_MATCH carries the wall: 'Jon Dough and' (base +# 'Dough and', a connective with a name word beside it, keeping +# 'J. D.'), 'Juan Velasquez y Garcia' (the joined run, keeping +# 'J. V. G.') and 'Juan Garcia Lopez y'. +name_regex = "^(?:Carod y de Rovira i|Fritz Freiherr und von Bar|Garcia y Lopez|John e Smith|John e Smith III|John e Smith, III|John y Jane|Jose e Maria|Juan de y|Juan y Garcia|Juan y Garcia née Jones|Lt\\.Gov\\. juan e garcia|Smith, John e, III, Jr|john e jones, III|johnny y|juan y garcia|محمد و علي)$" +fields = ["_initials"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective with a name word beside it stops contributing an initial" +# 13 names, the other direction of the same sentence: where the +# part holds a name word for the connective to join, the connective is +# doing a connective's work and contributes nothing. This baseline +# initialed it; the tree does not. 'John and Jane Smith' 'J. a. J. S.' +# -> 'J. J. S.'; 'Juan y Eva Garcia' 'J. y. E. G.' -> 'J. E. G.'; +# 'Jose e Maria Santos' 'J. e. M. S.' -> 'J. M. S.'; 'and Jon Dough' +# 'a. J. D.' -> 'J. D.' and 'the and Jon Dough' 't. a. J. D.' -> +# 'J. D.'; 'Rob And Beth Edmunds' 'R. A. B. E.' -> 'R. B. E.'; the two +# 'Dean of Chemistry Robert Johns' spellings 'D. o. C. R. J.' -> +# 'D. C. R. J.'; 'John and Jane Aznar y Lopez' 'J. a. J. A. L.' -> +# 'J. J. A. L.'; 'Mr. and Mrs. John and Jane Smith' 'J. a. J. S.' -> +# 'J. J. S.'; and the three Cyrillic spellings of 'Хосе И Мария +# Сантос' 'Х. И. М. С.' -> 'Х. М. С.', lower case and upper case +# alike. Both surfaces move together and to the same string. +# +# ONE CAUSE, measured. Each of these names agreed with this +# baseline's wheel before the bundle -- the parent 46651750 gives the +# wheel's string on every one, the Cyrillic three included, whose 'и' +# has been decided vocabulary since 2.0 and reaches no case fork. So +# none of the diff is #383/#479's and the whole of it is #461. At +# 1.4.0 these names diff too, but they diffed there BEFORE this +# bundle, on the 2.0.0 per-word GROUPING, and that ledger keeps them +# on `fix(initials-per-word) a connective run initials each word` +# with a dated paragraph naming both halves. +# +# CYRILLIC SPELLED OUT, not classed. A `\p{Cyrillic}`-style span +# would declare a script this rule is not about -- the sentence is +# about a PART, in any script -- and _SPAN_BEARING_RULES in +# tests/v2/test_ledger_guards.py records that this rule declares no +# span. +# +# Literal-anchored, for the GAIN rule's reason, and with the same +# wall: 'Jon Dough and', 'Juan Velasquez y Garcia' and 'Juan Garcia +# Lopez y' are _MUST_NOT_MATCH probes here too -- the criterion is the +# PART and never a word count, which is exactly what keeps those three +# fixed. +name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" +fields = ["_initials"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.3.0.toml b/tools/differential/expected_since_2.3.0.toml index 4b3368b1..705f19cf 100644 --- a/tools/differential/expected_since_2.3.0.toml +++ b/tools/differential/expected_since_2.3.0.toml @@ -140,8 +140,31 @@ orders = ["DEFAULT"] # derived view with no report beside it to suppress it. # --------------------------------------------------------------------- -[[change]] -issue = "fix(#383/#479) a bare capital connective in an all-upper name stops initialing" +# `fix(#383/#479) a bare capital connective in an all-upper name +# stops initialing` STOOD HERE until 2026-09-20 and is DELETED, not +# declared dormant. It covered '^JUAN Y GARCIA$' on +# `fields = ["_initials"]`, saying that the unmarked 'Y' joins in a +# one-case name and so contributes no initial: 'J. Y. G.' -> 'J. G.'. +# #461 asks the question of the PART instead, and the 'Y' holds the +# middle part alone with no name word beside it to join, so it +# initials again -- measured 2026-09-20, the 2.3.0 wheel gives +# 'J. Y. G.' from both surfaces and so does the tree. The name stops +# diffing at this baseline and the rule reported EXPLAINED NOTHING. +# Its REASON is wrong now, not merely unreached, so a `dormant` +# declaration would assert something false and deletion is the +# ledger's convention (tools/differential/README.md). +# +# The name keeps a rule at 2.0.0, 2.1.0 and 2.2.0, where it still +# diffs and `fix(#462)` explains it -- with the SURFACE flipped, the +# facade moving now where the core moved before, which each of those +# ledgers records in a dated paragraph. The readmission is 1.4.0 +# parity RESTORED on this name, not a deviation: the 1.4.0 wheel gave +# 'J. Y. G.' all along. decisions.md#R3. +# +# The original comment is kept below, in full, because the rule it +# describes is the one #461 reversed and the argument is the +# provenance the release note cites: +# # 'JUAN Y GARCIA': 'y' is not marked as reading both ways, so in a # name written wholly in one case it joins rather than being vetoed # into an initial -- and rules.md#P3's three-word carve-out then @@ -170,9 +193,8 @@ issue = "fix(#383/#479) a bare capital connective in an all-upper name stops ini # whose middle word is an unmarked single-letter connective", which # reaches 'JUAN GARCIA Y LOPEZ' (roles, the first rule above) and # 'juan y garcia' (unchanged) the moment it is written as a regex. -name_regex = "^JUAN Y GARCIA$" -fields = ["_initials"] -orders = ["DEFAULT"] +# +# (end of the deleted rule's original comment) [[change]] issue = "fix(#289) a written case contrast decides a bare ambiguous acronym" @@ -673,3 +695,192 @@ issue = "fix(#533) the maiden clause ends at the credential in a native-script n # larger diff from those baselines. name_regex = "^田中 太郎 旧姓 佐藤 MA$" fields = ["_ambiguities", "maiden", "suffix"] + +# --------------------------------------------------------------- +# #397 + #461: THE CATALAN LINK JOINS, AND A CONNECTIVE INITIALS +# WHERE IT JOINS NOTHING (2026-09-20). Four rules: the join's ROLES, +# the REPORT a one-case spelling makes instead of joining, and +# #461's two halves, which are one sentence read in its two +# directions and need two rules because a rule listing `_initials` +# may list nothing else (#484). +# +# LAST in the file, and that is narrow-first rather than a +# preference: the join rule declares {family, middle, suffix} and +# every rule above it that reaches one of its five names declares a +# strict subset, so an earlier position is an order-decided contest +# the run refuses (#382). +# +# rules.md#P3 states the join and its two boundaries; rules.md#R3 +# states what a connective contributes. decisions.md#P3 carries the +# count decision and the both-sides scope, decisions.md#R3 the +# criterion and its measured blast radius. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) the Catalan/Polish link joins two surnames" +# 'i' is connective vocabulary since #397, and a connective counts as +# a name word toward rules.md#P3's three-word carve-out whatever else +# the vocabulary says it is -- which matters here because 'i' is also +# the roman numeral. So a four-word name reaches the threshold and the +# link joins. This baseline read 'Josep Carod i Rovira' as middle +# 'Carod i' with family 'Rovira', and 'Carod i Rovira, Josep' -- where +# the link stands after a family comma -- as family 'Carod Rovira' +# with the link taken as a GENERATION in `suffix`, which is why the +# two move different field sets and are grouped here rather than +# split. The three longer spellings move with the first: 'Josep Carod +# i Rovira III' and 'Josep Lluis Carod i Rovira' the same +# {family, middle}, and 'Josep Carod i de Rovira' the same pair with +# the link joining PAST a particle (middle 'Carod i' -> 'Carod i de', +# family 'de Rovira' -> 'Rovira'). Five names, `fields` the union of +# their two shapes. +# +# All five entered this corpus with #397's own rules.md#P3 example +# lines and case rows; before that they were radar-tier names the +# contract did not answer for, which is why a rule appears at this +# baseline only now. +# +# Literal-anchored to the five. The shape is "a single letter between +# two name words", which reaches every corpus name carrying a bare +# initial and would pre-excuse the opposite reading -- #397's whole +# accepted cost is that some such letters must NOT join. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the wall: +# 'Josep i Rovira' (three words, under the carve-out), 'Carod i' and +# 'John Quincy Smith i' (the link with nothing to its right) and the +# two 'Josep Lluis Carod i ' spellings, whose trailing +# generation run is the fix(#436/#437) rule's at the head of this +# file. +name_regex = "^(?:Carod i Rovira, Josep|Josep Carod i Rovira|Josep Carod i Rovira III|Josep Carod i de Rovira|Josep Lluis Carod i Rovira)$" +fields = ["family", "middle", "suffix"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#397) the Catalan link reports a connective-or-initial in a one-case name" +# 'JOSEP CAROD I ROVIRA', 'JOSEP LLUIS CAROD I ROVIRA' and 'josep +# carod i rovira'. 'i' joined `conjunctions_ambiguous` with #397, so +# in a name written wholly in one case -- where the writing says +# nothing either way -- the letter reads as an INITIAL and the call is +# reported, exactly as 'e' has read since #383/#479. #527's mechanism, +# a new letter: rules.md#P3's marked subset. +# +# ROLES DO NOT MOVE on any of the three, and that is the point of the +# marked subset rather than an accident of these spellings: reading +# the letter as an initial leaves it the middle-name word this +# baseline already had, so the whole diff is the report. Measured +# against the parent 46651750 on 2026-09-20, the same three names' +# parses are byte-identical before and after the bundle and only +# `ambiguities` moved. The lower-case spellings where the letter WAS +# the generation before -- 'rovira, i', 'john smith i jr', 'maier, amy +# i, jr.', 'josep de carod i rovira' -- do move fields under the same +# marked-subset reading, and no corpus carries one of them, so this +# rule is about the report alone. +# +# No rule at 1.4.0 and that is structural: `_ambiguities` is a v2 +# field and cannot diff against a v1 worker at all. +# +# Literal, three names. _MUST_NOT_MATCH carries the mixed-case +# spellings, where the writing decides the letter and the JOIN is what +# moved, and 'JUAN GARCIA Y LOPEZ', the unmarked letter in the same +# one-case position, which reports nothing. +name_regex = "^(?:JOSEP CAROD I ROVIRA|JOSEP LLUIS CAROD I ROVIRA|josep carod i rovira)$" +fields = ["_ambiguities"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective holding its part alone contributes an initial" +# 17 names, each one a part whose only word is the connective, +# so there is nothing there for it to join and it initials as any +# other name word does -- which is what `family_base` and `given_base` +# have said about the same token since 2.2 (rules.md#R2/#R3). This +# baseline drops the letter; the tree keeps it. Measured from the +# 2.3.0 wheel and from the tree on 2026-09-20: 'Juan de y' 'J.' -> +# 'J. y.'; 'johnny y' 'j.' -> 'j. y.'; 'Garcia y Lopez' 'G. L.' -> +# 'G. y. L.'; 'Juan y Garcia' 'J. G.' -> 'J. y. G.' and 'juan y +# garcia' the lowercase twin; 'John y Jane' 'J. J.' -> 'J. y. J.'; +# 'Carod y de Rovira i' 'C. R.' -> 'C. y. R.'; 'John e Smith' 'J. S.' +# -> 'J. e. S.', with 'John e Smith III', 'John e Smith, III' and +# 'Smith, John e, III, Jr' giving the same pair; 'john e jones, III' +# 'j. j.' -> 'j. e. j.'; 'Jose e Maria' 'J. M.' -> 'J. e. M.'; +# 'Lt.Gov. juan e garcia' 'j. g.' -> 'j. e. g.'; and 'Fritz Freiherr +# und von Bar' 'F.' -> 'F. u.', the German spelling of the same shape. +# BOTH SURFACES move together and to the same string on every one of +# them, the two views reading one mark (#528, decisions.md#R3). +# 'محمد و علي' is the readmission RESTORING 1.4.0 parity rather than +# breaking it: 'م. ع.' here, 'م. و. ع.' in the tree, which is the +# string the 1.4.0 wheel gives. The 1.4.0 ledger's +# `feat(#269) a recognized non-Latin connective contributes no +# initial` explained that name and is DELETED in the same commit as +# this rule, the name having stopped diffing there. +# 'Juan y Garcia née Jones' is in this list at 2.2.0 and 2.3.0 and at +# no earlier baseline: below 2.2 the maiden clause moves ROLES on it, +# so `_initials` never enters its diff and `fix(#418)` explains what +# happened (compare.py's roles-identical guard, #484). +# +# ONE CAUSE, measured, not two. Every name here agreed with this +# baseline's wheel BEFORE the bundle -- the parent 46651750 gives the +# wheel's string on each -- so none of the diff belongs to #383/#479's +# one-case fork or to any older change; the whole of it is #461. +# +# Literal-anchored. The shape is "a part holding one connective and +# nothing else", which no regex over the raw string can state -- the +# reading is the parse's -- and the nearest writable shape, "a name +# carrying a lone cased letter", reaches every bare-initial name in +# the corpora. _MUST_NOT_MATCH carries the wall: 'Jon Dough and' (base +# 'Dough and', a connective with a name word beside it, keeping +# 'J. D.'), 'Juan Velasquez y Garcia' (the joined run, keeping +# 'J. V. G.') and 'Juan Garcia Lopez y'. +name_regex = "^(?:Carod y de Rovira i|Fritz Freiherr und von Bar|Garcia y Lopez|John e Smith|John e Smith III|John e Smith, III|John y Jane|Jose e Maria|Juan de y|Juan y Garcia|Juan y Garcia née Jones|Lt\\.Gov\\. juan e garcia|Smith, John e, III, Jr|john e jones, III|johnny y|juan y garcia|محمد و علي)$" +fields = ["_initials"] +orders = ["DEFAULT"] + +[[change]] +issue = "fix(#461) a connective with a name word beside it stops contributing an initial" +# 22 names, the other direction of the same sentence: where the +# part holds a name word for the connective to join, the connective is +# doing a connective's work and contributes nothing. This baseline +# initialed it; the tree does not. 'John and Jane Smith' 'J. a. J. S.' +# -> 'J. J. S.'; 'Juan y Eva Garcia' 'J. y. E. G.' -> 'J. E. G.'; +# 'Jose e Maria Santos' 'J. e. M. S.' -> 'J. M. S.'; 'and Jon Dough' +# 'a. J. D.' -> 'J. D.' and 'the and Jon Dough' 't. a. J. D.' -> +# 'J. D.'; 'Rob And Beth Edmunds' 'R. A. B. E.' -> 'R. B. E.'; the two +# 'Dean of Chemistry Robert Johns' spellings 'D. o. C. R. J.' -> +# 'D. C. R. J.'; 'John and Jane Aznar y Lopez' 'J. a. J. A. L.' -> +# 'J. J. A. L.'; 'Mr. and Mrs. John and Jane Smith' 'J. a. J. S.' -> +# 'J. J. S.'; and the three Cyrillic spellings of 'Хосе И Мария +# Сантос' 'Х. И. М. С.' -> 'Х. М. С.', lower case and upper case +# alike. Both surfaces move together and to the same string. +# NINE of these names carry a rule at 2.3.0 and at no earlier +# baseline, and the reason is the roles-identical guard rather than +# the behavior: 'Dean of Chemistry', 'Duke of Edinburgh', 'Duke of +# Wellington', 'John & Jane', 'John of the Doe', 'Juan & Garcia', +# 'Juan and Garcia' and the two 'part1 of ...' strings all move +# `_ambiguities` against 2.0.0 through 2.2.0 -- #449's lone-name-word +# report, which shipped in 2.3 -- and a derived view enters a diff +# only where every role AND every ambiguity kind agrees (#484). Their +# initials move at those baselines too; nothing there can see it. +# 'Duke of Edinburgh' 'D. o. E.' -> 'D. E.', 'Juan and Garcia' +# 'J. a. G.' -> 'J. G.', 'Juan & Garcia' 'J. &. G.' -> 'J. G.'. +# +# ONE CAUSE, measured. Each of these names agreed with this +# baseline's wheel before the bundle -- the parent 46651750 gives the +# wheel's string on every one, the Cyrillic three included, whose 'и' +# has been decided vocabulary since 2.0 and reaches no case fork. So +# none of the diff is #383/#479's and the whole of it is #461. At +# 1.4.0 these names diff too, but they diffed there BEFORE this +# bundle, on the 2.0.0 per-word GROUPING, and that ledger keeps them +# on `fix(initials-per-word) a connective run initials each word` +# with a dated paragraph naming both halves. +# +# CYRILLIC SPELLED OUT, not classed. A `\p{Cyrillic}`-style span +# would declare a script this rule is not about -- the sentence is +# about a PART, in any script -- and _SPAN_BEARING_RULES in +# tests/v2/test_ledger_guards.py records that this rule declares no +# span. +# +# Literal-anchored, for the GAIN rule's reason, and with the same +# wall: 'Jon Dough and', 'Juan Velasquez y Garcia' and 'Juan Garcia +# Lopez y' are _MUST_NOT_MATCH probes here too -- the criterion is the +# PART and never a word count, which is exactly what keeps those three +# fixed. +name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry|Dean of Chemistry Robert Johns|Duke of Edinburgh|Duke of Wellington|John & Jane|John and Jane Aznar y Lopez|John and Jane Smith|John of the Doe|Jose e Maria Santos|Juan & Garcia|Juan and Garcia|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|part1 of The part2 of the part3 and part4|part1 of and The part2 of the part3 And part4|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" +fields = ["_initials"] +orders = ["DEFAULT"] From e540d4c50017254188f0f7a8c30d5d662a8ac4ef Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 07:26:08 -0700 Subject: [PATCH 05/11] docs(#397/#461): the rules, the decisions and the release notes decisions.md gains five dated bullets and edits none. Under P3, the count decision and the question it closes: a word's membership in another vocabulary does not discount it from a count gating a rule keyed on connectives, which is the question #418's entry left open. The both-sides scope is recorded with its measured alternatives, and the measurement corrects what the design predicted: the absent scope is separated from the shipped one only by rows this bundle added, while the WIDE scope -- the same condition without the generational-vocabulary test -- moves three names that were in the corpus already, 'Mr. & Mrs. John Smith' among them. The review that replaced a position test with a class test is recorded with the credential it stopped swallowing. The one-case half is stated honestly: the letter is in the marked subset, so a one-case name reports wherever a bare i stands among the name's own words, and in an all-lower name that reading moves fields, each such name reading as its all-caps twin already did. The Excluded block now lists one letter against two marked ones, recorded as a decision rather than left implicit, and P3's Open block is closed with it. Under R3, the clause as taken and the three candidate answers it rules out, both views reading one mark, the blast radius recomputed over the population as it stands and over the parent's, the two names it restores to 1.4.0 parity, the six frames it saves, and the no-parse paths that keep the older reading. Under R2, an AMENDMENT to the 2026-08-29 bullet that said none of four names could move: two of them moved and two did not, and the bullet says which half survived and why -- the criterion is the PART and never a word count, which is exactly what keeps the other two fixed. The backout that bullet records was right about the mechanism even though the clause it preserved is the thing that changed. Under R4, the behavior of the clause did not move and the grounding did: the cross-reference to R3's carve-out is cut and replaced by this rule's own reason. A sweep run to check that claim found a second one the design had missed -- capitalized(force=True) leaves a lower-case i alone on ten names where nothing else moved, where every release title-cased it, so the vocabulary addition carries an unrecorded 1.4.0 parity break on the forced call alone. A cross-reference under 3-0-reevaluations notes the marked subset now holds two letters, so the parked per-locale entry describes a set of seven single-letter conjunctions rather than six. mechanisms.md records the new marker beside its twin, CLOSES the open paragraph about a view overriding the record -- correcting the two default-vocabulary values it quoted, both of which moved -- and gains a field note: an invariant judged on the part a join PRODUCED is satisfied by whatever the join absorbed, so judge on the pre-join classes or against a parse with the rule switched off. customize.rst gains the Catalan/Polish removal recipe beside the Portuguese and Dutch ones, and the release log gains two Behavior Changes bullets -- plus three IN-PLACE corrections, since the unreleased 2.4.0 notes claim a name's initials move this cycle when after this change they do not move at all, and say the marked subset holds one letter. One fix that predates this bundle and was found sweeping P3: the rule said the v1 facade's initials() still reads a one-letter connective by vocabulary and shape, so `HumanName("john e smith").initials()` stays "j. s." for now. #528 landed and it is "j. e. s.". Co-Authored-By: Claude Opus 5 (1M context) --- docs/customize.rst | 17 ++++++++++++++--- docs/design/decisions.md | 30 ++++++++++++++++++++++++++++-- docs/design/mechanisms.md | 7 ++++--- docs/design/rules.md | 8 ++++---- docs/release_log.rst | 10 +++++++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 326f53dd..e5e5a24a 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -299,9 +299,10 @@ ambiguity is recorded and it becomes part of the surname — under any A single letter written against the name's own case is an initial and one written with it is the connective — but a name written wholly in one case, all upper or all lower, says nothing either way, and this is -the set that decides it there. ``e`` is the one entry shipped: a bare -``E`` initial is common where an ``e`` between two surnames is rare, and -``y`` runs the other way, so ``y`` joins even written as a bare capital. +the set that decides it there. ``e`` and ``i`` are the two entries +shipped: a bare ``E`` or ``I`` initial is common where those letters +between two surnames are rarer, and ``y`` runs the other way, so ``y`` +joins even written as a bare capital. .. doctest:: @@ -329,6 +330,16 @@ does in Spanish, take it out and the connective reading comes back: >>> Parser(lexicon=lex).parse("jose e maria santos").given 'jose e maria' +If your data is Catalan or Polish, where ``i`` links two surnames the +way ``y`` does in Spanish, take that one out instead and the link +joins in a one-case name too: + +.. doctest:: + + >>> lex = Lexicon.default().remove(conjunctions_ambiguous={"i"}) + >>> Parser(lexicon=lex).parse("josep carod i rovira").family + 'carod i rovira' + If your data is Dutch, where a bare single letter is an initial and never a connective, add the other one instead: diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 2357ac48..26e55f05 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -346,6 +346,18 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf - 2026-09-13 #383/#479 — AMENDS the 2026-08-27 #445 entry under M4 below, which calls `initial` "the `initial` shape tag": since this change the tag is no longer purely a shape claim, because a marked single-letter connective in a one-case name carries it without being initial-SHAPED. rules.md#M4's statement now says "read as an initial" for that reason, and M4's carve-out itself is unchanged — it still reads the tag classify recorded rather than a predicate of its own. +- 2026-09-20 #397 — DECIDED AND SHIPPED: `i` joins `conjunctions` and `conjunctions_ambiguous` together, which is what the 2026-09-13 entry above said would happen and which half of it the Open block below called contested. `parse("Josep Carod i Rovira")` gives family "Carod i Rovira" where every release from 1.4.0 through 2.3.0 gave middle "Carod i" with family "Rovira"; `parse("Josep Lluis Carod i Rovira")` gives middle "Lluis" with that same family; and `parse("Carod i Rovira, Josep")` gives family "Carod i Rovira" where they read family "Carod Rovira" and took the link into `suffix` as a generation marker (all three measured 2026-09-20 on the released 1.4.0, 2.0.0, 2.1.0, 2.2.0 and 2.3.0 wheels from a throwaway environment, this tree shadowing a wheel otherwise). + THE OFF SWITCH is `Lexicon.default().remove(conjunctions={"i"})`, and on the v1 surface deleting `i` from `C.conjunctions`; both restore all three readings above byte for byte, measured 2026-09-20. That is a PROPERTY rather than an anecdote, and it is this change's durable statement: a name's fields equal its reading with the letter out of the vocabulary UNLESS a link joined between two name words, or the two readings disagree about the letter being an initial. `tests/v2/test_properties.py::test_a_link_that_joins_nothing_changes_no_field` runs it over a 12,880-name grid times eight parsers and carries the recorded control — 2,360 failing parses at `c8550b64`, the commit the review was written against, and 0 here. + THE COUNT REACHES THE CLASS (Derek, 2026-09-20), which ANSWERS the C-ii question the 2026-08-21 #418 bullet above left open — "whether a word's membership in one vocabulary should suppress it from a count gating a rule keyed on another". It does not: a single-letter connective counts as a name word toward P3's three-word carve-out whatever else the vocabulary says it is, and `i` is also the roman numeral. The alternative was rejected on its own arithmetic rather than on taste — counting the link as the generation it also spells lets a connective raise the bar for ITS OWN join, so "Josep Carod i Rovira", the shortest linked name and the commonest one, would be the only shape that failed to link while the four-word spellings joined. The arithmetic runs the other way too, which is why the count is scoped to where the word is PLACED TO JOIN: a connective joining nothing would otherwise LOWER the bar for an unrelated one, and "Carod y Rovira i" is that shape — it counted the trailing generation and let the "y" join. It reads given "Carod", middle "y", family "Rovira", suffix "i" here, the same FIELDS the parent gave — its initials moved, but under R3's rule rather than this one. + ACCEPTED CONSEQUENCE OF REACHING THE CLASS, and the one row the design's reading tables show it touching: under `Lexicon.default().add(particles={"y"})`, `Juan Velasquez y Garcia` goes from given "Juan" / middle "Velasquez" / family "y Garcia" to given "Juan" / family "Velasquez y Garcia" — the custom lexicon now AGREES with the default one, which is the right direction — and its `family_base` becomes "Velasquez Garcia", a pre-existing R2 limit (`family_base` drops a particle wherever it stands, not only leading) surfaced rather than created here. Measured 2026-09-20 on this branch. + THE BOTH-SIDES SCOPE, AND THE REVIEW THAT FIXED IT. A connective that is also generational vocabulary joins only where a NAME WORD stands on each side of it — a word the rest of the parse reads as a name word rather than as a credential, the trailing numeral or a title, looked for past any run of connectives standing between. The FIRST implementation tested POSITION (an interior piece) rather than the neighbours' CLASS, and swallowed a credential: "Josep Lluis Carod i III" read family "Carod i III" where the "III" is the generation. An independent review found it. It is recorded here because the invariant that ought to have caught it cannot: INV1 inspects the part the join PRODUCED, where the absorbed credential is itself the name word standing on the right, so that name SATISFIES it — the general lesson is a field note in mechanisms.md of this date, and the instrument is `tests/v2/test_properties.py::test_a_trailing_credential_never_joins_into_a_name_part`, which asks the off-switch parse instead and carries its own control (960 failing parses at `c8550b64`, 0 here). + THE SCOPE'S ALTERNATIVES, MEASURED 2026-09-20 rather than argued, by building each alternative from this tree and sweeping the 1558 non-empty corpus-union-cases names under eight configurations (default, `lenient_comma_suffixes=False`, both family-first orders, and the four locale packs). ABSENT — no both-sides condition at all — differs on five names, and every one of them is a row this bundle added: "John Quincy Smith i" would read family "Smith i", "Josep Lluis Carod i III" family "Carod i III", "Josep Lluis Carod i V" family "Carod i V", "Carod y de Rovira i" given "Carod y de" with family "Rovira i", and "Rovira, Josep Carod i Jr." middle "Carod i Jr.". Nothing that stood in the corpus at the parent separates absent from narrow, so those rows ARE the evidence and pruning one retires it. WIDE — the same condition with the generational-vocabulary test dropped, so it reaches every single-character connective — differs on five as well, and THREE of those were in the corpus already: "Mr. & Mrs. John Smith" would read title "Mr." with given "&" and middle "Mrs. John", "1 & 2, 3 4 5, Mr." would lose "2" out of the family, and "Jane née Jr y Jones" would read middle "née Jr y" with family "Jones". So the class test is not decoration — widening the rule past the link-and-generation class breaks "Mr. & Mrs." — and narrow is both what the design asked for and what measurement supports. (A "0 differences over 1545 names" figure was measured for this scope before the review supplied the class test and before those rows existed; it is superseded here.) + ACCEPTED, THE TWO-WORD SHAPE: "Carod i" keeps given "Carod" with suffix "i", and "Josep Carod i" keeps family "Carod" with suffix "i", because a connective with nothing to its right is connecting nothing. 1.4.0 read the first of those the same way. The ledger rule that called this NOT WANTED is rewritten in place rather than deleted — it still explains the name, and the standing is decided now rather than unwanted. + ACCEPTED, A CONNECTIVE RUN: "Henry i of England" reads given "Henry i of England" with a `given-or-family` report, where the parent read given "Henry" with family "i of England". The "i" and the "of" merge as a run and the run joins the whole name into one piece — the same thing "Duke of Edinburgh" already did. + ACCEPTED, THE PARTICLE ON THE RIGHT. A particle is a name word for the both-sides test, so the link joins across one: "Josep de Carod i Rovira" reads family "de Carod i Rovira", which is the Catalan reading, while "Josep Carod i de Rovira" reads middle "Carod i de" with family "Rovira" where the parent read middle "Carod i" with family "de Rovira" — the join takes the particle as its right-hand neighbour and the particle chain then follows the joined piece instead of the tail word. Recorded rather than repaired: which side a particle standing between a link and a surname belongs to is P2's question and not this one. + ACCEPTED, AND THE HONEST STATEMENT OF THE ONE-CASE HALF, written out because this branch's own first commit message overclaimed that one-case names keep their reading. `i` is in the MARKED subset, so in a name written wholly in one case it reads as an INITIAL and reports `conjunction-or-initial`, exactly as `e` has since #383/#479. For "JOSEP CAROD I ROVIRA" and "josep carod i rovira" the fields are indeed unchanged and only the report is new. But a one-case name reports wherever a bare `i`/`I` stands among the NAME'S OWN WORDS, which is a great many more names than the Catalan ones — "JOHN I SMITH" and "john i smith" one report each, "JOHN SMITH I" and "john smith i" two, "HENRY I" and "henry i" three. A letter inside a maiden clause is read by the clause's rules and stays silent, exactly as `e` does ("JANE DOE NEE I JONES" reports nothing), which is the own-words scope the 2026-09-13 #383/#479 entry above defines. And in ALL-LOWER names the initial reading MOVES FIELDS wherever the parent read the lower-case letter as the generation. Each such name now reads as its ALL-CAPS twin already did: "rovira, i" gives given "i" where it gave suffix "i" ("ROVIRA, I" already gave given "I"); "john smith i jr" gives middle "smith" with family "i" where it gave family "smith" with suffix "i jr" ("JOHN SMITH I JR" already did); "maier, amy i, jr." gives middle "i" with suffix "jr." where it gave suffix "i, jr." (the corpus name "Maier, Amy I, Jr." reads middle "I" and does not move); and "josep de carod i rovira" gives family "de carod i rovira" where it gave middle "de carod i" with family "rovira". None of those four is a corpus name; all are measured 2026-09-20 on this tree and against the parent's. + NO CHANGE TO THE EXCLUDED BLOCK, recorded as a decision rather than left implicit (Derek, 2026-09-20). It now lists "y" against TWO marked letters rather than one, and that is still right: the argument that put "y" outside is about "y" — the commonest Hispanic compound and this library's oldest fixture — while "i" matches "e" exactly, a bare I initial being as common as a bare E. + - Provenance: the single-letter-connective guard is v1's fix for Google Code issue 11 ("john e smith", 2013, commit 33676c9) — the "#11" citations that circulated pointed at a GitHub accident, not the real source. Recorded so the archaeology stays done. Excluded (Lexicon.conjunctions_ambiguous, the marked half of nameparser/config/conjunctions.py — an entry here reads as an INITIAL in a name written wholly in one case): @@ -355,8 +367,7 @@ Excluded (Lexicon.conjunctions_ambiguous, the marked half of nameparser/config/c - `و` and any other caseless letter — inert rather than harmful, and excluded so nobody adds one believing it does something: the fork requires a token whose `upper()` and `lower()` differ, so a caseless member is never consulted (2026-09-13). - Multi-letter entries (`and`, `та`, `και`) and `&` — the fork tests a single CHARACTER, so an entry longer than one letter is silently inert. `nameparser/config/conjunctions.py` asserts against it at import (2026-09-13). -Open (Lexicon.conjunctions — contested membership; the issue is canonical): -[#397](https://github.com/derek73/python-nameparser/issues/397) whether Catalan `i` belongs in the conjunctions at all. Its membership in `conjunctions_ambiguous` is NOT the open half — that is decided, and it ships in the subset in the same change that adds it to the base set, a bare I initial being as common as a bare E. +Open (Lexicon.conjunctions — contested membership; the issue is canonical): none. The block held one entry, [#397](https://github.com/derek73/python-nameparser/issues/397) — whether Catalan `i` belongs in the conjunctions at all — answered 2026-09-20 by the bullet of that date above: it does, and it joins `conjunctions_ambiguous` in the same change, which was never the contested half. The heading stays with nothing under it on purpose, the same reason an empty ledger roster section is a statement and a missing one is nobody having looked. ### given-name-titles — deliberately unvalidated @@ -1228,6 +1239,9 @@ Declined: - 2026-08-29 #461 — TRIED AND BACKED OUT; the question is back on the issue. The unjoined mark readmits the words of an all-particle part to `initials()`, and #461 narrowed that readmission to the `particle` tag alone, on the authority of R3's "A CONJUNCTION never initials, so a base that is one contributes nothing even then". It was reverted before merge, in the same PR. What the narrowing got wrong is best said in terms of the MARK rather than of the fields. The mark is a statement about a whole PART — none of its words is doing the work its tag names — and #461 honored that statement for some of the part's words while keeping one of them out. Under `Lexicon.default().add(particles={'y'})`, `parse("Juan de y")` is such a part: `family_base` is `de y`, both words carry the mark, and #461 initialed `J. d.`, admitting the `de` as the name word the mark makes it and refusing the `y`. R2's reasoning does not split that way. A particle with nothing left to join is not acting as a particle, which is the whole reason the mark exists; a conjunction inside that same part has nothing left to join either, for the same reason and at the same moment, so it is a name word of the part and initials with the rest. Scope that carefully, because its general form is far wider than the claim and would be wrong: this is about a part the mark has ALREADY turned into name words, not about any conjunction that happens to join nothing. Outside such a part the skip stands whatever the conjunction is or is not joining, and none of those moved or may move — `parse("Juan y Garcia")` initials `J. G.` with the `y` a middle name word by P3, `parse("Juan Velasquez y Garcia")` initials `J. V. G.` over base `Velasquez y Garcia`, `parse("Jon Dough and")` initials `J. D.` over base `Dough and`. Base and initials differ in every one of those, legitimately, and R3's own `"Juan de y" → initials="J."` line is a fourth, its `family_base` being `y` under the default vocabulary. So "the base holds a word the initials do not" is NOT the criterion and was not the finding; the mark is. What is therefore in question is R3's "even then" clause, which carries the carve-out into the one part where the joining has stopped — not the code that failed to implement it. The clause is left standing and #461 now asks whether it belongs there; nothing else in R3 moved, and the paragraph #461 added under it is gone. R4's conjunction sentence rests on that clause by name ("the carve-out R3 states for initials") and is untouched here, so whoever settles #461 settles R4's cross-reference with it. Note what the backout cannot record: a `deviates:` marker hangs on an example LINE and this shape has none to hang on — `particles` and `conjunctions` are disjoint in the default vocabulary and in every locale pack, so no input string the doc runner parses reaches it. rules.md itself therefore carries no trace, and this bullet with mechanisms.md#RENDER-HONORS-THE-PARSE is where the gap is written down. It is not the only gap under R3's conjunction sentence, and the other is older, wider and markable: "A CONJUNCTION never initials" is unqualified, while a conjunction in the GIVEN group has always initialed — `parse("John and Jane Smith")` gives `J. a. J. S.` and `parse("Duke of Edinburgh")` `D. o. E.`, 25 of the 1094 corpus names, all reachable from the default vocabulary. Recorded here because it was found here; it is not #461's to fix. RESOLVED into the normative document on 2026-08-29, in the review of PR #463: rules.md#R3 now scopes its carve-out to the middle and base family words and declares the given group unsettled, in prose and WITHOUT a `deviates:` marker. The marker was considered and rejected on its own definition — it states the INTENDED value, and there is none to state. R3 counts name words while P3 makes a connective and its neighbours ONE name word, so `John and Jane Smith` has four candidate answers and no entry anywhere picks one: `J. a. J. S.` today, `J. J. S.` reading the carve-out as written, `J. S.` reading P3's join as a single name word, and `J a J. S.` at 1.4.0 (measured on the released wheel, which joins the run rather than skipping the conjunction). Marking it would put an invented value in a normative document and hold the parser to it, since the runner asserts the today-value strictly. The marker's `#N` slot has no owner either — the sentence above disclaims #461 for this gap, and filing its own issue is a maintainer's call rather than a docs commit's. What "markable" meant two sentences up is the MECHANICAL property that an input string exists to hang a marker on, unlike the #461 shape where none does; it was never a claim that the intended value is known. Neither direction is visible to the gate: 0 of the 1094 corpus names move either way, and the counts hold at 229 / 194 / 102 with 0 unexplained at 1.4.0 / 2.0.0 / 2.1.0. Both views of the contested token are pinned rather than left to prose — `test_repair_keeps_a_conjunction_lowercase_in_a_particle_part` holds R4's ungated conjunct, gating which had passed the entire suite until #461's test existed, and `test_initials_readmits_a_conjunction_in_a_particle_part` holds today's initials answer the way a `deviates:` marker would, so re-deciding #461 fails the suite until that pin moves with it. +- 2026-09-20 #461 — AMENDS the 2026-08-29 bullet above, which said of four readings that "none of those moved or may move". Two of the four moved and two did not, and the half that survived is the half the criterion was always about. WHAT MOVED: `parse("Juan y Garcia")` initials "J. y. G." where that bullet recorded "J. G.", and R3's own `"Juan de y"` example line reads "J. y." where it read "J.". WHAT DID NOT: `parse("Juan Velasquez y Garcia")` is still "J. V. G." over base "Velasquez y Garcia", and `parse("Jon Dough and")` is still "J. D." over base "Dough and" (all four measured 2026-09-20 on this branch). The criterion that separates them is the one the bullet above named, and it is still not a word count: the question is asked of the WHOLE PART, and a part holding another name word for the connective to join — the part's working particles set aside — is a part where the connective is doing a connective's work and contributes nothing. "Velasquez y Garcia" and "Dough and" each hold such a word, so the skip stands there; the family "y", the family "de y" and the middle "y" of "Juan y Garcia" hold none, so the connective is its part's own name word and initials with it. + So that bullet was RIGHT ABOUT THE MECHANISM and wrong only about the clause it preserved. Its finding — that a mark honored for some of a part's words and not for one of them is honoring nothing — is exactly what rules.md#R3 now states, for all three groups at once, and it is why the fact is computed ONCE over the whole part rather than re-derived per word by each view. What changed is the clause it left standing: R3's "A CONJUNCTION never initials, so a base that is one contributes nothing even then" is conditional now, and "none of those may move" was the reading of the day rather than a decision. decisions.md#R3's 2026-09-20 bullet carries the rule as taken. The pin that bullet describes did its job: `test_initials_readmits_a_conjunction_in_a_particle_part` was written to fail when #461 was re-decided, and it failed. + Declined: - 2026-08-18 — the GROUPING half of #404: a particle run that joins nothing does not chain, so "Jong van der" would split into middle 'van' plus family 'der'. Measured and rejected, though NOT for the reason first recorded here. The first draft said the split makes the family "no longer all-particle so the base fix stops firing" — false, and `der` and `la` are both shipped particles, so a family of either IS all-particle and the rule fires on it ("Juan Smith der" gives base 'der'). What the split actually costs is the SCOPE of the base and a stray particle relocated: grouping can decline to merge but cannot keep the words apart, because roles re-assemble them and two adjacent same-role pieces are one part at the field level. "Juan Smith van der" becomes middle 'Smith van', family 'der' — a base of 'der' rather than 'van der', and a middle name nobody wrote. Keeping the run whole in one part is what gives the base its full extent. The split reading needs the leftover distribution to know these are separate units, which is mechanisms.md#UNIT-PARTITION's problem. @@ -1255,6 +1269,13 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. DIFFERENTIAL. The facade's `_initials` is compared at every baseline, 1.4.0 included, and the core's from 2.0.0; the pseudo-field enters a name's diff only where every role and every ambiguity kind agrees. So at 1.4.0 FOUR of the six take a new rule (`fix(#528) the facade's initials follow the parse's connective tags`) and two do not — `jose e maria santos` and `JUAN GARCIA Y LOPEZ` move roles against that baseline, and #383/#479's role rule explains them. At 2.0.0 through 2.2.0 nothing new: the e-names' `_ambiguities` diff keeps `_initials` out of their diff, and `JUAN Y GARCIA`'s single `_initials` row survives with the facade half of its two causes closed, which `fix(#462)`'s dated paragraph in `expected_since_2.0.0.toml` now says. At 2.3.0 the existing rule stands and both surfaces move together. All five gates re-run at 0 unexplained on 2026-09-13. +- 2026-09-20 #461 — DONE, and the clause is settled for every group at once. THE RULE AS TAKEN: a connective contributes nothing where it is JOINING — a part holding another name word for it to join, the part's working particles set aside — and where its part holds nothing else it is joining nothing, so it initials like any other name word and agrees with the base. That picks ONE of the four candidate answers the 2026-08-29 bullet under `decisions.md#R2` listed for "John and Jane Smith" and makes the other three unreachable: `parse("John and Jane Smith").initials()` is "J. J. S.", not the "J. a. J. S." of 2.0 through 2.3, not P3's one-name-word "J. S.", and not 1.4.0's run-together "J a J. S.". THE CRITERION IS NEVER A WORD COUNT, which is what keeps `parse("Jon Dough and")` at "J. D." over base "Dough and" and `parse("Juan Velasquez y Garcia")` at "J. V. G." — a part of two words where one is the connective is still a part where it joins, and a part of three is no different. + BOTH VIEWS, ONE MARK. The fact is about a PART, so it is decided once where the parts are settled and recorded on the token (`UNJOINED_CONJUNCTION_TAG`, mechanisms.md#MARK-DONT-STRIP), and the core view and the v1 facade both read it. That is the direct answer to what the first #461 attempt found: a mark honored for some of a part's words and not for one of them is honoring nothing. A SEPARATE marker rather than a widening of `UNJOINED_TAG`, because that one is also read by `family_particles`/`family_base` and by `_cap_word`, and none of those three may move. `Parser.revise` recomputes both marks, the same reason `_remarked` recomputes the first. + BLAST RADIUS, MEASURED 2026-09-20 against the parent commit `46651750` over the deduped corpus-union-cases set under eight configurations, both processes asserting their own `nameparser.__file__`. Over the population AS IT STANDS — 1558 non-empty names, grown by this bundle's own rows — the default order moves 6 names' roles, 3 more names' reports with the roles fixed, and 50 names' `initials()`, 19 gaining a letter and 31 losing one; `HumanName.initials()` moves on THE SAME 50 names with THE SAME values. Family-first moves 5 / 3 / 28 and `FAMILY_FIRST_GIVEN_LAST` 6 / 3 / 58; the 35 names that move their initials under given-last and not under the default order are mostly the ones whose joined run lands in the GIVEN group there, `Juan Velasquez y Garcia` going from "V. y. G. J." to "V. G. J." — family-first now agreeing with given-first on it. Restricted to the 1544 names that stood in the corpus at the parent — the comparable population, since the rows this bundle added are themselves movers — the same sweep reads 2 / 3 / 46 at the default order, 18 gaining and 28 losing, with 1 / 3 / 24 family-first and 2 / 3 / 52 given-last. RECOMPUTE by parsing every name of the deduped `tools/differential/corpus*.jsonl` glob plus `tests/v2/cases.py`'s texts under those eight configurations in two processes, one on this tree and one on the parent's, and diffing the seven role fields, the ambiguity kinds and `initials()` on both surfaces. The gate cannot answer this on its own: its `_initials` pseudo-field sees only names whose roles stayed put, so a count taken from the classified summary is a FLOOR. + TWO OF THE GAINS ARE 1.4.0 PARITY RESTORED, and that is what retires two ledger rules: `JUAN Y GARCIA` gives "J. Y. G." and `محمد و علي` gives "م. و. ع.", which is what 1.4.0 gave for both (measured on the released wheel 2026-09-20; 2.0.0 through 2.3.0 gave "J. G." and "م. ع." on the facade for the reasons the 2026-09-13 #383/#479 entry under P3 records). The readmission is a parity RESTORATION on those two names rather than a deviation, and the given-group half in the other direction is the deviation — see rules.md#R3's Accepted clause, which states both. + THE FRAME BUDGET WENT DOWN: `tools/perf/call_count.py` reads `py3.11 parse=406.00 facade=443.00` where the parent read `412.00 / 449.00`. The two marks now come off ONE walk over the roles instead of two generator expressions, and on 3.11 a genexp costs a frame for its creation and one per resume, so the reference name's two non-empty parts pay exactly six. `_CALL_BASELINE[(3,11)]` is `{"parse": 410, "facade": 447}` with `_BAND = 0.02`, so the bands are [401.8, 418.2] and [438.06, 455.94] and both figures land inside: NO baseline edit and no `decisions.md#parse-cost` decision is owed. PEP 709 inlined comprehensions and not generator expressions, so 3.12 through 3.15 should drop by the same six. + THE NO-PARSE PATHS KEEP THE OLDER READING, the same degradation the 2026-09-13 #528 bullet above accepted and for the same reason: a token the parse never saw carries no mark, so the view falls back to the vocabulary. Measured 2026-09-20, `HumanName("JUAN GARCIA Y LOPEZ").initials()` is "J. G. L." live and "J. G. Y. L." after `pickle`, `copy.copy` or `copy.deepcopy`; a name built from keyword fields carries no tags either, and so does one whose `*_list` property a subclass overrides. The remedy is the v1 one — assign `full_name` again — since the facade has no `revise`. + ### R4 — case repair reads the unjoined mark - 2026-08-29 #407 — DECIDED: case repair consults the mark #404 introduced, and consults it for the WHOLE PART rather than for a particle standing alone. Until now three views read it and the fourth did not, which is how `parse("ANH DO")` came to report family_base 'DO' and initials 'A. D.' while `capitalized()` returned 'Anh do' — the same word treated as an ordinary name word by three readers and as a particle by the fourth. The scope question is the only real choice here, and standing-alone loses on the first Vietnamese name that tests it: `anh van do` has family `van do`, two particle-vocabulary words and neither of them alone, so a rule keyed on aloneness would capitalize `Anh Do` and leave `Anh van do`, making the same surname read differently for having a given-name word in front of it. rules.md#R2 states the test as "Every word", not "standing alone", and this is the case that shows why it is stated that way. One predicate, four views. @@ -1276,6 +1297,10 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-08-29 — WHY THE BOUNDARY WENT UNNOTICED UNTIL #407, which is where a future reader should look for it. For an ALL-PARTICLE part the other three tag-driven views give the same answer through `replace()` and `revise()` alike: measured over `de la`, `van der`, `do`, `de` and `van de la`, all five agree on `family_particles=''`, on a `family_base` holding the whole part, and on initials from every word. They converge because an UNTAGGED part and a MARKED all-particle part reach the same place by different routes — untagged, no word is recognized as a particle; marked, none is ACTING as one — and all three views only ask which words are particles. Case repair is the one view that asks a second question, since it must also decide whether to lowercase, so it is where the two routes first come apart. The mirror case confirms the reading: on a MIXED part the convergence is the other way round — `de la vega` and `van der berg` diverge in all three views between `replace()` and `revise()` (`replace()` reports particles `''` and base `'de la vega'` where `revise()` reports `'de la'` and `'vega'`) and AGREE on case repair, R4's all-particle clause not reaching them. So before #407 the distinction was invisible on exactly the shape the clause is about, and visible only on shapes the clause does not govern. +- 2026-09-20 #461/#397 — THE BEHAVIOR OF THE CLAUSE DID NOT MOVE, THE GROUNDING DID. R4's sentence rested on R3 by name ("the carve-out R3 states for initials"), and R3's carve-out is conditional now while R4's is not, so the cross-reference is CUT and replaced by this rule's own reason: a connective that initials because it joins nothing is still not written the way a name is written. Measured 2026-09-20, plain and forced, over `juan y`, `john and jane smith`, `duke of edinburgh`, `juan de y`, `juan y garcia`, `JUAN Y GARCIA`, `josep carod i rovira`, `JOSEP CAROD I ROVIRA` and — under `add(particles={"y"})` — `Anh y Van`: all nine are byte-identical to the same call at the parent, and `tests/test_capitalization.py` is green. + WHAT THE PLAIN CALL DOES, stated over a population rather than over nine names, because "capitalized() does not move" was drafted for this bullet and is false as written. Swept 2026-09-20 over the 1558 non-empty corpus-union-cases names under eight configurations against the parent commit, `capitalized()` moves on SIX distinct names and every one of them is a name whose ROLES moved under the join — it renders different fields rather than treating a word differently. The correct statement is therefore: where only `initials()` moves, `capitalized()` is byte-identical; where the roles move, it follows them. + WHAT THE FORCED CALL DOES, and this one IS a word treated differently — an unrecorded 1.4.0 parity break that belongs to #397's vocabulary rather than to #461's rule. `capitalized(force=True)` moves on TEN further names in that sweep whose roles, initials and plain repair all stayed put, and every one is a lower-case `i` that repair used to title-case: `parse("Carod i").capitalized(force=True)` gives "Carod i" where 1.4.0 and 2.3.0 both gave "Carod I", and so do "Carod i Rovira", "Josep i Rovira", "Josep Carod i", "Lluis Carod i", "John Quincy Smith i", "Josep Lluis Carod i III", "Josep Lluis Carod i V", "Carod y de Rovira i" and "Rovira, Josep Carod i Jr.". The cause is this rule working as written: the letter is connective vocabulary now, and a connective keeps its lower case wherever it stands. Only the FORCED call witnesses it — R5's gate refuses a mixed-case name before any of this is consulted — and the differential cannot see it at all, case being no compared surface (the 2026-08-29 bullets above settle why). Recorded rather than repaired, and named here because the sweep that found it was run for a claim about the PLAIN call. + ### R5 — the case-repair gate - 2026-08-29 (#407 arc) — EXTRACTION, not a decision: the parser is untouched. The two halves of R5 have separate provenance, and conflating them is easy enough that the first draft of this entry did. The REFUSAL — repair skips any name already carrying more than one case — is older than the git history: `git log -S "name == name.upper() or name == name.lower()" --reverse` bottoms out at 45a1539 (2011-02-03), the initial import from svn, where `capitalize()` already opens with that guard and a bare `return`. (A path-filtered search answers 280895b instead, the same-day commit that moved the module into `nameparser/`; the code did not change there.) The OVERRIDE is bf1e0a5, 2016-06-02, which did not add the refusal but wrapped it — `if not (name == name.upper() or ...)` became `if not force and not (...)` — and shipped in 0.4.0 (June 2, 2016; its own release-log line is docs/release_log.rst under that heading). So R5's statement as a whole holds from 0.4.0 on. rules.md had never said any of it, though bf1e0a5's diff shows the API docstring already did: "It will not adjust the case of names entered in mixed case" was there before that commit edited around it. The gap was rules.md's alone. Evidence, measured on the released 1.4.0 wheel — the last v1 release and one of the differential baselines, so a natural thing to measure against and not a release that introduced anything here — and re-measured on this branch today, facade and core agreeing: `HumanName('Shirley Maclaine').capitalize()` leaves `'Shirley Maclaine'` — mixed case, wrong, and kept — while the same name under `force=True` gives `'Shirley MacLaine'`; `HumanName('Juan McDonald').capitalize()` leaves `'Juan McDonald'`. rules.md's own preamble classifies behavior in this position as "pinned-but-undocumented — an extraction gap to close, not a specification", which is why the fix lands in the document rather than in `_render.py`. @@ -1376,3 +1401,4 @@ Promoted 2026-08-15 from session memory (Derek's 2026-07-30 ask; promotion appro - (A) 2026-08-18 — LOCALE CONTROL OF P3's SINGLE-LETTER CARVE-OUT, and the sharper requirement it puts on the entry above: a pack would have to carry per-word POLICY, not just membership. Which single letters a tradition wants joined differs by language — Spanish writes "y" and would want "e" read as an initial, Portuguese wants the reverse, Dutch would want every single letter read as an initial. One global rule serves all three today: a single ALPHABETIC character in a name of fewer than four pieces stays a name word, and a bare Latin capital never joins (#383). It is v1's Google Code issue 11 fix ("john e smith") generalized to a vocabulary v1 did not have — the set now holds six single-letter conjunctions (y, e, и, і, й, و), and the carve-out reaches all of them because it counts characters, while the capital veto reaches only the Latin ones because it tests a Latin shape. That asymmetry is the visible seam and it is why #383 is open. A pack that could only ADD or REPLACE vocabulary could not express any of it; the join threshold and the initial veto would have to be per-word, per-pack values. Measured while documenting P3: "&" is single-character but not a letter, so it joins at any length ("Juan & Garcia" is one part while "Juan y Garcia" is three). - (A) 2026-09-13 — SHOULD A LOCALE PACK BE ABLE TO REMOVE BASE VOCABULARY? `Locale.lexicon` is UNIONED onto the base and never removes — the field's own comment says so — which makes `conjunctions_ambiguous` (decisions.md#P3, 2026-09-13) expressible by a pack in ONE DIRECTION ONLY: an `nl` pack can add "y", and a `pt` pack cannot remove "e", so a Portuguese caller has to write `Parser(lexicon=Lexicon.default().remove(conjunctions_ambiguous={"e"}))` by hand. Accepted for 2.x and parked here rather than filed, because 3.0 is when a wider set of locale packs gets designed and this is part of that work (Derek, 2026-09-13). It is the 2026-08-17 PARTICLE VOCABULARY AS LOCALE PACKS bullet's "overrides rather than additions" requirement meeting a second tenant, which is what makes it a shape question rather than one pack's inconvenience. - (A) 2026-09-13 — THE DIFFERENTIAL HARNESS CANNOT SEE A LOCALE PACK'S EFFECT AT ALL. `compare.py` builds one `Parser` per name ORDER and nothing else, and a `Case.locale` row projects into the corpora as a bare name string — no corpus row carries a locale key, verified over every `tools/differential/corpus*.jsonl` file on 2026-09-13 — so every corpus name is parsed under the DEFAULT `Lexicon` on both sides. A pack's effect on a name can therefore never diff, at any baseline. That is fine while packs only ADD vocabulary the default already carries most of, and stops being fine the moment the bullet above is answered yes. The shape of the fix is a `locale` carried per corpus ENTRY the way `shape` already carries an order, resolved at the baselines that ship the pack and skipped below them. Parked with the bullet above rather than filed, for the same reason: the two are one piece of work (Derek, 2026-09-13). +- (A) 2026-09-20 #397 — A SECOND INSTANCE OF THE SAME PER-LOCALE QUESTION, cross-referenced rather than parked again. Catalan/Polish `i` joins `conjunctions_ambiguous` beside `e`, so the marked subset now carries two letters and the 2026-08-18 bullet above — which counted six single-letter conjunctions and named the per-word-policy requirement — describes a set of seven, `i` included. Nothing in that bullet's argument moves: the subset IS the per-word policy the bullet asked for, made configurable without packs, and the pack-side half is still what the 2026-09-13 (A) bullet above asks, since a `ca` or `pl` caller still has to write `Lexicon.default().remove(conjunctions_ambiguous={"i"})` by hand. See `decisions.md#P3`'s 2026-09-20 bullet. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 9fa6e560..07db3dbb 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -33,7 +33,7 @@ Problem shape. A rule counts "one name word", but the input holds words that ano ## MARK-DONT-STRIP — record the decision, keep the fact -Problem shape. A stage concludes that a vocabulary fact is not operative for one occurrence — a particle that joins nothing, a middle that will render as family. Contract statement. It MARKS the token with a namespaced marker rather than removing the vocabulary tag: the tag says what the word IS, the marker says what was decided about it, and consumers ask for the pair. The cost, which has to be paid in the docs rather than avoided: where a stable tag was documented with a RECIPE — `particle` combined with Role.FAMILY "for actual family particles" — the marker makes the recipe diverge from the view it reproduced, so the recipe's documentation has to say so. How it works. Stripping is tempting because every downstream consumer becomes correct for free, with no predicate to change; it pays for that by destroying the record, and asymmetrically. `particle` is the only tag 31 of the 33 never-given particles carry, so stripping it leaves nothing on the token, while every ambiguous one still carries `vocab:particle-ambiguous` (the two exceptions, `mc` and `vd`, carry `vocab:suffix` — the same dual membership P6's S2 precedence turns on) — the half where a later fork is most interesting is the half that loses its evidence, and ParsedName has no lexicon to re-derive from. Marking also leaves a POSITIVE predicate: "this token used to have a tag" is not testable, `UNJOINED_TAG in tags` is, which is what an ambiguity emitter needs (#405). And `particle` is documented stable API meaning "a word from the particle vocabulary wherever it lands", so removing it would be a breaking change as well as a lossy one. Lives in. nameparser/_types.py — FOLDED_TAG (O3's fold, P6's attachment) and UNJOINED_TAG (R2's standalone particle), both namespaced and internal, qualifying stable tags that stay put. Reach for it when. A rule decides a word is not doing the job its vocabulary implies. +Problem shape. A stage concludes that a vocabulary fact is not operative for one occurrence — a particle that joins nothing, a middle that will render as family. Contract statement. It MARKS the token with a namespaced marker rather than removing the vocabulary tag: the tag says what the word IS, the marker says what was decided about it, and consumers ask for the pair. The cost, which has to be paid in the docs rather than avoided: where a stable tag was documented with a RECIPE — `particle` combined with Role.FAMILY "for actual family particles" — the marker makes the recipe diverge from the view it reproduced, so the recipe's documentation has to say so. How it works. Stripping is tempting because every downstream consumer becomes correct for free, with no predicate to change; it pays for that by destroying the record, and asymmetrically. `particle` is the only tag 31 of the 33 never-given particles carry, so stripping it leaves nothing on the token, while every ambiguous one still carries `vocab:particle-ambiguous` (the two exceptions, `mc` and `vd`, carry `vocab:suffix` — the same dual membership P6's S2 precedence turns on) — the half where a later fork is most interesting is the half that loses its evidence, and ParsedName has no lexicon to re-derive from. Marking also leaves a POSITIVE predicate: "this token used to have a tag" is not testable, `UNJOINED_TAG in tags` is, which is what an ambiguity emitter needs (#405). And `particle` is documented stable API meaning "a word from the particle vocabulary wherever it lands", so removing it would be a breaking change as well as a lossy one. Lives in. nameparser/_types.py — FOLDED_TAG (O3's fold, P6's attachment), UNJOINED_TAG (R2's standalone particle) and UNJOINED_CONJUNCTION_TAG (R3's connective with nothing in its part to join, #461), all namespaced and internal, qualifying stable tags that stay put. Reach for it when. A rule decides a word is not doing the job its vocabulary implies. ## STRUCTURE-GATES — comma shape as an explicit state @@ -59,7 +59,7 @@ Problem shape. Two stages need the same answer about the same input, and the one ## RENDER-HONORS-THE-PARSE — the parse decides it, the views honor it -Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which is filed and open here. How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is OPEN rather than closed: `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings give `A. V.` and `J.`, and the second is a rules.md#R3 example line the doc runner asserts. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. +Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a LONE connective, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. ## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins @@ -191,10 +191,11 @@ Problem shape. A test pins an ordering, a sort, a dedup or a partition, and its - Run all the gates, not the ones you remember: ruff runs before mypy and pytest in CI, and each has caught what the others passed. - Purge __pycache__ between same-length source mutations; stale bytecode makes a changed file measure as unchanged. - After NARROWING a rule, check the receiver: the names a narrowed rule sheds land on a neighbour, and nothing guarantees the neighbour's prose describes what it inherited — #375 fixed an over-claiming rule and relocated the bug onto its neighbour. Ask also what the old behavior was CONCEALING: #379's attachment removed the input a test used to build an all-particle middle name (#402), and #400's reserve fix exposed the dual-membership count shape #397 names -- twice in one session a fix's real yield was a defect it stopped hiding. -- A ledger rule EXPLAINS a diff; nothing checks that its own sentence still DESCRIBES it, so a rule can go on covering a name it has stopped being true of and the gate stays green. Measured on #533: a `(?i)` anchor written for `John née Jones Smith Ma` — the clause keeping the credential — also reached the capitals spelling, which that change made read the opposite way, the clause giving the credential up. The rule still matched the name and still covered its fields, so `unexplained: 0` was silent about it. What found it was ATTRIBUTION rather than the gate: measuring every corpus name the change touches against the PARENT commit, the EXPLAINED ones as well as the unexplained, and asking of each which change actually moves it. Do that before writing a ledger rule's prose, not only when a name goes unexplained — the failure mode is a rule that reads as authoritative and argues for the reading it lost. +- A ledger rule EXPLAINS a diff; nothing checks that its own sentence still DESCRIBES it, so a rule can go on covering a name it has stopped being true of and the gate stays green. Measured on #533: a `(?i)` anchor written for `John née Jones Smith Ma` — the clause keeping the credential — also reached the capitals spelling, which that change made read the opposite way, the clause giving the credential up. The rule still matched the name and still covered its fields, so `unexplained: 0` was silent about it. What found it was ATTRIBUTION rather than the gate: measuring every corpus name the change touches against the PARENT commit, the EXPLAINED ones as well as the unexplained, and asking of each which change actually moves it. Do that before writing a ledger rule's prose, not only when a name goes unexplained — the failure mode is a rule that reads as authoritative and argues for the reading it lost. Second instance, #397/#461 (2026-09-20), and it is the same method finding the OPPOSITE shape: `fix(initials-per-word)` had been silently ABSORBING eleven 1.4.0-parity breaks that belong to the new connective rule, so the gate was green and the attribution was wrong about which change owned them. A rule can over-cover as well as mis-describe, and neither is a diff the gate can report. - A skip is indistinguishable from "correctly declined": pytest turns an empty parametrize into a skip, and a filter that widens its own skip set cannot fail. After changing any selection shape, verify the guard still REACHES the code it watches — assert the selected set is non-empty, or force-a-decision on its size. - A differential corpus cannot evidence behavior keyed to OUT-of-vocabulary shapes: it holds only names someone wrote down, and an unrecognized word is by definition outside the vocabulary — a green run over the corpus proves nothing about such a rule. - The corpus can be near-blind to a WRITING CONVENTION even where the vocabulary is well covered, and a small honest count then reads as a small blast radius. Measured for rules.md#P6: of 782 corpus names, 245 carry a comma, TWO of those end in a particle, and ONE of the two clears the words-to-spare guard and actually moves — so the Dutch trailing-tussenvoegsel listing the rule exists for is essentially unsampled. Before reporting "N names move", report the size of the population that COULD move; when that is ~1, the number is evidence about the corpus. This note's first wording said "exactly ONE ends in a particle", conflating the population with the movers — the very error it exists to prevent, committed inside the correction. Caught by re-running the count against AGENTS.md's axis 1 rather than by re-reading it. +- An invariant judged on the part a JOIN PRODUCED is satisfied by whatever the join absorbed. #397's both-sides condition was stated as "a name word stands on each side of the link", and the invariant written from that sentence walks the joined part and asks whether a non-connective word stands on each side of the letter — which a swallowed credential answers yes to, being that word. `Josep Lluis Carod i III` read family 'Carod i III' and PASSED it. The merge has already destroyed the evidence, so the question cannot be re-asked downstream at all. Judge on the PRE-join classes, or against a second parse with the rule turned off — the off-switch parse puts the credential in `suffix`, and "no word the off-switch reading calls a suffix lands inside a joined name part" caught it on 960 parses where the first invariant caught none. The same pairing is worth reaching for whenever a rule MERGES tokens. - A detector that re-implements a rule's grouping will get the grouping wrong. Derive the boundary from the same vocabulary the rule reads, not from the half you happen to be thinking about: walking a particle run over the NEVER-GIVEN set alone (the rule chains through ANY particle) split "de la Vega" after "de la" and reported 50 false movers for #364, where the true count is one. Both wrong answers were plausible and printed cleanly. - Guard the whole family, parametrize over it: a defect on one of N parallel entry points hides behind a per-example test — three times in one session (a guard on one class of two, a decode hint on 3 of 5 entry points, a sync roster missing 4 copies) — and a {class}×{field}×{bad-value} parametrization is what caught each. - A growth guard needs calibration, not just existence: benchmark guards that compare n vs 4n catch the quadratic the absolute-time tests are blind to, but calibrate against the WEAKEST signal you must detect and confirm a planted regression fails across repeated runs, not once — a stochastic check "verified" on one sample verifies nothing. diff --git a/docs/design/rules.md b/docs/design/rules.md index fe22580b..387e1422 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -592,10 +592,10 @@ P3. Rationale: connective words ("y", "of the") bind name words into and .capitalized() gives "John E Smith", where the connective reading gave "j. s." and "John e Smith" — a connective contributing no initial (R3) and keeping its lowercase (R4), - where an initial does neither. The v1 facade's initials() still - reads the letter by vocabulary and written shape rather than by - the parse's reading, so HumanName("john e smith").initials() - stays "j. s." for now; decisions.md#P3 records the split. + where an initial does neither. The v1 facade's initials() reads + the same parse, so HumanName("john e smith").initials() is + also "j. e. s."; #528 closed the split and decisions.md#P3 + records it. Accepted: two 1.4.0 parity breaks, one in each direction. A bare capital in a name written wholly in upper case no longer reads as an initial, so "JUAN GARCIA Y LOPEZ" joins where 1.4.0 and diff --git a/docs/release_log.rst b/docs/release_log.rst index 12e9f2ba..7eeb7bf6 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -6,9 +6,9 @@ Release Log **Behavior Changes** - - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) + - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves its ``capitalize()`` the same way in reverse, giving ``Juan y Garcia``, while its initials do not move at all: ``parse(...).initials()`` is ``J. Y. G.``, what 2.3.0 gave and what 1.4.0's own view gave, the ``Y`` holding its part alone and contributing an initial again under the connective-initials fix further down this list (closes #461). ``HumanName.initials()`` agrees with the core on both -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) - - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), or built from keyword fields (``HumanName(first=..., middle=..., last=...)``) carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. A subclass overriding one of the public ``first_list``, ``middle_list`` or ``last_list`` properties keeps working too: that member takes the pre-2.4 vocabulary reading instead of the change above, while an un-overridden member still moves. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``; ``JUAN Y GARCIA`` does not move at all this cycle, giving ``J. Y. G.`` on both surfaces as 2.3.0 and 1.4.0 did, since the connective-initials fix further down this list (closes #461) gives its ``Y`` an initial again. That first name read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases it does not move at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), or built from keyword fields (``HumanName(first=..., middle=..., last=...)``) carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. A subclass overriding one of the public ``first_list``, ``middle_list`` or ``last_list`` properties keeps working too: that member takes the pre-2.4 vocabulary reading instead of the change above, while an un-overridden member still moves. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) - **Fix a credential acronym that is also a surname being read by position alone.** ``HumanName("Jack MA")`` gives suffix ``MA`` where 2.0 through 2.3 gave last ``MA``, and ``John Smith Ma`` gives last ``Ma`` where they gave suffix ``Ma``. In a name written in more than one case, an ambiguous acronym written in capitals is written the way a credential is written and is read as one even where removing it leaves no surname; one written in any other cased form that is not wholly lower is written the way a surname is written and stays one even where there are words to spare (``John Smith ma`` and ``John Smith ed`` -- all lower, no contrast -- give suffix ``ma``/``ed`` instead). A name written wholly in one case says nothing either way and keeps the reading it had: ``JOHN SMITH MA`` is still a credential, ``ANH DO`` still a surname, ``jack ma`` still a surname. The same reading reaches the comma forms, where the words-to-spare count is now a count of NAME words: ``Smith, MA`` gives last ``Smith``, suffix ``MA``; ``Smith Jr., MA`` keeps last ``Smith``; and ``John Smith, MA``, ``John Smith, Ed``, ``john smith, ma`` and ``JOHN SMITH, MA`` all give a suffix again, which is what 1.4.0 read and 2.0 through 2.3 did not. ``Jack Ma`` and ``Anh Do`` are unchanged. The LEAN is inert on a caseless script, but the comma count above is not -- it asks name-word count, not case -- so ``마틴 킹, MA`` and ``田中 太郎, MA`` also give a suffix again (1.4.0 parity on the suffix, two pre-comma name words each) while the single-token ``毛泽东, MA`` does not move, having no case to write a contrast in either way. See the ``S2`` entry of ``docs/design/decisions.md`` (closes #289) @@ -22,9 +22,13 @@ Release Log - **Fix a maiden marker's clause swallowing a trailing credential in silence.** ``HumanName("Jane Doe nee Smith MA")`` gives maiden ``Smith`` with suffix ``MA``, where 2.0 through 2.3 gave maiden ``Smith MA`` and said nothing; 1.4.0 read the ``MA`` as a suffix too. ``Doe, Jane nee Smith MA`` moves with it, and so do the one-case spellings ``JANE DOE NEE SMITH MA`` and ``jane doe nee smith ma``. The words a marker takes now end where a trailing credential begins, which is what the marker's other two stops -- a suffix word, a trailing roman numeral -- have always done. Until this release it was the last trailing position in the library where a word of the ambiguous credential class was read without a report, and it was order-sensitive besides: ``Jane Doe nee Smith MA PhD`` gave maiden ``Smith MA`` while ``Jane Doe nee Smith PhD MA`` gave maiden ``Smith``, so whether the word was read at all depended on which side of the unambiguous credential the writer put it. Both now give maiden ``Smith``, with suffix ``MA PhD`` and ``PhD MA``. The writing still decides, exactly as it does for the same word ending a name with no clause: ``Jane Doe nee Smith Ma`` keeps maiden ``Smith Ma``, and ``Jane Doe nee Yo-Yo Ma`` keeps a two-word birth surname whole. The one member of this class that is also a surname particle keeps the carve-out it has outside a clause -- ``Doe, Jane nee Smith DO`` gives suffix ``DO`` while ``Doe, Jane nee Smith do`` and ``Doe, Jane nee Smith Do`` keep maiden ``Smith do`` and ``Smith Do``, and the comma-less ``Jane Doe nee Smith do`` gives suffix ``do`` as ``John Doe do`` does. Either reading is now reported, and there is no third: a word the clause gives up reads as a post-nominal, or the clause keeps it and says so. A name word behind the credential ends its reach and stays silent -- ``Jane Doe nee MA Smith`` gives maiden ``MA Smith`` and reports nothing -- and this stop never takes the first word after the marker, whatever its writing says: ``Jane Doe nee MA`` keeps maiden ``MA`` and reports, the marker having announced a name where there would otherwise be none, and ``Jane Doe nee MA PhD`` keeps it too. That differs on purpose from what a certain post-nominal gets there, ``Jane Smith nee PhD`` and ``Jane Smith nee V`` leaving the marker standing as an ordinary word as before. Where no trailing rule reads the clause's tail nothing is decided and the clause keeps every word: ``Smith nee Jones MA, Jane`` and ``Smith, John, Jr nee Jones MA`` both keep maiden ``Jones MA``, unchanged and with no ``suffix-or-name`` report. A trailing title is not transparent here and the two spellings disagree -- ``Jane Doe nee Smith MA Prof.`` is unchanged and silent while ``Jane Doe nee Smith Prof. MA`` gives maiden ``Smith Prof.`` with suffix ``MA`` -- which is recorded as a boundary rather than fixed. The clause also keeps a word it cannot promise a credential reading for, which is where three shapes that look like they should move do not. Where the part the word would land in holds no name of its own there is nothing to read it as a credential, so ``Doe, Dr. nee Smith MA`` and ``Jane Doe, Jr nee Smith MA`` both keep maiden ``Smith MA`` and report. Where a join would swallow it first the same applies, and it is the birth name that would lose the word: ``Berg, abdul nee Jones MA`` keeps maiden ``Jones MA`` rather than reading first ``abdul MA``, and ``Berg, Jane van der nee Smith DO`` keeps maiden ``Smith DO`` rather than letting the particle chain carry the ``DO`` into last ``van der DO Berg``. Each of those reads as 2.3.0 read it. Delimiters settle the question outright and always did: ``HumanName("Jane Doe (nee Smith MA)")`` keeps the whole span as the maiden name and reports nothing, the writer having drawn the boundary, while ``Jane Doe (nee Smith) MA`` gives suffix ``MA`` for the word left outside it. One name is a restoration rather than a change: ``John Smith nee Jones R.A.I.`` gives suffix ``R.A.I.`` again, as 2.3.0 read it, this unreleased cycle having moved it into the maiden name when the unlisted-dotted reading above took the word out of the certain-suffix class. See the ``M2`` and ``S2`` entries of ``docs/design/decisions.md`` (closes #533) + - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). One more view moves, and only on a forced call: ``capitalize(force=True)`` leaves a lower-case ``i`` alone where every release title-cased it, so ``HumanName("Carod i")`` forced gives ``Carod i`` where 1.4.0 and 2.3.0 gave ``Carod I`` -- a connective keeps its lower case wherever it stands. A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) + + - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) + **Additions** - - **Add Lexicon.conjunctions_ambiguous, the one-letter connectives that read as initials.** A subset of ``conjunctions`` holding ``e`` by default; it is the knob for the change above rather than a switch. Portuguese data, where ``e`` links surnames the way ``y`` does in Spanish, takes it out: ``Lexicon.default().remove(conjunctions_ambiguous={"e"})`` restores the joining reading. Dutch data, where a bare single letter is an initial and never a connective, adds the other one: ``Lexicon.default().add(conjunctions_ambiguous={"y"})``. A v1 ``Constants`` has no manager of its own for it -- deleting the word from ``conjunctions`` is what turns the marking off, the same rule the glued-honorific tails follow. See ``docs/customize.rst`` (#383, #479) + - **Add Lexicon.conjunctions_ambiguous, the one-letter connectives that read as initials.** A subset of ``conjunctions`` holding ``e`` and ``i`` by default; it is the knob for the change above rather than a switch. Portuguese data, where ``e`` links surnames the way ``y`` does in Spanish, takes it out: ``Lexicon.default().remove(conjunctions_ambiguous={"e"})`` restores the joining reading. Dutch data, where a bare single letter is an initial and never a connective, adds the other one: ``Lexicon.default().add(conjunctions_ambiguous={"y"})``. A v1 ``Constants`` has no manager of its own for it -- deleting the word from ``conjunctions`` is what turns the marking off, the same rule the glued-honorific tails follow. See ``docs/customize.rst`` (#383, #479) - **Add AmbiguityKind.CONJUNCTION_OR_INITIAL, reported when a one-letter connective in a name written wholly in one case is read as an initial:** ``parse("jose e maria santos").ambiguities`` and ``parse("JOSE E MARIA SANTOS").ambiguities`` both name it, and ``detail`` names the letter. That is the call the behavior change above had to make. A letter outside the marked set reports nothing, its reading not being in doubt, so ``JUAN GARCIA Y LOPEZ`` is silent; so is every mixed-case name, where the writing decided it. See the ``P3`` entry of ``docs/design/decisions.md`` (#383, #479) From f76963b73bb85980d79fcc3b80162fca68cd07bc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 08:00:17 -0700 Subject: [PATCH 06/11] fix(#397): a generation repairs as a generation; docs review A word can be the Catalan link and the roman numeral at once, and where the parse read the GENERATION the token still carries the `conjunction` tag classify gave it. Case repair's conjunction conjunct read that tag without asking the role, so a suffix was lower-cased as a connective: parse("John Quincy Smith i").capitalized(force=True) gave 'John Quincy Smith i' where every release through 2.3 gave 'John Quincy Smith I', and so did 'Carod i', 'Josep Carod i', 'Lluis Carod i', 'Josep Lluis Carod i III', 'Josep Lluis Carod i V', 'Carod y de Rovira i' and 'Rovira, Josep Carod i Jr.'. Both surfaces, the facade repairing through the same helper. _cap_word now asks the ROLE the parse decided and not the tag alone -- one comparison on a value already in hand, ahead of both conjunction arms. The role is what the parse decided; the vocabulary is only what the word could have been. Frame-neutral: py3.11 parse=406.00 facade=443.00, unchanged, and the benchmark band is untouched. What is NOT a defect and stays: a connective the parse placed among the NAME words keeps its lowercase, so 'Carod i Rovira' and 'Josep i Rovira' repair to themselves -- exactly the answer the same shape has always had for 'y' ('Carod y Rovira', 'Josep y Rovira'). Those two are the whole of what this cycle moves on that view, down from ten. tests/v2/test_properties.py gains INV7: a letter that became no connective of this name repairs as the off-switch parser repairs it, plain and forced, on both surfaces, exempting only the names whose fields moved, the initial-reading disagreement, and R4's own name-part connective. Recorded control: 11,341 repairs fail it at e540d4c5 (10,424 core, 917 v1) and 0 here; removing the role test alone fails it on the same 11,341, and fails the named unit test in tests/v2/test_render.py while leaving the two existing R4 pins green. rules.md: R4's sentence gains its second clause and two capitalized_forced example lines, both names already in the corpora so no corpus or claim moves. P3's both-sides clause said the neighbour must not be a credential or an honorific and omitted the GENERATION, which licensed the reading its own example forbids; the _group.py citation moves with it. P3 and S2 now name each other under interacts:, the new clause having moved a pre-comma generation. decisions.md, correcting three claims in bullets this branch added: 'Carod i' is 2.0.0-2.3.0 parity and not 1.4.0's, which read family 'i'; 'JUAN Y GARCIA' gave 'J. G.' at 2.0.0-2.2.0 and 1.4.0's own answer at 2.3.0, while the Arabic name gave the 2.x answer across the whole range; and R4's forced-call paragraph is rewritten from "recorded rather than repaired" to the repair, with the invariant's numbers. mechanisms.md narrows RENDER-HONORS-THE-PARSE's remaining disagreement a second time and says what the defect taught: a view honors what the parse DECIDED, and a role is that decision where a tag is only what the word could have been. release_log.rst no longer ships the defect as a behavior change. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/decisions.md | 11 ++-- docs/design/mechanisms.md | 2 +- docs/design/rules.md | 28 ++++++---- docs/release_log.rst | 2 +- nameparser/_pipeline/_group.py | 4 +- nameparser/_render.py | 33 +++++++++--- tests/v2/test_properties.py | 97 +++++++++++++++++++++++++++++++++- tests/v2/test_render.py | 38 +++++++++++++ 8 files changed, 187 insertions(+), 28 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 26e55f05..77513d93 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -352,7 +352,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf ACCEPTED CONSEQUENCE OF REACHING THE CLASS, and the one row the design's reading tables show it touching: under `Lexicon.default().add(particles={"y"})`, `Juan Velasquez y Garcia` goes from given "Juan" / middle "Velasquez" / family "y Garcia" to given "Juan" / family "Velasquez y Garcia" — the custom lexicon now AGREES with the default one, which is the right direction — and its `family_base` becomes "Velasquez Garcia", a pre-existing R2 limit (`family_base` drops a particle wherever it stands, not only leading) surfaced rather than created here. Measured 2026-09-20 on this branch. THE BOTH-SIDES SCOPE, AND THE REVIEW THAT FIXED IT. A connective that is also generational vocabulary joins only where a NAME WORD stands on each side of it — a word the rest of the parse reads as a name word rather than as a credential, the trailing numeral or a title, looked for past any run of connectives standing between. The FIRST implementation tested POSITION (an interior piece) rather than the neighbours' CLASS, and swallowed a credential: "Josep Lluis Carod i III" read family "Carod i III" where the "III" is the generation. An independent review found it. It is recorded here because the invariant that ought to have caught it cannot: INV1 inspects the part the join PRODUCED, where the absorbed credential is itself the name word standing on the right, so that name SATISFIES it — the general lesson is a field note in mechanisms.md of this date, and the instrument is `tests/v2/test_properties.py::test_a_trailing_credential_never_joins_into_a_name_part`, which asks the off-switch parse instead and carries its own control (960 failing parses at `c8550b64`, 0 here). THE SCOPE'S ALTERNATIVES, MEASURED 2026-09-20 rather than argued, by building each alternative from this tree and sweeping the 1558 non-empty corpus-union-cases names under eight configurations (default, `lenient_comma_suffixes=False`, both family-first orders, and the four locale packs). ABSENT — no both-sides condition at all — differs on five names, and every one of them is a row this bundle added: "John Quincy Smith i" would read family "Smith i", "Josep Lluis Carod i III" family "Carod i III", "Josep Lluis Carod i V" family "Carod i V", "Carod y de Rovira i" given "Carod y de" with family "Rovira i", and "Rovira, Josep Carod i Jr." middle "Carod i Jr.". Nothing that stood in the corpus at the parent separates absent from narrow, so those rows ARE the evidence and pruning one retires it. WIDE — the same condition with the generational-vocabulary test dropped, so it reaches every single-character connective — differs on five as well, and THREE of those were in the corpus already: "Mr. & Mrs. John Smith" would read title "Mr." with given "&" and middle "Mrs. John", "1 & 2, 3 4 5, Mr." would lose "2" out of the family, and "Jane née Jr y Jones" would read middle "née Jr y" with family "Jones". So the class test is not decoration — widening the rule past the link-and-generation class breaks "Mr. & Mrs." — and narrow is both what the design asked for and what measurement supports. (A "0 differences over 1545 names" figure was measured for this scope before the review supplied the class test and before those rows existed; it is superseded here.) - ACCEPTED, THE TWO-WORD SHAPE: "Carod i" keeps given "Carod" with suffix "i", and "Josep Carod i" keeps family "Carod" with suffix "i", because a connective with nothing to its right is connecting nothing. 1.4.0 read the first of those the same way. The ledger rule that called this NOT WANTED is rewritten in place rather than deleted — it still explains the name, and the standing is decided now rather than unwanted. + ACCEPTED, THE TWO-WORD SHAPE: "Carod i" keeps given "Carod" with suffix "i", and "Josep Carod i" keeps family "Carod" with suffix "i", because a connective with nothing to its right is connecting nothing. 2.0.0 through 2.3.0 read BOTH of those the same way; 1.4.0 read the two-word one differently, giving family "i" with given "Carod" (measured on the wheels 2026-09-21, correcting an earlier draft of this bullet that claimed 1.4.0 parity for it). That does not move the accepted cost, because the 1.4.0 difference is not this rule's: the two-word shape's routing left 1.4.0's reading in 2.0, which is why the case row carries `fix(suffix-routing)` rather than a connective classification. What this rule decides is that the link does not join there, and 2.0 through 2.3 agree with it. The ledger rule that called this NOT WANTED is rewritten in place rather than deleted — it still explains the name, and the standing is decided now rather than unwanted. ACCEPTED, A CONNECTIVE RUN: "Henry i of England" reads given "Henry i of England" with a `given-or-family` report, where the parent read given "Henry" with family "i of England". The "i" and the "of" merge as a run and the run joins the whole name into one piece — the same thing "Duke of Edinburgh" already did. ACCEPTED, THE PARTICLE ON THE RIGHT. A particle is a name word for the both-sides test, so the link joins across one: "Josep de Carod i Rovira" reads family "de Carod i Rovira", which is the Catalan reading, while "Josep Carod i de Rovira" reads middle "Carod i de" with family "Rovira" where the parent read middle "Carod i" with family "de Rovira" — the join takes the particle as its right-hand neighbour and the particle chain then follows the joined piece instead of the tail word. Recorded rather than repaired: which side a particle standing between a link and a surname belongs to is P2's question and not this one. ACCEPTED, AND THE HONEST STATEMENT OF THE ONE-CASE HALF, written out because this branch's own first commit message overclaimed that one-case names keep their reading. `i` is in the MARKED subset, so in a name written wholly in one case it reads as an INITIAL and reports `conjunction-or-initial`, exactly as `e` has since #383/#479. For "JOSEP CAROD I ROVIRA" and "josep carod i rovira" the fields are indeed unchanged and only the report is new. But a one-case name reports wherever a bare `i`/`I` stands among the NAME'S OWN WORDS, which is a great many more names than the Catalan ones — "JOHN I SMITH" and "john i smith" one report each, "JOHN SMITH I" and "john smith i" two, "HENRY I" and "henry i" three. A letter inside a maiden clause is read by the clause's rules and stays silent, exactly as `e` does ("JANE DOE NEE I JONES" reports nothing), which is the own-words scope the 2026-09-13 #383/#479 entry above defines. And in ALL-LOWER names the initial reading MOVES FIELDS wherever the parent read the lower-case letter as the generation. Each such name now reads as its ALL-CAPS twin already did: "rovira, i" gives given "i" where it gave suffix "i" ("ROVIRA, I" already gave given "I"); "john smith i jr" gives middle "smith" with family "i" where it gave family "smith" with suffix "i jr" ("JOHN SMITH I JR" already did); "maier, amy i, jr." gives middle "i" with suffix "jr." where it gave suffix "i, jr." (the corpus name "Maier, Amy I, Jr." reads middle "I" and does not move); and "josep de carod i rovira" gives family "de carod i rovira" where it gave middle "de carod i" with family "rovira". None of those four is a corpus name; all are measured 2026-09-20 on this tree and against the parent's. @@ -1272,7 +1272,7 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-20 #461 — DONE, and the clause is settled for every group at once. THE RULE AS TAKEN: a connective contributes nothing where it is JOINING — a part holding another name word for it to join, the part's working particles set aside — and where its part holds nothing else it is joining nothing, so it initials like any other name word and agrees with the base. That picks ONE of the four candidate answers the 2026-08-29 bullet under `decisions.md#R2` listed for "John and Jane Smith" and makes the other three unreachable: `parse("John and Jane Smith").initials()` is "J. J. S.", not the "J. a. J. S." of 2.0 through 2.3, not P3's one-name-word "J. S.", and not 1.4.0's run-together "J a J. S.". THE CRITERION IS NEVER A WORD COUNT, which is what keeps `parse("Jon Dough and")` at "J. D." over base "Dough and" and `parse("Juan Velasquez y Garcia")` at "J. V. G." — a part of two words where one is the connective is still a part where it joins, and a part of three is no different. BOTH VIEWS, ONE MARK. The fact is about a PART, so it is decided once where the parts are settled and recorded on the token (`UNJOINED_CONJUNCTION_TAG`, mechanisms.md#MARK-DONT-STRIP), and the core view and the v1 facade both read it. That is the direct answer to what the first #461 attempt found: a mark honored for some of a part's words and not for one of them is honoring nothing. A SEPARATE marker rather than a widening of `UNJOINED_TAG`, because that one is also read by `family_particles`/`family_base` and by `_cap_word`, and none of those three may move. `Parser.revise` recomputes both marks, the same reason `_remarked` recomputes the first. BLAST RADIUS, MEASURED 2026-09-20 against the parent commit `46651750` over the deduped corpus-union-cases set under eight configurations, both processes asserting their own `nameparser.__file__`. Over the population AS IT STANDS — 1558 non-empty names, grown by this bundle's own rows — the default order moves 6 names' roles, 3 more names' reports with the roles fixed, and 50 names' `initials()`, 19 gaining a letter and 31 losing one; `HumanName.initials()` moves on THE SAME 50 names with THE SAME values. Family-first moves 5 / 3 / 28 and `FAMILY_FIRST_GIVEN_LAST` 6 / 3 / 58; the 35 names that move their initials under given-last and not under the default order are mostly the ones whose joined run lands in the GIVEN group there, `Juan Velasquez y Garcia` going from "V. y. G. J." to "V. G. J." — family-first now agreeing with given-first on it. Restricted to the 1544 names that stood in the corpus at the parent — the comparable population, since the rows this bundle added are themselves movers — the same sweep reads 2 / 3 / 46 at the default order, 18 gaining and 28 losing, with 1 / 3 / 24 family-first and 2 / 3 / 52 given-last. RECOMPUTE by parsing every name of the deduped `tools/differential/corpus*.jsonl` glob plus `tests/v2/cases.py`'s texts under those eight configurations in two processes, one on this tree and one on the parent's, and diffing the seven role fields, the ambiguity kinds and `initials()` on both surfaces. The gate cannot answer this on its own: its `_initials` pseudo-field sees only names whose roles stayed put, so a count taken from the classified summary is a FLOOR. - TWO OF THE GAINS ARE 1.4.0 PARITY RESTORED, and that is what retires two ledger rules: `JUAN Y GARCIA` gives "J. Y. G." and `محمد و علي` gives "م. و. ع.", which is what 1.4.0 gave for both (measured on the released wheel 2026-09-20; 2.0.0 through 2.3.0 gave "J. G." and "م. ع." on the facade for the reasons the 2026-09-13 #383/#479 entry under P3 records). The readmission is a parity RESTORATION on those two names rather than a deviation, and the given-group half in the other direction is the deviation — see rules.md#R3's Accepted clause, which states both. + TWO OF THE GAINS ARE 1.4.0 PARITY RESTORED, and that is what retires two ledger rules: `JUAN Y GARCIA` gives "J. Y. G." and `محمد و علي` gives "م. و. ع.", which is what 1.4.0 gave for both. The two ranges differ and an earlier draft of this bullet ran them together, so state them apart (measured on all five wheels 2026-09-21): `محمد و علي` gave "م. ع." at 2.0.0, 2.1.0, 2.2.0 AND 2.3.0, while `JUAN Y GARCIA` gave "J. G." at 2.0.0, 2.1.0 and 2.2.0 and went back to 1.4.0's "J. Y. G." at 2.3.0 — the reason being the one the 2026-09-13 #383/#479 entry under P3 records, and the 2.4.0 release note says the same. The readmission is a parity RESTORATION on those two names rather than a deviation, and the given-group half in the other direction is the deviation — see rules.md#R3's Accepted clause, which states both. THE FRAME BUDGET WENT DOWN: `tools/perf/call_count.py` reads `py3.11 parse=406.00 facade=443.00` where the parent read `412.00 / 449.00`. The two marks now come off ONE walk over the roles instead of two generator expressions, and on 3.11 a genexp costs a frame for its creation and one per resume, so the reference name's two non-empty parts pay exactly six. `_CALL_BASELINE[(3,11)]` is `{"parse": 410, "facade": 447}` with `_BAND = 0.02`, so the bands are [401.8, 418.2] and [438.06, 455.94] and both figures land inside: NO baseline edit and no `decisions.md#parse-cost` decision is owed. PEP 709 inlined comprehensions and not generator expressions, so 3.12 through 3.15 should drop by the same six. THE NO-PARSE PATHS KEEP THE OLDER READING, the same degradation the 2026-09-13 #528 bullet above accepted and for the same reason: a token the parse never saw carries no mark, so the view falls back to the vocabulary. Measured 2026-09-20, `HumanName("JUAN GARCIA Y LOPEZ").initials()` is "J. G. L." live and "J. G. Y. L." after `pickle`, `copy.copy` or `copy.deepcopy`; a name built from keyword fields carries no tags either, and so does one whose `*_list` property a subclass overrides. The remedy is the v1 one — assign `full_name` again — since the facade has no `revise`. @@ -1297,9 +1297,12 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-08-29 — WHY THE BOUNDARY WENT UNNOTICED UNTIL #407, which is where a future reader should look for it. For an ALL-PARTICLE part the other three tag-driven views give the same answer through `replace()` and `revise()` alike: measured over `de la`, `van der`, `do`, `de` and `van de la`, all five agree on `family_particles=''`, on a `family_base` holding the whole part, and on initials from every word. They converge because an UNTAGGED part and a MARKED all-particle part reach the same place by different routes — untagged, no word is recognized as a particle; marked, none is ACTING as one — and all three views only ask which words are particles. Case repair is the one view that asks a second question, since it must also decide whether to lowercase, so it is where the two routes first come apart. The mirror case confirms the reading: on a MIXED part the convergence is the other way round — `de la vega` and `van der berg` diverge in all three views between `replace()` and `revise()` (`replace()` reports particles `''` and base `'de la vega'` where `revise()` reports `'de la'` and `'vega'`) and AGREE on case repair, R4's all-particle clause not reaching them. So before #407 the distinction was invisible on exactly the shape the clause is about, and visible only on shapes the clause does not govern. -- 2026-09-20 #461/#397 — THE BEHAVIOR OF THE CLAUSE DID NOT MOVE, THE GROUNDING DID. R4's sentence rested on R3 by name ("the carve-out R3 states for initials"), and R3's carve-out is conditional now while R4's is not, so the cross-reference is CUT and replaced by this rule's own reason: a connective that initials because it joins nothing is still not written the way a name is written. Measured 2026-09-20, plain and forced, over `juan y`, `john and jane smith`, `duke of edinburgh`, `juan de y`, `juan y garcia`, `JUAN Y GARCIA`, `josep carod i rovira`, `JOSEP CAROD I ROVIRA` and — under `add(particles={"y"})` — `Anh y Van`: all nine are byte-identical to the same call at the parent, and `tests/test_capitalization.py` is green. +- 2026-09-20/21 #461/#397 — THE GROUNDING MOVED, AND THEN A DEFECT THE GROUNDING EXPOSED. R4's sentence rested on R3 by name ("the carve-out R3 states for initials"), and R3's carve-out is conditional now while R4's is not, so the cross-reference is CUT and replaced by this rule's own reason: a connective that initials because it joins nothing is still not written the way a name is written. Measured 2026-09-20, plain and forced, over `juan y`, `john and jane smith`, `duke of edinburgh`, `juan de y`, `juan y garcia`, `JUAN Y GARCIA`, `josep carod i rovira`, `JOSEP CAROD I ROVIRA` and — under `add(particles={"y"})` — `Anh y Van`: all nine are byte-identical to the same call at the parent, re-measured after the 2026-09-21 repair below and still byte-identical, and `tests/test_capitalization.py` is green. WHAT THE PLAIN CALL DOES, stated over a population rather than over nine names, because "capitalized() does not move" was drafted for this bullet and is false as written. Swept 2026-09-20 over the 1558 non-empty corpus-union-cases names under eight configurations against the parent commit, `capitalized()` moves on SIX distinct names and every one of them is a name whose ROLES moved under the join — it renders different fields rather than treating a word differently. The correct statement is therefore: where only `initials()` moves, `capitalized()` is byte-identical; where the roles move, it follows them. - WHAT THE FORCED CALL DOES, and this one IS a word treated differently — an unrecorded 1.4.0 parity break that belongs to #397's vocabulary rather than to #461's rule. `capitalized(force=True)` moves on TEN further names in that sweep whose roles, initials and plain repair all stayed put, and every one is a lower-case `i` that repair used to title-case: `parse("Carod i").capitalized(force=True)` gives "Carod i" where 1.4.0 and 2.3.0 both gave "Carod I", and so do "Carod i Rovira", "Josep i Rovira", "Josep Carod i", "Lluis Carod i", "John Quincy Smith i", "Josep Lluis Carod i III", "Josep Lluis Carod i V", "Carod y de Rovira i" and "Rovira, Josep Carod i Jr.". The cause is this rule working as written: the letter is connective vocabulary now, and a connective keeps its lower case wherever it stands. Only the FORCED call witnesses it — R5's gate refuses a mixed-case name before any of this is consulted — and the differential cannot see it at all, case being no compared surface (the 2026-08-29 bullets above settle why). Recorded rather than repaired, and named here because the sweep that found it was run for a claim about the PLAIN call. + WHAT THE FORCED CALL DID, AND THE DEFECT THAT WAS FOUND WRITING THIS RECORD. The sweep above was run for the PLAIN claim and turned up a second answer: `capitalized(force=True)` moved on ten further names whose roles, initials and plain repair all stayed put, every one a lower-case `i` — `parse("Carod i")` forced gave "Carod i" where 1.4.0 and 2.3.0 gave "Carod I", and so did "John Quincy Smith i", "Josep Carod i", "Lluis Carod i", "Josep Lluis Carod i III", "Josep Lluis Carod i V", "Carod y de Rovira i", "Rovira, Josep Carod i Jr.", "Carod i Rovira" and "Josep i Rovira". Nine of those were a DEFECT and are REPAIRED here; the remaining two are the rule. + WHY IT WAS A DEFECT, and the line R4 now draws. In those names the parse read the letter as the GENERATION it also spells and roled the token SUFFIX — but classify had tagged it `conjunction`, and repair's conjunction conjunct read the tag without asking the role, so a generation was lower-cased as a connective. Derek's principle decides it: a connective with nothing to join is not acting as a connective. Repair now asks the ROLE the parse decided rather than the tag alone, and a suffix-roled token is repaired as the suffix it was read as. `parse("John Quincy Smith i")` forced gives "John Quincy Smith I" again on BOTH surfaces, the facade repairing through the same helper. + WHAT REMAINS AND IS NOT A DEFECT: `Carod i Rovira` and `Josep i Rovira`, where the letter is a lone MIDDLE word the parse tagged `conjunction` — three-word names, so P3's carve-out leaves the letter a name word and nothing joined. Forced repair keeps it lower-case, which is R4's own rule and exactly the answer the same shape has always had for `y`: `Carod y Rovira` and `Josep y Rovira` give "Carod y Rovira" and "Josep y Rovira" too. Those two names are the whole of what #397 moves on this view. + MEASURED, and the numbers are the fix's own control. Over the same eight configurations the forced-only set is ten names before the fix and two after; the invariant `tests/v2/test_properties.py::test_a_letter_that_did_not_join_repairs_as_the_off_switch_does` — a letter that became no connective of this name repairs as the off-switch parse repairs it, plain and forced, on both surfaces — fails on 11,341 repairs at `e540d4c5` (10,424 core, 917 v1) and on 0 here, and removing the role test alone fails it on the same 11,341. The differential still cannot see any of it, case being no compared surface (the 2026-08-29 bullets above settle why), which is why this needed an invariant rather than a ledger rule. ### R5 — the case-repair gate diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 07db3dbb..7512d12f 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -59,7 +59,7 @@ Problem shape. Two stages need the same answer about the same input, and the one ## RENDER-HONORS-THE-PARSE — the parse decides it, the views honor it -Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a LONE connective, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. +Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a connective the parse placed among the NAME words with nothing to join, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). Narrowed twice, the second time by a defect the record itself turned up (2026-09-21): where the parse read such a word as the GENERATION it also spells, repair was reading classify's `conjunction` tag without asking the role and lower-casing a suffix, so `parse("John Quincy Smith i")` forced gave 'John Quincy Smith i'. A view honors what the parse DECIDED, and the role is that decision where the tag is only what the word could have been -- the same distinction this entry draws between re-deriving an answer and reading the record, one level up. The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. ## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins diff --git a/docs/design/rules.md b/docs/design/rules.md index 387e1422..aa5f6628 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -528,9 +528,9 @@ P3. Rationale: connective words ("y", "of the") bind name words into connective's join instead. A connective that is also generational vocabulary joins only where a name word stands on each side of it — a word the rest of the - parse reads as a name word rather than as a credential or an - honorific, looked for past any run of connectives standing - between. A connective with nothing to its right is connecting + parse reads as a name word rather than as a generation, a + credential or an honorific, looked for past any run of + connectives standing between. A connective with nothing to its right is connecting nothing, and a word of that vocabulary ending a name, or standing before the credential a name ends with, is the generation it also spells. @@ -635,7 +635,7 @@ P3. Rationale: connective words ("y", "of the") bind name words into same two words unjoined are two name words and H1 does not fire. P1's leading run is the second (#395, landed): its run takes the "Vega y Santos" join whole or stops before it. - history: decisions.md#P3 · interacts: H1, P1, M2, R3, R4 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_post_rules.py + history: decisions.md#P3 · interacts: H1, P1, M2, R3, R4, S2 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_post_rules.py P4. Rationale: a particle links forward from inside a name; at the very front there is no name yet to be inside. @@ -1103,7 +1103,7 @@ S2. Rationale: generational suffixes and credentials are recognized and unchanged (decisions.md#v1-xfail-triage: `king` stays a title, for the addressing forms). "Dr Jr" → suffix="Jr" - history: decisions.md#S2 · interacts: H1, H2, H3, H5, C1, S3, P2, P5, P6 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py + history: decisions.md#S2 · interacts: H1, H2, H3, H5, C1, S3, P2, P3, P5, P6 · implemented: nameparser/_pipeline/_classify.py, nameparser/_pipeline/_group.py, nameparser/_pipeline/_pieces.py, nameparser/_pipeline/_vocab.py S3. Rationale: credentials are often written run together with periods; the chunks between the periods are what carry the @@ -2125,12 +2125,16 @@ R4. Rationale: case repair is a display concern, applied only on convention (McDonald), not only ordinary word-by-word casing, and a part whose every word is particle vocabulary is repaired as ordinary name words, since none of them is doing a particle's - work there (R2). A CONJUNCTION keeps its lowercase wherever it - stands, including inside a part whose other words the unjoined - mark has turned into ordinary name words. That is this rule's own - reading and not a borrowing from R3: a connective that initials - because it joins nothing is still not written the way a name is - written. A name already written the way repair would + work there (R2). A CONNECTIVE the parse placed among the name + words keeps its lowercase wherever it stands there, including + inside a part whose other words the unjoined mark has turned into + ordinary name words; one the parse read as the generation it also + spells is not a connective of this name at all, and is repaired + as the generation it was read as. That is this rule's own reading + and not a borrowing from R3: a connective that initials because + it joins nothing is still not written the way a name is written, + while a generation is written the way a generation is written. + A name already written the way repair would write it comes back unchanged, measured by repair's own conventions rather than by the bearer's. A spelling written in a single case is repaired even where its bearer meant it, because @@ -2141,6 +2145,8 @@ R4. Rationale: case repair is a display concern, applied only on "ANH DO" → capitalized="Anh Do" "anh van do" → capitalized="Anh Van Do" "john smith phd" → capitalized="John Smith Ph.D." + "John Quincy Smith i" → capitalized_forced="John Quincy Smith I" + "Carod i" → capitalized_forced="Carod I" "juan de la vega" → capitalized="Juan de la Vega" · boundary Accepted: the clause reaches a part the parser read. A field spliced in as raw text after the parse carries no reading of its diff --git a/docs/release_log.rst b/docs/release_log.rst index 7eeb7bf6..bc18df0e 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -22,7 +22,7 @@ Release Log - **Fix a maiden marker's clause swallowing a trailing credential in silence.** ``HumanName("Jane Doe nee Smith MA")`` gives maiden ``Smith`` with suffix ``MA``, where 2.0 through 2.3 gave maiden ``Smith MA`` and said nothing; 1.4.0 read the ``MA`` as a suffix too. ``Doe, Jane nee Smith MA`` moves with it, and so do the one-case spellings ``JANE DOE NEE SMITH MA`` and ``jane doe nee smith ma``. The words a marker takes now end where a trailing credential begins, which is what the marker's other two stops -- a suffix word, a trailing roman numeral -- have always done. Until this release it was the last trailing position in the library where a word of the ambiguous credential class was read without a report, and it was order-sensitive besides: ``Jane Doe nee Smith MA PhD`` gave maiden ``Smith MA`` while ``Jane Doe nee Smith PhD MA`` gave maiden ``Smith``, so whether the word was read at all depended on which side of the unambiguous credential the writer put it. Both now give maiden ``Smith``, with suffix ``MA PhD`` and ``PhD MA``. The writing still decides, exactly as it does for the same word ending a name with no clause: ``Jane Doe nee Smith Ma`` keeps maiden ``Smith Ma``, and ``Jane Doe nee Yo-Yo Ma`` keeps a two-word birth surname whole. The one member of this class that is also a surname particle keeps the carve-out it has outside a clause -- ``Doe, Jane nee Smith DO`` gives suffix ``DO`` while ``Doe, Jane nee Smith do`` and ``Doe, Jane nee Smith Do`` keep maiden ``Smith do`` and ``Smith Do``, and the comma-less ``Jane Doe nee Smith do`` gives suffix ``do`` as ``John Doe do`` does. Either reading is now reported, and there is no third: a word the clause gives up reads as a post-nominal, or the clause keeps it and says so. A name word behind the credential ends its reach and stays silent -- ``Jane Doe nee MA Smith`` gives maiden ``MA Smith`` and reports nothing -- and this stop never takes the first word after the marker, whatever its writing says: ``Jane Doe nee MA`` keeps maiden ``MA`` and reports, the marker having announced a name where there would otherwise be none, and ``Jane Doe nee MA PhD`` keeps it too. That differs on purpose from what a certain post-nominal gets there, ``Jane Smith nee PhD`` and ``Jane Smith nee V`` leaving the marker standing as an ordinary word as before. Where no trailing rule reads the clause's tail nothing is decided and the clause keeps every word: ``Smith nee Jones MA, Jane`` and ``Smith, John, Jr nee Jones MA`` both keep maiden ``Jones MA``, unchanged and with no ``suffix-or-name`` report. A trailing title is not transparent here and the two spellings disagree -- ``Jane Doe nee Smith MA Prof.`` is unchanged and silent while ``Jane Doe nee Smith Prof. MA`` gives maiden ``Smith Prof.`` with suffix ``MA`` -- which is recorded as a boundary rather than fixed. The clause also keeps a word it cannot promise a credential reading for, which is where three shapes that look like they should move do not. Where the part the word would land in holds no name of its own there is nothing to read it as a credential, so ``Doe, Dr. nee Smith MA`` and ``Jane Doe, Jr nee Smith MA`` both keep maiden ``Smith MA`` and report. Where a join would swallow it first the same applies, and it is the birth name that would lose the word: ``Berg, abdul nee Jones MA`` keeps maiden ``Jones MA`` rather than reading first ``abdul MA``, and ``Berg, Jane van der nee Smith DO`` keeps maiden ``Smith DO`` rather than letting the particle chain carry the ``DO`` into last ``van der DO Berg``. Each of those reads as 2.3.0 read it. Delimiters settle the question outright and always did: ``HumanName("Jane Doe (nee Smith MA)")`` keeps the whole span as the maiden name and reports nothing, the writer having drawn the boundary, while ``Jane Doe (nee Smith) MA`` gives suffix ``MA`` for the word left outside it. One name is a restoration rather than a change: ``John Smith nee Jones R.A.I.`` gives suffix ``R.A.I.`` again, as 2.3.0 read it, this unreleased cycle having moved it into the maiden name when the unlisted-dotted reading above took the word out of the certain-suffix class. See the ``M2`` and ``S2`` entries of ``docs/design/decisions.md`` (closes #533) - - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). One more view moves, and only on a forced call: ``capitalize(force=True)`` leaves a lower-case ``i`` alone where every release title-cased it, so ``HumanName("Carod i")`` forced gives ``Carod i`` where 1.4.0 and 2.3.0 gave ``Carod I`` -- a connective keeps its lower case wherever it stands. A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) + - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). Case repair follows the reading: a lower-case ``i`` the parse read as the generation is still title-cased by ``capitalize(force=True)`` (``Carod i`` gives ``Carod I``, as every release did), while one standing among the name words keeps its lower case as ``y`` always has (``Carod i Rovira`` gives ``Carod i Rovira``, where ``Carod I Rovira`` was the pre-2.4 answer). A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 29f23e49..600bfbde 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -631,8 +631,8 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], # rules.md#P3: "a word the rest of the parse reads as a name word -# rather than as a credential or an honorific, looked for past any -# run of connectives standing between" (#397) +# rather than as a generation, a credential or an honorific, looked +# for past any run of connectives standing between" (#397) def _name_word_beside(k: int, step: int, lo: int, hi: int, pieces: Sequence[Piece], ptags: Sequence[Set[str]], tokens: Sequence[WorkToken]) -> bool: diff --git a/nameparser/_render.py b/nameparser/_render.py index 435dec43..0b95522b 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -219,11 +219,27 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # repaired as ordinary name words, since none of them is doing a # particle's work there" -- UNJOINED_TAG is that mark (#407). # Only the PARTICLE conjunct is gated on it, and that is the rule - # rather than an omission -- rules.md#R4: "A CONJUNCTION keeps its - # lowercase wherever it stands, including inside a part whose - # other words the unjoined mark has turned into ordinary name - # words" -- so a conjunction keeps conjunction treatment even - # inside a part the mark has turned into ordinary name words. + # rather than an omission -- rules.md#R4: "A CONNECTIVE the parse + # placed among the name words keeps its lowercase wherever it + # stands there, including inside a part whose other words the + # unjoined mark has turned into ordinary name words" -- so a + # conjunction keeps conjunction treatment even inside a part the + # mark has turned into ordinary name words. + # The ROLE test is the other half of that sentence, and it is why + # a single comparison stands ahead of both conjunction arms: a + # word this vocabulary holds can ALSO be the generation it spells + # ('i' is the Catalan link and the roman numeral), and where the + # parse read the generation the token still carries the + # `conjunction` tag classify gave it -- so without the test, + # `parse("John Quincy Smith i").capitalized(force=True)` gave + # 'John Quincy Smith i' where every release through 2.3 gave + # 'John Quincy Smith I' (#397 review). A suffix-roled token is + # repaired as the suffix it was read as, which is the rest of + # R4's sentence: "one the parse read as the generation it also + # spells is not a connective of this name at all". The test is + # the ROLE and not the suffix VOCABULARY, because the role is + # what the parse decided and the vocabulary is only what the word + # could have been (mechanisms.md#RENDER-HONORS-THE-PARSE). # No SHIPPED name witnesses the difference: `particles` and # `conjunctions` are disjoint in the default vocabulary and in # every locale pack, so no shipped conjunction can sit in an @@ -280,9 +296,10 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # the same words parsed gave 'E-F' (#463 review). if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY) and UNJOINED_TAG not in tags) - or "conjunction" in tags - or (UNCLASSIFIED_TAG in tags - and _reads_as_conjunction(word, lex))): + or (role is not Role.SUFFIX + and ("conjunction" in tags + or (UNCLASSIFIED_TAG in tags + and _reads_as_conjunction(word, lex))))): return word.lower() # v1 cap_word tries the edge-stripped form, then the period-free # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map) diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 017e2b35..adc54b71 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -18,8 +18,10 @@ from nameparser import ( DEFAULT_SCRIPT_ORDERS, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, - GIVEN_FIRST, Lexicon, Parser, PatronymicRule, Policy, Script, parse, + GIVEN_FIRST, HumanName, Lexicon, Parser, PatronymicRule, Policy, + Script, parse, ) +from nameparser.config import Constants from nameparser._lexicon import _VOCAB_FIELDS from nameparser._pipeline import run from nameparser._pipeline._state import ParseState @@ -1468,3 +1470,96 @@ def test_a_trailing_credential_never_joins_into_a_name_part() -> None: assert not failures, ( f"{len(failures)} credential(s) joined into a name part:\n" + "\n".join(failures[:10])) + + +def _placed_as_a_connective(on: ParsedName, + letters: frozenset[str]) -> bool: + """Whether the parse put a class letter among the NAME words as a + connective -- rules.md#R4's own reading, and the one thing an + off-switch comparison of case repair cannot hold fixed: a + connective keeps its lowercase there and the off-switch parse, + where the letter is no connective at all, capitalizes it. Asked + of the token's WORDS rather than of its whole text, because a + merged piece renders word by word and repair asks per word.""" + for tok in on.tokens: + if tok.role is Role.SUFFIX or "conjunction" not in tok.tags: + continue + if letters & {w.strip(".,").lower() for w in tok.text.split()}: + return True + return False + + +def _v1_off_switch(letters: frozenset[str]) -> Constants: + constants = Constants() + for letter in letters: + constants.conjunctions.remove(letter) + return constants + + +def _v1_capitalized(text: str, constants: Constants | None, + force: bool) -> str: + name = HumanName(text) if constants is None else HumanName(text, + constants) + name.capitalize(force=force) + return str(name) + + +def test_a_letter_that_did_not_join_repairs_as_the_off_switch_does() -> None: + """INV7 (#397 review). Case repair is the third view, and the one + the differential cannot see at all, so it gets the same treatment + INV6 gives the fields: turning the class letter OFF is the + parent's reading, so a letter that became no connective of this + name must leave every repair where the parent left it, plain and + forced, on both surfaces. + + Three exemptions. The first two are INV6's -- a name whose FIELDS + moved is a name the join changed, and a letter the two parses + disagree about being an INITIAL is the marked-subset clause. The + third is rules.md#R4's own sentence: a connective the parse + placed among the name words keeps its lowercase there, which the + off-switch parse cannot agree with, since for it the letter is + not a connective at all. What is left is the generation, and the + rule for it is that it repairs as the generation it was read as. + + Mutation-checked, 2026-09-21: this fails on 11,341 repairs at + e540d4c5, where the suffix-roled letter still took the connective + conjunct, and on 0 here; removing the role test alone fails it on + the same 11,341. The v1 arm runs on the default-lexicon, + default-policy rows, the only ones a `Constants` can express, and + 917 of those failures are its. + """ + failures = [] + v1_off: dict[frozenset[str], Constants] = {} + for text, parser, label in _OFF_SWITCH_GRID: + letters = _present(text, _class_letters(parser.lexicon)) + if not letters: + continue + on = parser.parse(text) + off_parser = _off_switch(parser, letters) + off = off_parser.parse(text) + if on.as_dict() != off.as_dict(): + continue + if _initial_reading_moved(on, off, letters): + continue + if _placed_as_a_connective(on, letters): + continue + for force in (False, True): + here = str(parser.capitalized(on, force=force)) + there = str(off_parser.capitalized(off, force=force)) + if here != there: + failures.append( + f"[{label}] {text!r} force={force}: {here!r} != " + f"off-switch {there!r}") + if label != "default/default": + continue + if letters not in v1_off: + v1_off[letters] = _v1_off_switch(letters) + v1_here = _v1_capitalized(text, None, force) + v1_there = _v1_capitalized(text, v1_off[letters], force) + if v1_here != v1_there: + failures.append( + f"[v1] {text!r} force={force}: {v1_here!r} != " + f"off-switch {v1_there!r}") + assert not failures, ( + f"{len(failures)} repair(s) moved for a letter that joined " + f"nothing:\n" + "\n".join(failures[:10])) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index a49f6032..1eecb79b 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -253,6 +253,44 @@ def test_repair_keeps_a_lone_connective_lowercase_where_it_initials( assert parse("juan y").capitalized(force=True).family == "y" +def test_repair_capitalizes_a_generation_the_connective_also_spells( +) -> None: + """rules.md#R4's other half (#397 review), and the row that made + the sentence need two clauses. + + A word can be the connective and the generation at once -- 'i' is + the Catalan link and the roman numeral -- and where the parse read + the GENERATION the token still carries the `conjunction` tag + classify gave it. Repair reads the ROLE the parse decided, not the + tag alone, so a suffix-roled letter is repaired as the suffix it + was read as. Without that test these gave 'John Quincy Smith i' + and 'Carod i', where every release through 2.3 gave the capital. + + Both surfaces, because the v1 facade repairs through this same + helper and a fix on one of them would be a split. + """ + assert parse("John Quincy Smith i").suffix == "i" + assert str(parse("John Quincy Smith i").capitalized( + force=True)) == "John Quincy Smith I" + assert str(parse("Carod i").capitalized(force=True)) == "Carod I" + assert str(parse("Josep Lluis Carod i III").capitalized( + force=True)) == "Josep Lluis Carod I III" + v1 = HumanName("John Quincy Smith i") + v1.capitalize(force=True) + assert str(v1) == "John Quincy Smith I" + v1_two = HumanName("Carod i") + v1_two.capitalize(force=True) + assert str(v1_two) == "Carod I" + # the contrast that keeps the role test honest: the SAME letter + # in a NAME part is a connective and keeps its lowercase, which + # is the answer 'y' has always had there + assert parse("Josep i Rovira").middle == "i" + assert str(parse("Josep i Rovira").capitalized( + force=True)) == "Josep i Rovira" + assert str(parse("Josep y Rovira").capitalized( + force=True)) == "Josep y Rovira" + + def test_initials_order_folded_words_first_like_the_family_field() -> None: """#408: the view and the field must read one parse the same way. From 0fbcaa0b0829111d7eb26dcf44cb4e1a78f353c8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 09:09:47 -0700 Subject: [PATCH 07/11] test(#397/#461): the invariant tests share one parsed grid The seven invariants this PR added took 104s of the property module's 115s locally, and carried CI's build jobs from about 5 minutes to 17-25 (3.11 17m26s, 3.12 25m29s, 3.13 16m32s) -- every parse costs several times more under the coverage tracing CI runs, and every later PR pays it. Two causes, both multiplicative, neither of them a shape the tests needed. FOUR TESTS, ONE PARSE. The four connective invariants asked four questions of the same 170,100-row grid and parsed it four times over; the three off-switch invariants parsed every row of theirs twice apiece, six parses of a row. Each grid is now walked ONCE -- `_connective_findings`, `_off_switch_findings` -- and each test reads its own failure list out of that walk: same rows, same order, same message, every predicate left where it was and spelled in the terms its own rule is stated in. The walk decides nothing; it only asks them all while the parse is in hand. ROWS THAT COULD NOT DIFFER. Both grids were a full cross product of texts x lexicons x policies, and most of it was one configuration re-deriving another's answer: a lexicon that ADDS a word changes no parse of a name that does not hold that word, and lenient_comma_suffixes decides nothing in a name with no comma. Every variant now declares the words that reach it and takes only the texts holding them (`_rows`). The TEXTS are untouched -- 14,175 and 12,880, the two pinned counts that did NOT move, which is the number that would say a shape was dropped -- and the rows fall 170,100 -> 55,800 and 103,040 -> 53,312. A shape trim and not a sample, measured before it was taken: over the full cross product of both grids, 177,980 collapsed rows, 0 of them differing from the row they collapse onto in fields, token tags and spans, initials() on every group, the four derived views, capitalized() plain and forced, or the reports. And self-maintaining, which a hand-picked subset would not be: no text of the connective grid holds a standalone 'v', so conj+v earns no row there until a generator grows one, and then earns them again by itself. Measured locally, py3.11, this machine: the seven tests 103.7s -> 14.8s (7.0x); tests/v2/test_properties.py 114.9s -> 25.6s; the whole suite 134.6s -> 44.1s; the suite under --cov=nameparser, which is what CI pays, 383.1s -> 122.2s (3.1x), with the coverage table byte-identical -- 100% still on _group.py, _render.py, _post_rules.py and _types.py. Counts unchanged at 9557 passed / 324 skipped / 4 xfailed. FAULT DETECTION IS PRESERVED, and a trim that is not proven to be is worth nothing. Twelve mutants of the code this arc got wrong, plus the two historical commits the invariants were written against, run against a scratch copy of the package; every one that killed a test before still kills the same test, count before -> after: M-a freeze gate tests POSITION (the c8550b64 bug) INV6 1120 -> 610, INV1-strengthened 960 -> 534 M-b count drops the frozen exclusion INV6 640 -> 342 M-c both-sides condition dropped whole INV1 504 -> 162, INV6 3920 -> 2070, INV1-str 960 -> 534 M-d _cap_word drops `role is not SUFFIX` (the e540d4c5 bug) INV7 11,341 -> 7,382 M-e lone-part mark never written INV2 14,078 -> 4,639 M-f mark walk's branches swapped INV3/4 188 -> 94 M-g given-group exemption restored INV5 27,103 -> 9,308 M-h _UNJOINED_MARKS narrowed to one INV5 14,078 -> 4,639 M-i facade predicate ignores the mark survives both (v1 initials are no view of these seven; test_facade.py owns it) M-j _name_word_beside ignores is_title_piece survives both M-k ... ignores the `hi` bound INV6 640 -> 382, INV1-strengthened 640 -> 382 M-l ... ignores the `lo` bound survives both c8550b64 INV6 2,360 -> 1,250, INV1-str 960 -> 534, INV7 8,518 -> 5,691 e540d4c5 INV7 11,341 -> 7,382, of which the v1 arm's 917 both before and after -- those rows were never duplicated No mutant had to have shapes put back. The counts in the invariants' docstrings are counts over the grid, so every one of them is re-measured here and re-recorded there beside the figure the full cross product gave; INV1's two stale ones, describing a gate the review replaced at 41de3df6, are restated as what today's tree does. AGENTS.md carries both rules now, next to the guard-test convention: an invariant over an existing grid joins that grid's walk, a new grid states its runtime, and a grid is trimmed by shape with the duplicate rows proven duplicate first. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 + tests/v2/test_properties.py | 520 ++++++++++++++++++++++-------------- 2 files changed, 326 insertions(+), 196 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 14f83214..52174566 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,8 @@ Three committed contributor docs carry the parser's normative rules and their re **Guard tests** SHOULD carry a recorded negative control — the answer with the guard off, stored as data (the _EXCLUSION_EFFECT shape; see mechanisms.md's Verification shapes). +**Invariant tests over a generated grid share ONE parsed grid, and a grid pairs a text only with configurations that can read it differently.** Both rules are in `tests/v2/test_properties.py` (`_connective_findings`, `_off_switch_findings`, `_rows`) and both were bought at a price: four tests each walking the same 170,100-row grid, and a grid that was a full cross product of texts × lexicons × policies, together took that module from 11s to 115s and CI's build jobs from ~5 minutes to 17–25 (under coverage every parse costs several times more, and every later PR pays it). A new invariant over an existing grid joins that grid's walk instead of opening its own; a new grid states its measured runtime where it is built. Trim a grid by SHAPE — a variant lexicon reaches only the texts holding the word it adds — never by sampling, and prove the dropped rows were duplicates by signature before dropping them. Mutation counts in the tests' docstrings are counts over the grid, so re-measure them whenever the grid moves. + ## Commands ```bash diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index adc54b71..3fb21815 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -7,6 +7,7 @@ against regressions; exploratory fuzzing happened during review. """ import dataclasses +import functools import hashlib import itertools import re @@ -1002,6 +1003,96 @@ def test_bad_policy_field_fails_cleanly(field: str, value: object) -> None: # mixed/ALL-CAPS/lower, class members supplied by a CUSTOM lexicon # (a second single letter that is also generational vocabulary, and a # connective that is also particle vocabulary), and four policies. +# +# ONE PARSE, MANY CHECKS (2026-09-20). The four invariants below ask +# four questions of the SAME parse, so the grid is walked ONCE, in +# `_connective_findings`, and each test reads its own answers out of +# that walk. Four tests each walking the grid for themselves is what +# took this module from 11s to 115s and CI's build jobs from about 5 +# minutes to 17-25 (every parse costs several times more under +# coverage tracing, which is what CI runs). A new invariant over this +# grid joins the walk rather than opening a fifth. + + +#: A grid's configuration variants: (label, value, the features of a +#: text that reach it). See `_rows`. +_Variants = tuple[tuple[str, Lexicon, frozenset[str]], ...] + +#: The policies both grids below run, each with the feature a text +#: must hold for the policy to be distinguishable at all -- see +#: `_rows`. +_GRID_POLICIES: tuple[tuple[str, Policy, frozenset[str]], ...] = ( + ("default", Policy(), frozenset()), + ("family-first", Policy(name_order=FAMILY_FIRST), frozenset()), + ("given-last", Policy(name_order=FAMILY_FIRST_GIVEN_LAST), + frozenset()), + # lenient_comma_suffixes decides what a COMMA hands to the suffix + # run, and a name without one has no such hand-off to decide + ("strict-comma", Policy(lenient_comma_suffixes=False), + frozenset({","})), +) + + +def _reaching(text: str) -> frozenset[str]: + """What of a text a configuration variant could read at all: its + words, folded the way the vocabulary matches them, plus ',' when + it carries one.""" + out = {w.strip(".,").lower() for w in text.split()} + if "," in text: + out.add(",") + return frozenset(out) + + +def _rows(texts: list[str], + lexicons: _Variants) -> list[tuple[str, Parser, str]]: + """Every text against the parsers that can read it DIFFERENTLY. + + A variant that ADDS vocabulary changes no parse of a name not + containing the added word, and `lenient_comma_suffixes` decides + nothing in a name with no comma, so most of a full cross product + is one configuration re-deriving another's answer. Each variant + declares the words that reach it and is paired with the texts + holding them. + + A SHAPE trim and not a sample -- the TEXTS are untouched, and so + is every shape class among them; what goes is duplicate pairings. + Measured 2026-09-20 over the full cross product of both grids: + 177,980 collapsed rows, 0 of them differing from the row they + collapse onto in fields, token tags and spans, `initials()` on + every group, the four derived views, `capitalized()` plain and + forced, or the reports. Reproduce with the signature comparison + in the commit that introduced this (`test(#397/#461): the + invariant tests share one parsed grid`). + + Self-maintaining, which a hand-picked subset would not be: give a + generator below a 'v' word and the conj+v rows come back on their + own. That is the one to know about -- no text of the connective + grid holds a standalone 'v', so `conj+v` earns no row there and + earns them in the off-switch grid, whose suffixes include 'V'. + """ + parsers = [(f"{ln}/{pn}", Parser(lexicon=lex, policy=pol), + lex_need | pol_need) + for ln, lex, lex_need in lexicons + for pn, pol, pol_need in _GRID_POLICIES] + out: list[tuple[str, Parser, str]] = [] + for text in texts: + reach = _reaching(text) + out += [(text, parser, label) + for label, parser, need in parsers if need <= reach] + return out + + +@functools.cache +def _class_letters(lexicon: Lexicon) -> frozenset[str]: + """The class rules.md#P3's both-sides condition is about: a + one-letter connective that is ALSO generational vocabulary. + + Cached on the lexicon -- a frozen, hashable value -- because both + grids ask this of every row and there are five lexicons between + them. + """ + return frozenset(w for w in lexicon.conjunctions + if len(w) == 1 and w in lexicon.suffix_words) def _connective_grid() -> list[tuple[str, Parser, str]]: @@ -1014,13 +1105,12 @@ def _connective_grid() -> list[tuple[str, Parser, str]]: [], ["Rovira"], ["de", "Rovira"], ["Rovira", "Puig"]) suffixes: tuple[list[str], ...] = ( [], ["III"], ["Jr."], ["I"], ["MA"]) - lexicons = (("default", Lexicon.default()), - ("conj+v", Lexicon.default().add(conjunctions={"v"})), - ("part+y", Lexicon.default().add(particles={"y"}))) - policies = (("default", Policy()), - ("family-first", Policy(name_order=FAMILY_FIRST)), - ("given-last", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)), - ("strict-comma", Policy(lenient_comma_suffixes=False))) + lexicons: _Variants = ( + ("default", Lexicon.default(), frozenset()), + ("conj+v", Lexicon.default().add(conjunctions={"v"}), + frozenset({"v"})), + ("part+y", Lexicon.default().add(particles={"y"}), + frozenset({"y"}))) texts: list[str] = [] seen: set[str] = set() for head, mid, conn, tail, suffix, comma in itertools.product( @@ -1036,9 +1126,7 @@ def _connective_grid() -> list[tuple[str, Parser, str]]: if written not in seen: seen.add(written) texts.append(written) - parsers = [(f"{ln}/{pn}", Parser(lexicon=lex, policy=pol)) - for ln, lex in lexicons for pn, pol in policies] - return [(t, p, label) for t in texts for label, p in parsers] + return _rows(texts, lexicons) _CONNECTIVE_GRID = _connective_grid() @@ -1072,6 +1160,79 @@ def _predicted_initials(part: tuple[Token, ...], role: Role) -> list[str]: return out +@functools.cache +def _connective_findings() -> dict[str, list[str]]: + """One walk of the connective grid; four invariants' answers. + + Each key below is one test's failure list, built with the parse + in hand and in grid order, so a test reads exactly what it would + have found walking the grid itself -- same rows, same order, same + message. What the walk does NOT do is decide anything: every + predicate stays where it was, spelled in the terms its own rule + is stated in, and this function only asks them all at once. + """ + out: dict[str, list[str]] = {k: [] for k in + ("INV1", "INV2", "INV3/4", "INV5")} + for text, parser, label in _CONNECTIVE_GRID: + letters = _class_letters(parser.lexicon) + name = parser.parse(text) + for role in _NAME_ROLES: + part = name.tokens_for(role) + # hoisted out of the INV3/INV4 comprehensions below, where + # it used to be recomputed once per token of the part + predicted = _predicted_initials(part, role) + if len(part) >= 2: + for i, tok in enumerate(part): + if ("conjunction" not in tok.tags + or len(tok.text) != 1 + or tok.text.lower() not in letters): + continue + left = any("conjunction" not in t.tags + for t in part[:i]) + right = any("conjunction" not in t.tags + for t in part[i + 1:]) + if not (left and right): + out["INV1"].append( + f"[{label}] {text!r}: {role.value} " + f"{tok.text!r}") + for tok in part: + if "conjunction" not in tok.tags: + continue + joinable = _has_something_to_join(tok, part) + if joinable == _readmitted(tok): + out["INV2"].append( + f"[{label}] {text!r}: {role.value} {tok.text!r} " + f"joinable={joinable} " + f"marked={_readmitted(tok)}") + got = name.initials( + f"{{{role.value}}}").replace(".", "").split() + if got != predicted: + out["INV5"].append( + f"[{label}] {text!r}: {role.value} {got!r} != " + f"predicted {predicted!r}") + if role is not Role.FAMILY: + continue + base = name.family_base.split() + contributing = [t for t in part + if t.text[0] in predicted + and (not ("conjunction" in t.tags + or "particle" in t.tags) + or _readmitted(t))] + for tok in contributing: + if tok.text not in base: + out["INV3/4"].append( + f"[{label}] {text!r}: INV3 {tok.text!r} initials " + f"but is not in base {base!r}") + for tok in part: + if (tok.text in base and tok not in contributing + and "conjunction" not in tok.tags): + out["INV3/4"].append( + f"[{label}] {text!r}: INV4 {tok.text!r} is a base " + f"word, contributes no initial, and is no " + f"connective") + return out + + def test_the_connective_grid_can_fail() -> None: """The reachability probe every grid in this file carries. @@ -1079,13 +1240,21 @@ def test_the_connective_grid_can_fail() -> None: vacuously, and a comparison over a population of zero is the same silence as a clean run. These three counts are a dated recorded control, measured 2026-09-20 on the shipped tree. + + The row count fell from 170,100 to 55,800 when `_rows` stopped + pairing a text with configurations that cannot read it; the TEXT + count did not move, and that is the number to watch here, because + it is the one that says a shape was dropped. 1,580 of the first + 2,000 rows join, against 1,280 of the old grid's first 2,000 -- + the same texts, reached sooner, the old slice having held twelve + rows per text where this one holds three or four. """ - assert len(_CONNECTIVE_GRID) == 170100, len(_CONNECTIVE_GRID) + assert len(_CONNECTIVE_GRID) == 55800, len(_CONNECTIVE_GRID) assert len({t for t, _, _ in _CONNECTIVE_GRID}) == 14175 joined = sum(1 for text, parser, _ in _CONNECTIVE_GRID[:2000] if any("conjunction" in tok.tags for tok in parser.parse(text).tokens)) - assert joined > 500, joined + assert joined > 1000, joined def test_a_generational_connective_joins_only_with_both_sides() -> None: @@ -1093,30 +1262,17 @@ def test_a_generational_connective_joins_only_with_both_sides() -> None: generational vocabulary: if it shares its role part with any other word, a non-connective word stands before it AND after it. - Mutation-checked, 2026-09-20: dropping the position test fails - this on 1575 parses, dropping the whole both-sides condition on - 504, and dropping the rootname count arm on 816. The CLASS test + Mutation-checked, 2026-09-20, re-measured on the trimmed grid: + dropping the whole both-sides condition fails this on 162 parses, + where the full cross product gave 504. The gate's other two + failure modes stay invisible here and are INV6's and INV1- + strengthened's to catch -- testing POSITION instead of class, and + dropping the count's frozen exclusion, each leave this invariant + green on every row, which is why those two exist. The CLASS test is NOT covered here and has its own rows -- see tests/v2/pipeline/test_group.py. """ - failures = [] - for text, parser, label in _CONNECTIVE_GRID: - letters = {w for w in parser.lexicon.conjunctions - if len(w) == 1 and w in parser.lexicon.suffix_words} - name = parser.parse(text) - for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): - part = name.tokens_for(role) - if len(part) < 2: - continue - for i, tok in enumerate(part): - if ("conjunction" not in tok.tags or len(tok.text) != 1 - or tok.text.lower() not in letters): - continue - left = any("conjunction" not in t.tags for t in part[:i]) - right = any("conjunction" not in t.tags for t in part[i + 1:]) - if not (left and right): - failures.append( - f"[{label}] {text!r}: {role.value} {tok.text!r}") + failures = _connective_findings()["INV1"] assert not failures, ( f"{len(failures)} parse(s) joined a generational connective " f"without a name word on each side:\n" + "\n".join(failures[:10])) @@ -1128,23 +1284,15 @@ def test_the_mark_is_exactly_the_parts_with_nothing_to_join() -> None: working particle. Stated over the PAIR of marks deliberately: keyed on the new mark - alone it fails 366 times under `add(particles={"y"})`, where the - part is all-particle and R2's OLD mark does the readmitting. - Mutation-checked: dropping the new mark fails this on 14,066. + alone it fails 272 times, every one of them under + `add(particles={"y"})`, where the part is all-particle and R2's + OLD mark does the readmitting. Mutation-checked, re-measured + 2026-09-20 on the trimmed grid: neutering the walk's + lone-connective arm, so the new mark is never written, fails this + on 4,639 rows -- 14,078 of the full cross product, which is what + the trim costs a count and not what it costs detection. """ - failures = [] - for text, parser, label in _CONNECTIVE_GRID: - name = parser.parse(text) - for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): - part = name.tokens_for(role) - for tok in part: - if "conjunction" not in tok.tags: - continue - if _has_something_to_join(tok, part) == _readmitted(tok): - failures.append( - f"[{label}] {text!r}: {role.value} {tok.text!r} " - f"joinable={_has_something_to_join(tok, part)} " - f"marked={_readmitted(tok)}") + failures = _connective_findings()["INV2"] assert not failures, ( f"{len(failures)} connective token(s) disagree with the " f"criterion:\n" + "\n".join(failures[:10])) @@ -1159,32 +1307,13 @@ def test_initials_take_only_base_words_and_drop_only_joiners() -> None: one parse cannot come apart again, which is the defect #461 was filed about. - Mutation-checked: swapping the two branches of the mark walk fails - this on 188 parses (`Carod y` under `add(particles={"y"})`, whose - base is empty while its `y` initials). + Mutation-checked, re-measured 2026-09-20 on the trimmed grid: + swapping the two branches of the mark walk fails this on 94 + parses (188 over the full cross product) -- `Carod y` under + `add(particles={"y"})`, whose base is empty while its `y` + initials. """ - failures = [] - for text, parser, label in _CONNECTIVE_GRID: - name = parser.parse(text) - family = name.tokens_for(Role.FAMILY) - base = name.family_base.split() - contributing = [t for t in family - if t.text[0] in _predicted_initials(family, - Role.FAMILY) - and (not ("conjunction" in t.tags - or "particle" in t.tags) - or _readmitted(t))] - for tok in contributing: - if tok.text not in base: - failures.append( - f"[{label}] {text!r}: INV3 {tok.text!r} initials but " - f"is not in base {base!r}") - for tok in family: - if (tok.text in base and tok not in contributing - and "conjunction" not in tok.tags): - failures.append( - f"[{label}] {text!r}: INV4 {tok.text!r} is a base " - f"word, contributes no initial, and is no connective") + failures = _connective_findings()["INV3/4"] assert not failures, ( f"{len(failures)} disagreement(s) between initials() and " f"family_base:\n" + "\n".join(failures[:10])) @@ -1196,21 +1325,13 @@ def test_initials_emit_exactly_the_predicted_contributors() -> None: The output-level statement of the same rule, and the one that catches a view reading the marks correctly and then rendering - something else. Mutation-checked: restoring the given group's old - blanket exemption fails this on 27,343 parses, and dropping the - new mark from the readmitting set on 14,066. + something else. Mutation-checked, re-measured 2026-09-20 on the + trimmed grid: restoring the given group's old blanket exemption + (`_SKIP_TAGS_GIVEN` back to the empty set) fails this on 9,308 + parses, and dropping the new mark from the readmitting set on + 4,639 -- 27,103 and 14,078 over the full cross product. """ - failures = [] - for text, parser, label in _CONNECTIVE_GRID: - name = parser.parse(text) - for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): - predicted = _predicted_initials(name.tokens_for(role), role) - got = name.initials( - f"{{{role.value}}}").replace(".", "").split() - if got != predicted: - failures.append( - f"[{label}] {text!r}: {role.value} {got!r} != " - f"predicted {predicted!r}") + failures = _connective_findings()["INV5"] assert not failures, ( f"{len(failures)} group(s) emitted something other than the " f"criterion's contributors:\n" + "\n".join(failures[:10])) @@ -1227,6 +1348,10 @@ def test_initials_emit_exactly_the_predicted_contributors() -> None: # peel takes ('V', 'i', beside the 'III'/'Jr.'/'I'/'MA' both carry), # a maiden clause, a head that is nothing but an initial or a # particle, and the SUFFIX comma as a third comma shape. +# +# The same ONE PARSE, MANY CHECKS rule as the grid above, and here it +# buys twice as much: the three invariants share BOTH parses of every +# row -- six parses became two -- in `_off_switch_findings`. def _off_switch_grid() -> list[tuple[str, Parser, str]]: @@ -1238,12 +1363,10 @@ def _off_switch_grid() -> list[tuple[str, Parser, str]]: tails: tuple[list[str], ...] = ([], ["Rovira"], ["de", "Rovira"]) suffixes: tuple[list[str], ...] = ( [], ["III"], ["Jr."], ["I"], ["V"], ["MA"], ["i"], ["nee", "Puig"]) - lexicons = (("default", Lexicon.default()), - ("conj+v", Lexicon.default().add(conjunctions={"v"}))) - policies = (("default", Policy()), - ("family-first", Policy(name_order=FAMILY_FIRST)), - ("given-last", Policy(name_order=FAMILY_FIRST_GIVEN_LAST)), - ("strict-comma", Policy(lenient_comma_suffixes=False))) + lexicons: _Variants = ( + ("default", Lexicon.default(), frozenset()), + ("conj+v", Lexicon.default().add(conjunctions={"v"}), + frozenset({"v"}))) texts: list[str] = [] seen: set[str] = set() for head, mid, conn, tail, suffix, comma in itertools.product( @@ -1265,23 +1388,13 @@ def _off_switch_grid() -> list[tuple[str, Parser, str]]: if written not in seen: seen.add(written) texts.append(written) - parsers = [(f"{ln}/{pn}", Parser(lexicon=lex, policy=pol)) - for ln, lex in lexicons for pn, pol in policies] - return [(t, p, label) for t in texts for label, p in parsers] + return _rows(texts, lexicons) _OFF_SWITCH_GRID = _off_switch_grid() -_NAME_ROLES = (Role.GIVEN, Role.MIDDLE, Role.FAMILY) _OFF_PARSERS: dict[tuple[int, frozenset[str]], Parser] = {} -def _class_letters(lexicon: Lexicon) -> frozenset[str]: - """The class rules.md#P3's both-sides condition is about: a - one-letter connective that is ALSO generational vocabulary.""" - return frozenset(w for w in lexicon.conjunctions - if len(w) == 1 and w in lexicon.suffix_words) - - def _off_switch(parser: Parser, letters: frozenset[str]) -> Parser: """The same parser with those letters out of the connectives -- the parent-equivalent reading, where nothing can have joined. @@ -1371,10 +1484,94 @@ def _link_joins_between_name_words(on: ParsedName, off: ParsedName, return False +@functools.cache +def _off_switch_findings() -> dict[str, list[str]]: + """One walk of the off-switch grid; three invariants' answers. + + Both parses of a row -- as configured, and with the class letters + out of the connectives -- are taken once here and handed to all + three predicates, which is the whole of what this walk does. The + exemptions stay each test's own: INV6 takes the join and the + initial reading, INV1-strengthened only the initial reading, INV7 + those two plus R4's placed-connective sentence, exactly as their + docstrings say. + """ + out: dict[str, list[str]] = {k: [] for k in + ("INV6", "INV1-strengthened", "INV7")} + v1_off: dict[frozenset[str], Constants] = {} + for text, parser, label in _OFF_SWITCH_GRID: + letters = _present(text, _class_letters(parser.lexicon)) + if not letters: + continue + on = parser.parse(text) + off_parser = _off_switch(parser, letters) + off = off_parser.parse(text) + # the SEVEN FIELDS, and not comparison_key: a parse carries + # more than its fields, and what the off-switch legitimately + # moves besides them is the conjunction-or-initial report + same_fields = on.as_dict() == off.as_dict() + moved = _initial_reading_moved(on, off, letters) + if not (same_fields + or _link_joins_between_name_words(on, off, letters) + or moved): + out["INV6"].append(f"[{label}] {text!r}: {on.as_dict()} != " + f"off-switch {off.as_dict()}") + if not moved: + off_role = _off_roles(off) + for role in _NAME_ROLES: + part = on.tokens_for(role) + if len(part) < 2: + continue + for tok in part: + # not the LINK itself: it is generational + # vocabulary by definition of the class, so the + # off-switch parse reads it as the suffix in every + # name it ends. This rule is about the OTHER word + # -- the credential a link must not take with it. + # `span is None` is a typing guard and nothing + # else: every token of a PARSER-produced name + # carries one, and this grid holds no spliced + # parse. + if "conjunction" in tok.tags or tok.span is None: + continue + if off_role.get(tok.span) is Role.SUFFIX: + out["INV1-strengthened"].append( + f"[{label}] {text!r}: {tok.text!r} reads as " + f"the suffix and joined into {role.value}") + if not (same_fields and not moved + and not _placed_as_a_connective(on, letters)): + continue + for force in (False, True): + here = str(parser.capitalized(on, force=force)) + there = str(off_parser.capitalized(off, force=force)) + if here != there: + out["INV7"].append( + f"[{label}] {text!r} force={force}: {here!r} != " + f"off-switch {there!r}") + if label != "default/default": + continue + if letters not in v1_off: + v1_off[letters] = _v1_off_switch(letters) + v1_here = _v1_capitalized(text, None, force) + v1_there = _v1_capitalized(text, v1_off[letters], force) + if v1_here != v1_there: + out["INV7"].append( + f"[v1] {text!r} force={force}: {v1_here!r} != " + f"off-switch {v1_there!r}") + return out + + def test_the_off_switch_grid_can_fail() -> None: """The reachability probe, the shape every grid in this file - carries. Dated recorded control, measured 2026-09-20.""" - assert len(_OFF_SWITCH_GRID) == 103040, len(_OFF_SWITCH_GRID) + carries. Dated recorded control, measured 2026-09-20. + + The row count fell from 103,040 to 53,312 with `_rows` (see the + connective probe above); the TEXT count is unmoved, and 2,011 of + the first 4,000 rows carry a class letter, against 1,800 of the + old grid's first 4,000. Over the whole grid 25,088 rows carry one + and are parsed twice; the old grid parsed 46,088 twice, six times + over.""" + assert len(_OFF_SWITCH_GRID) == 53312, len(_OFF_SWITCH_GRID) assert len({t for t, _, _ in _OFF_SWITCH_GRID}) == 12880 reached = sum(1 for text, parser, _ in _OFF_SWITCH_GRID[:4000] if _present(text, _class_letters(parser.lexicon))) @@ -1395,27 +1592,13 @@ def test_a_link_that_joins_nothing_changes_no_field() -> None: clause, which the switch decides along with the join and so cannot hold fixed. - Mutation-checked, 2026-09-20: this fails on 2,360 parses at - c8550b64, the commit the review was written against. + Mutation-checked, 2026-09-20: this fails on 1,250 parses at + c8550b64, the commit the review was written against (2,360 over + the full cross product), and on 610 where the both-sides gate + tests POSITION rather than class -- the c8550b64 defect isolated, + which is the one INV1 above cannot see. """ - failures = [] - for text, parser, label in _OFF_SWITCH_GRID: - letters = _present(text, _class_letters(parser.lexicon)) - if not letters: - continue - on = parser.parse(text) - off = _off_switch(parser, letters).parse(text) - # the SEVEN FIELDS, and not comparison_key: a parse carries - # more than its fields, and what the off-switch legitimately - # moves besides them is the conjunction-or-initial report - if on.as_dict() == off.as_dict(): - continue - if _link_joins_between_name_words(on, off, letters): - continue - if _initial_reading_moved(on, off, letters): - continue - failures.append(f"[{label}] {text!r}: {on.as_dict()} != " - f"off-switch {off.as_dict()}") + failures = _off_switch_findings()["INV6"] assert not failures, ( f"{len(failures)} parse(s) moved a field with no link joining " f"anything:\n" + "\n".join(failures[:10])) @@ -1435,38 +1618,11 @@ def test_a_trailing_credential_never_joins_into_a_name_part() -> None: is deliberately absent -- a link joining elsewhere in the name never licenses a credential joining here. - Mutation-checked, 2026-09-20: this fails on 960 parses at - c8550b64, where INV1 fails on none of them. + Mutation-checked, 2026-09-20: this fails on 534 parses at + c8550b64 (960 over the full cross product), where INV1 fails on + none of them. """ - failures = [] - for text, parser, label in _OFF_SWITCH_GRID: - letters = _present(text, _class_letters(parser.lexicon)) - if not letters: - continue - on = parser.parse(text) - off = _off_switch(parser, letters).parse(text) - if _initial_reading_moved(on, off, letters): - continue - off_role = _off_roles(off) - for role in _NAME_ROLES: - part = on.tokens_for(role) - if len(part) < 2: - continue - for tok in part: - # not the LINK itself: it is generational vocabulary - # by definition of the class, so the off-switch parse - # reads it as the suffix in every name it ends. This - # rule is about the OTHER word -- the credential a - # link must not take with it. - # `span is None` is a typing guard and nothing else: - # every token of a PARSER-produced name carries one, - # and this grid holds no spliced parse. - if "conjunction" in tok.tags or tok.span is None: - continue - if off_role.get(tok.span) is Role.SUFFIX: - failures.append( - f"[{label}] {text!r}: {tok.text!r} reads as the " - f"suffix and joined into {role.value}") + failures = _off_switch_findings()["INV1-strengthened"] assert not failures, ( f"{len(failures)} credential(s) joined into a name part:\n" + "\n".join(failures[:10])) @@ -1521,45 +1677,17 @@ def test_a_letter_that_did_not_join_repairs_as_the_off_switch_does() -> None: not a connective at all. What is left is the generation, and the rule for it is that it repairs as the generation it was read as. - Mutation-checked, 2026-09-21: this fails on 11,341 repairs at - e540d4c5, where the suffix-roled letter still took the connective - conjunct, and on 0 here; removing the role test alone fails it on - the same 11,341. The v1 arm runs on the default-lexicon, - default-policy rows, the only ones a `Constants` can express, and - 917 of those failures are its. + Mutation-checked, re-measured 2026-09-20 on the trimmed grid: + this fails on 7,382 repairs at e540d4c5, where the + suffix-roled letter still took the connective conjunct, and on 0 + here; removing the role test alone fails it on the same 7,382. + (11,341 for both over the full cross product.) The v1 arm runs on + the default-lexicon, default-policy rows, the only ones a + `Constants` can express, and 917 of those failures are its -- the + same 917 as before the trim, those rows never having been + duplicated. """ - failures = [] - v1_off: dict[frozenset[str], Constants] = {} - for text, parser, label in _OFF_SWITCH_GRID: - letters = _present(text, _class_letters(parser.lexicon)) - if not letters: - continue - on = parser.parse(text) - off_parser = _off_switch(parser, letters) - off = off_parser.parse(text) - if on.as_dict() != off.as_dict(): - continue - if _initial_reading_moved(on, off, letters): - continue - if _placed_as_a_connective(on, letters): - continue - for force in (False, True): - here = str(parser.capitalized(on, force=force)) - there = str(off_parser.capitalized(off, force=force)) - if here != there: - failures.append( - f"[{label}] {text!r} force={force}: {here!r} != " - f"off-switch {there!r}") - if label != "default/default": - continue - if letters not in v1_off: - v1_off[letters] = _v1_off_switch(letters) - v1_here = _v1_capitalized(text, None, force) - v1_there = _v1_capitalized(text, v1_off[letters], force) - if v1_here != v1_there: - failures.append( - f"[v1] {text!r} force={force}: {v1_here!r} != " - f"off-switch {v1_there!r}") + failures = _off_switch_findings()["INV7"] assert not failures, ( f"{len(failures)} repair(s) moved for a letter that joined " f"nothing:\n" + "\n".join(failures[:10])) From dc3bdf9cb5493a7ca254496452a8c7dc21522194 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 10:58:56 -0700 Subject: [PATCH 08/11] fix(#397): a link inside a maiden clause stays in the birth name The maiden walk ends the birth name at the first suffix WORD after the marker, and the Catalan link is also the roman numeral, so `Jane Doe nee Puig i Soler` read maiden 'Puig' and left 'i Soler' standing. That truncation predates this PR: master 46651750 read middle 'Doe i', family 'Soler'. What this PR added is the harm -- with the link joining, the leaked words merged into the CURRENT surname, and 0fbcaa0b read family 'Doe i Soler'. The comma form leaked into the given part (`Doe, Jane nee Puig i Soler` -> given 'Jane i Soler') and a Catalan head lost the distinction between its own link and the clause's (`Maria Carod i Rovira nee Puig i Soler` -> family 'Carod i Rovira i Soler'). The spec's Q3 called this interaction unchanged; it was written from the rule, not from a parse. Derek decided 2026-09-20 to fix it here. The walk now steps over a suffix piece that is a connective PLACED TO JOIN inside the clause, through the join's own predicates (`_is_conj_piece`, `_name_word_beside`) rather than a restatement of them. Two bounds make it the clause's question: the left is the first piece after the whole marker run, so the marker is no name word; the right is where assign's trailing run begins over the pieces as written, read off the peel pair already in hand. After: maiden 'Puig i Soler' / 'Kowalska i Nowak', family 'Doe' and 'Carod i Rovira'. Controls unmoved -- `... nee Puig i` keeps suffix 'i', `... i III` suffix 'i III', `... i MA` suffix 'i MA', `... nee i Soler` declines, and the capital `I` spellings and the delimited form were never truncated. `remove(conjunctions={"i"})` restores master's reading. Two invariants over a new 15,810-row clause grid (4,590 texts; five heads, all seventeen markers, six bodies, five tails, comma and none; four policies), sharing one parsed grid per AGENTS.md. THE y TWIN fails on all 10,540 guarded rows at 0fbcaa0b and at 46651750, 0 here. NO BIRTH-NAME WORD IN THE CURRENT NAME fails on 28,985 tokens at 0fbcaa0b against 22,185 at 46651750 -- the gap is what this branch added -- and 0 here. INV6's exemption is NOT widened: 1,680 off-switch rows carry a marker and a class letter, 0 put the letter after the marker. Eight mutations, no survivors. Four case rows, ten unit tests, three rules.md example lines. All five differential gates exit 0 with no unexplained diff: one new rule per ledger plus `Jane Doe nee Puig i III` joining the existing fix(#436/#437) alternation at the three 2.x baselines, with _MUST_NOT_MATCH probes and re-recorded _CORPUS_CLAIMS (+4/+1/+1 on three 1.4.0 rules, 14->15 on the 2.x run rule, verified name by name). Frames: `Jane Doe nee Smith` 234 unchanged, `... Smith MA` 361 unchanged, `Josep Carod i Rovira` 303 unchanged, `John Smith` 164 unchanged; the moved names fall (371->308, 384->331) and the controls rise by seven (331->338, 391->399). call_count parse=406.00 facade=443.00 unchanged; `'Puig i ' * k` linear (x1.86/x2.03/x1.98 per doubling, k=50..400); test_properties 25.6s -> 29.9s; nameparser/_pipeline/_group.py at 100% line coverage. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/decisions.md | 8 + docs/design/rules.md | 14 +- docs/release_log.rst | 2 +- nameparser/_pipeline/_group.py | 85 +++++++- tests/v2/cases.py | 64 ++++++ tests/v2/pipeline/test_group.py | 130 ++++++++++++ tests/v2/test_ledger_guards.py | 68 ++++++- tests/v2/test_properties.py | 198 +++++++++++++++++++ tools/differential/corpus_rules.jsonl | 2 + tools/differential/corpus_shapes.jsonl | 4 + tools/differential/expected_since_1.4.0.toml | 49 +++++ tools/differential/expected_since_2.0.0.toml | 53 ++++- tools/differential/expected_since_2.1.0.toml | 53 ++++- tools/differential/expected_since_2.2.0.toml | 53 ++++- tools/differential/expected_since_2.3.0.toml | 42 ++++ 15 files changed, 808 insertions(+), 17 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 77513d93..4ad2f923 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -206,6 +206,14 @@ the fullwidth-colon marker (旧姓：佐藤 arrives as one word; the head-peel q - 2026-09-19 (Derek), #533 review — THE DELIMITERS TRUMP ANY OTHER READING, and that is a settled position rather than an un-asked fork. `Jane Doe (nee Smith Ma)` and `Jane Doe (nee Smith MA)` both keep the WHOLE span as the maiden name and report nothing, in every release including this one; the marker INSIDE the delimiters is what tells the parser the span is a maiden clause, even where the pair is not configured in `maiden_delimiters` (M3). The writer drew the boundary, so no fork is called and there is nothing to report — which is a different kind of quiet from the silences M2 and `AmbiguityKind.SUFFIX_OR_NAME` document, where a fork WAS available and the parser declined to ask it. The boundary cuts both ways: `Jane Doe (nee Smith) MA` gives suffix 'MA' and reports, the member standing outside the span being an ordinary trailing credential. Behavior is unchanged from 2f57ff21 and was not touched; what the review added is the three rules.md example lines, the case rows, and the statement in M2 and in the `SUFFIX_OR_NAME` docstring, because an undocumented settled position reads as an oversight to the next person who measures it. +- 2026-09-20 (Derek), #397 review — A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME, AND THE SPEC'S Q3 WAS WRONG ABOUT THIS BY MEASUREMENT. The walk has ended the birth name at the first suffix WORD after the marker since the #420 reorder, and the Catalan link is also the roman numeral, so `Jane Doe nee Puig i Soler` read maiden 'Puig' and left 'i Soler' outside the clause — middle 'Doe i', family 'Soler' at master 46651750, and the same truncation at every 2.x release. THE TRUNCATION PREDATES #397. What #397 added is the harm: with the link joining, those leaked words merged into the CURRENT surname, and 0fbcaa0b read family 'Doe i Soler' — a word of one person's birth name inside the surname they carry now, which is the class of failure #424 and #533 exist to prevent, arrived at from a third direction. The comma form leaks into the given part instead (`Doe, Jane nee Puig i Soler` read given 'Jane i Soler'), and a Catalan head loses the distinction between its own link and the clause's (`Maria Carod i Rovira nee Puig i Soler` read family 'Carod i Rovira i Soler'). THE SPEC'S Q3 said this interaction was unchanged; it was written from the rule and not from a parse, and the four names above are what a measurement said instead. Derek decided on 2026-09-20 to fix it in #397's own PR rather than file it, on the ground that the PR is what turned a truncation into a leak. + THE FIX IS P3'S OWN CLASS TEST, ASKED WITH THE CLAUSE'S BOUNDS. The walk does not stop at a suffix piece that is a connective PLACED TO JOIN inside the clause — a name word of the clause on its left and one on its right, which is the condition the 41de3df6 join already asks through `_name_word_beside`, called here rather than restated (mechanisms.md#ONE-PREDICATE-PER-QUESTION). Two bounds make it the CLAUSE's question and not the segment's, and each is a control: the left bound is the first piece after the whole marker run, so the marker is no name word (`Jane Doe nee i Soler` declines as before, family 'nee i Soler'); the right bound is where assign's trailing run begins over the pieces as written, so a generation or credential ending the clause is outside it (`Jane Doe nee Puig i III` keeps suffix 'i III', `Jane Doe nee Puig i MA` suffix 'i MA' — and the MA one needs the BOUND, carrying no `vocab:suffix` tag for the piece test to refuse it by). The right bound is `trailing_start`'s own answer read off the peel pair already in hand, not a second call. #533's trailing-credential stop and #424's numeral stop are byte-identical across the change, measured. + MEASURED, and the pair of numbers is the argument. A new property grid — 4,590 clause texts (five heads including a Catalan one with its own link, all seventeen shipped markers, six clause bodies, five trailing runs, comma and no-comma) against the default policy, both family-first orders and strict comma, 15,810 rows, of which 10,540 hold a link the `y` spelling puts between two birth-name words — carries two invariants in `tests/v2/test_properties.py`. THE y TWIN (`y` is the same connective with the ambiguity removed, so it is an oracle and not a second subject): the `i` parse's seven fields equal the `y` parse's, letter for letter apart from the link. It fails on ALL 10,540 guarded rows at 0fbcaa0b AND at the parent 46651750 — no clause shape holding a joining link read as its twin did — and on 0 here. NO BIRTH-NAME WORD IN THE CURRENT NAME, delimited by the twin's own maiden tokens: 28,985 tokens at 0fbcaa0b against 22,185 at 46651750, and 0 here. The GAP between those two is exactly what this branch added, and it is why the y twin alone would not have told the story: the twin fails identically at both commits while the harm doubled between them. + THE GUARD IS NARROWER THAN "the twin keeps the link", by measurement rather than by preference. `Jane Doe nee Puig y` keeps its `y` in maiden 'Puig y' while `Jane Doe nee Puig i` reads maiden 'Puig' with suffix 'i' — deliberately, the link joining nothing there — so a guard asking only whether the twin kept the letter would demand the two agree where the rules say they must not. `Puig y III` and `y Soler` are the same shape from the other two sides, and the grid keeps all three as the bodies the guard has to refuse. + INV6's off-switch exemption is NOT widened, and the reason is the one AGENTS.md gives for not widening an exemption a grid cannot reach: measured 2026-09-20, 1,680 rows of that grid carry both a class letter and a marker and 0 of them put the letter after the marker, its generator placing the connective among the name's own words. A clause link has its own grid instead. + TWO SECOND-ORDER MOVEMENTS, both recorded rather than repaired. The two one-case spellings part company, which is P3's own Accepted clause reaching a taken marker rather than a declined one: classify leaves a letter after a marker to the mixed-case rule, so `JANE DOE NEE PUIG I SOLER` never moved (the capital is an initial, and the initial veto kept the walk going all along) while `jane doe nee puig i soler` goes from family 'doe i soler' to maiden 'puig i soler'. And a report is GAINED where a longer clause ends on an ambiguous credential: `Jane Doe nee Puig i Ma` keeps maiden 'Puig i Ma' and says `suffix-or-name` where 0fbcaa0b read family 'Doe i Ma' in silence — the clause's own emitter, reaching a word the truncation had put out of its reach. + + ### N3 — the lone-word nickname rule - 2026-07 (v2 core, PR #288; recorded plan deviation #2 of the core plan) — v1's rule counted pieces before grouping; the v2 port fires only when the nickname accompanies exactly ONE piece in total — a title counts against it, so "'Smitty' Dr. Jones" reaches H1 with a title and one name word left standing — through 2.1 that meant given="Jones" with the family empty, and since #410 (2026-08-25) H1 names the family, so it reads family="Jones". The count is unchanged; what moved is what happens after it declines. The rule lives in assignment because that is where the piece count is settled. (An earlier wording here said "one non-title piece", predicting the opposite output; the coherence review measured the truth.) diff --git a/docs/design/rules.md b/docs/design/rules.md index aa5f6628..1b9d2cef 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1264,6 +1264,13 @@ M2. Rationale: a maiden marker announces that what follows it is the credential, and never the first word after the marker — as the maiden name, and the marker itself is dropped. + One suffix word does not stop it. Where such a word is also a + connective standing between two name words of the clause (P3), + a link inside the birth name does not end it, and the words on + both sides of the link are the maiden name. A link with the + marker on one side of it, or with the trailing run on the + other, is joining nothing there and ends the clause like any + other suffix word. Those last two stops are each asked TWICE for one reason: the count of words to spare includes the very words the marker removes, so a reading taken over the name as written can be @@ -1343,6 +1350,9 @@ M2. Rationale: a maiden marker announces that what follows it is the "Doe, Dr. nee Smith MA" → maiden="Smith MA" · boundary "Berg, abdul nee Jones MA" → maiden="Jones MA" · boundary "Jane Doe nee Smith DO DO" → maiden="Smith DO DO" · boundary + "Jane Doe nee Puig i Soler" → maiden="Puig i Soler" + "Jane Doe nee Puig i" → maiden="Puig" · boundary + "Jane Doe nee Puig i" → suffix="i" · boundary "Jane Doe (nee Smith MA)" → maiden="Smith MA" "Jane Doe (nee Smith Ma)" → maiden="Smith Ma" "Jane Doe (nee Smith) MA" → suffix="MA" @@ -1367,7 +1377,9 @@ M2. Rationale: a maiden marker announces that what follows it is the Accepted: the marker reads the words as written, so a suffix word inside the maiden name ends it even where a connective beside it would have bound the two into one name word (P3); the connective - then builds a family name out of what is left. + then builds a family name out of what is left. A suffix word + that IS the connective is the exception stated above, and ends + the clause only where it joins nothing. "Jane née Jr y Jones" → maiden="" Accepted: a bare acronym the reading declines is maiden text all the same — the writing decides this one (S2), and the count such diff --git a/docs/release_log.rst b/docs/release_log.rst index bc18df0e..6af905f1 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -22,7 +22,7 @@ Release Log - **Fix a maiden marker's clause swallowing a trailing credential in silence.** ``HumanName("Jane Doe nee Smith MA")`` gives maiden ``Smith`` with suffix ``MA``, where 2.0 through 2.3 gave maiden ``Smith MA`` and said nothing; 1.4.0 read the ``MA`` as a suffix too. ``Doe, Jane nee Smith MA`` moves with it, and so do the one-case spellings ``JANE DOE NEE SMITH MA`` and ``jane doe nee smith ma``. The words a marker takes now end where a trailing credential begins, which is what the marker's other two stops -- a suffix word, a trailing roman numeral -- have always done. Until this release it was the last trailing position in the library where a word of the ambiguous credential class was read without a report, and it was order-sensitive besides: ``Jane Doe nee Smith MA PhD`` gave maiden ``Smith MA`` while ``Jane Doe nee Smith PhD MA`` gave maiden ``Smith``, so whether the word was read at all depended on which side of the unambiguous credential the writer put it. Both now give maiden ``Smith``, with suffix ``MA PhD`` and ``PhD MA``. The writing still decides, exactly as it does for the same word ending a name with no clause: ``Jane Doe nee Smith Ma`` keeps maiden ``Smith Ma``, and ``Jane Doe nee Yo-Yo Ma`` keeps a two-word birth surname whole. The one member of this class that is also a surname particle keeps the carve-out it has outside a clause -- ``Doe, Jane nee Smith DO`` gives suffix ``DO`` while ``Doe, Jane nee Smith do`` and ``Doe, Jane nee Smith Do`` keep maiden ``Smith do`` and ``Smith Do``, and the comma-less ``Jane Doe nee Smith do`` gives suffix ``do`` as ``John Doe do`` does. Either reading is now reported, and there is no third: a word the clause gives up reads as a post-nominal, or the clause keeps it and says so. A name word behind the credential ends its reach and stays silent -- ``Jane Doe nee MA Smith`` gives maiden ``MA Smith`` and reports nothing -- and this stop never takes the first word after the marker, whatever its writing says: ``Jane Doe nee MA`` keeps maiden ``MA`` and reports, the marker having announced a name where there would otherwise be none, and ``Jane Doe nee MA PhD`` keeps it too. That differs on purpose from what a certain post-nominal gets there, ``Jane Smith nee PhD`` and ``Jane Smith nee V`` leaving the marker standing as an ordinary word as before. Where no trailing rule reads the clause's tail nothing is decided and the clause keeps every word: ``Smith nee Jones MA, Jane`` and ``Smith, John, Jr nee Jones MA`` both keep maiden ``Jones MA``, unchanged and with no ``suffix-or-name`` report. A trailing title is not transparent here and the two spellings disagree -- ``Jane Doe nee Smith MA Prof.`` is unchanged and silent while ``Jane Doe nee Smith Prof. MA`` gives maiden ``Smith Prof.`` with suffix ``MA`` -- which is recorded as a boundary rather than fixed. The clause also keeps a word it cannot promise a credential reading for, which is where three shapes that look like they should move do not. Where the part the word would land in holds no name of its own there is nothing to read it as a credential, so ``Doe, Dr. nee Smith MA`` and ``Jane Doe, Jr nee Smith MA`` both keep maiden ``Smith MA`` and report. Where a join would swallow it first the same applies, and it is the birth name that would lose the word: ``Berg, abdul nee Jones MA`` keeps maiden ``Jones MA`` rather than reading first ``abdul MA``, and ``Berg, Jane van der nee Smith DO`` keeps maiden ``Smith DO`` rather than letting the particle chain carry the ``DO`` into last ``van der DO Berg``. Each of those reads as 2.3.0 read it. Delimiters settle the question outright and always did: ``HumanName("Jane Doe (nee Smith MA)")`` keeps the whole span as the maiden name and reports nothing, the writer having drawn the boundary, while ``Jane Doe (nee Smith) MA`` gives suffix ``MA`` for the word left outside it. One name is a restoration rather than a change: ``John Smith nee Jones R.A.I.`` gives suffix ``R.A.I.`` again, as 2.3.0 read it, this unreleased cycle having moved it into the maiden name when the unlisted-dotted reading above took the word out of the certain-suffix class. See the ``M2`` and ``S2`` entries of ``docs/design/decisions.md`` (closes #533) - - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). Case repair follows the reading: a lower-case ``i`` the parse read as the generation is still title-cased by ``capitalize(force=True)`` (``Carod i`` gives ``Carod I``, as every release did), while one standing among the name words keeps its lower case as ``y`` always has (``Carod i Rovira`` gives ``Carod i Rovira``, where ``Carod I Rovira`` was the pre-2.4 answer). A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) + - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). Case repair follows the reading: a lower-case ``i`` the parse read as the generation is still title-cased by ``capitalize(force=True)`` (``Carod i`` gives ``Carod I``, as every release did), while one standing among the name words keeps its lower case as ``y`` always has (``Carod i Rovira`` gives ``Carod i Rovira``, where ``Carod I Rovira`` was the pre-2.4 answer). A link inside a maiden clause stays in the birth name, which no release read that way: ``HumanName("Jane Doe nee Puig i Soler")`` gives maiden ``Puig i Soler`` with last ``Doe``, where 2.0 through 2.3 ended the birth name at the link and gave maiden ``Puig`` with middle ``Doe i``, last ``Soler`` -- and the same words would have joined into last ``Doe i Soler`` under the change above, carrying a word of the birth name into the current surname. ``Doe, Jane nee Puig i Soler`` and ``Jane Doe née Kowalska i Nowak`` move with it, as does the all-lower ``jane doe nee puig i soler``; the ``y`` spelling always read this way and is untouched. The link still has to be joining: ``Jane Doe nee Puig i`` keeps maiden ``Puig`` with suffix ``i``, and ``Jane Doe nee Puig i III`` suffix ``i III``. A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 600bfbde..717938c2 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -285,6 +285,42 @@ def _join_takes_the_member(view: Sequence[Sequence[int]], and leading_titles(view, view_tags, tokens) == at - 1) +# rules.md#M2: "a link inside the birth name does not end it" -- the +# one shape the walk below steps over rather than stopping at. +# rules.md#P3: "A connective that is also generational vocabulary +# joins only where a name word stands on each side of it" is the +# reason, and the class test is that rule's own. The take runs BEFORE +# every join, so the link is still a piece of its own here and the +# question is asked of the pieces as classify left them -- the same +# inputs `_group_segment`'s `frozen` logic gives `_name_word_beside`, +# which is why this calls that predicate rather than restating the +# class (mechanisms.md#ONE-PREDICATE-PER-QUESTION). +# +# The two halves of "also generational vocabulary" are already settled +# where this is called from: the walk asks `is_suffix_piece` first and +# only consults this when the answer was yes, so what is left to ask +# is the connective half, through the predicate that owns it. A lone +# link is therefore no link at all ('Jane Doe nee Puig i' keeps maiden +# 'Puig' and suffix 'i'), and neither is one standing before the +# generation or the credential a clause ends with ('... nee Puig i +# III', '... i MA'): `hi` is where assign's peel begins, so those +# stand at or past it and `_name_word_beside` refuses them by bound. +def _link_joins_inside_the_clause(k: int, lo: int, hi: int, + pieces: Sequence[Sequence[int]], + ptags: Sequence[Set[str]], + tokens: Sequence[WorkToken]) -> bool: + """Whether the suffix piece at `k` is a connective PLACED TO JOIN + between two name words of the clause `lo`..`hi`. + + Defined here, beside its one caller, and forward-referencing the + two predicates it is built out of: `_is_conj_piece` and + `_name_word_beside` are the JOIN's, further down this module, and + moving them up to meet this would say they belonged to the clause.""" + return (_is_conj_piece(pieces[k], ptags[k], tokens) + and _name_word_beside(k, -1, lo, hi, pieces, ptags, tokens) + and _name_word_beside(k, 1, lo, hi, pieces, ptags, tokens)) + + def _maiden_take(pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], tokens: Sequence[WorkToken], @@ -384,6 +420,20 @@ def _maiden_take(pieces: Sequence[Sequence[int]], rest = peel_walk(seen[m], ptags, skip) peeled = peel_trailing(rest, pieces, ptags, tokens, one_case) trailing = rest[-1] if peeled.numeral is not None else len(pieces) + # Where assign's trailing run begins over the pieces as WRITTEN: + # `trailing_start`'s whole answer, read off the peel pair it wraps + # rather than re-running that pair, which is the reading its own + # docstring sends this caller here for. Only the link exception in + # the walk below wants it, as the right bound of the clause -- the + # walk's own stop is `trailing`, and this is never past it: the + # numeral fork's `trailing` is the walk's LAST piece and the + # acronym fork's `stop` is a max over this one, so the two never + # disagree about where the clause ends, only about what the + # exception may reach across. Measured 2026-09-20 with a probe + # here over the whole suite -- 93,408 reaches of this site, + # `peel_start > trailing` 0 of them. + peel_start = (rest[peeled.names] if peeled.names < len(rest) + else len(pieces)) # The fork reads the piece before the numeral, and the take # REMOVES that piece: afterwards assign sees the piece before the # marker there, and if that is initial-shaped the fork will not @@ -539,15 +589,35 @@ def _maiden_take(pieces: Sequence[Sequence[int]], if takes and not _join_takes_the_member( view, view_tags, tokens, at): trailing = stop + # The walk starts past the WHOLE marker: a phrase's second word is + # the marker, not the first word it takes. With nothing behind the + # marker at all there is no clause to walk and no first word to + # bound it with, so the decline the `j <= m + run` test below + # reaches is taken here instead -- `lo` would have no piece to name + # ('Jane van der Berg née'). + if m + run >= len(seen): + return None + # rules.md#M2: "a link inside the birth name does not end it" -- + # the clause's OWN bounds, which are not the segment's: `lo` is the + # first piece after the marker run, so the marker is never the name + # word on a link's left, and `peel_start` is where the trailing run + # begins, so the generation or credential a clause ends with is + # never the name word on its right ('... nee Puig i III', '... i + # MA', whose MA carries no `vocab:suffix` tag for the piece test to + # refuse it by). A core between the marker and that first word is + # below `lo` by construction and so cannot pass for the name word + # either ('PhD née - i Jones'). + lo = seen[m + run] j = m + run while (j < len(seen) and seen[j] < trailing - and not is_suffix_piece(pieces[seen[j]], ptags[seen[j]], - tokens)): + and (not is_suffix_piece(pieces[seen[j]], ptags[seen[j]], + tokens) + or _link_joins_inside_the_clause(seen[j], lo, peel_start, + pieces, ptags, tokens))): j += 1 # j == m + run means nothing followed the marker but a suffix, so # the pass declines and the marker stays ordinary words - # (rules.md#M2). The walk starts past the WHOLE marker: a phrase's - # second word is the marker, not the first word it takes. + # (rules.md#M2). if j <= m + run: return None # #533, mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE: "Emit at the @@ -633,8 +703,13 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], # rules.md#P3: "a word the rest of the parse reads as a name word # rather than as a generation, a credential or an honorific, looked # for past any run of connectives standing between" (#397) +# `Sequence[Sequence[int]]` rather than `Sequence[Piece]`, widened +# when the maiden walk became a second caller: this reads a piece and +# never edits one, and `_maiden_take` holds its pieces at the wider +# type the stage's entry point hands it. def _name_word_beside(k: int, step: int, lo: int, hi: int, - pieces: Sequence[Piece], ptags: Sequence[Set[str]], + pieces: Sequence[Sequence[int]], + ptags: Sequence[Set[str]], tokens: Sequence[WorkToken]) -> bool: """Whether such a word stands on the `step` side of the connective piece at `k`. diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 276c5918..eded423d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -2748,6 +2748,70 @@ def _check_cjk_shape_purity(self) -> None: "middle 'Garcia y Lopez' -- measured. Identical at " "1.4.0", shape=1), + Case("a_link_inside_a_maiden_clause_stays_in_the_birth_name", + "Jane Doe nee Puig i Soler", + {"given": "Jane", "family": "Doe", "maiden": "Puig i Soler"}, + classification="fix(#397)", + notes="the shape the link fix turned from a truncation into " + "a LEAK, and the reason it is fixed here rather than " + "left for its own issue: the maiden walk has always " + "stopped at the first suffix word after the marker, so " + "2.0.0 through 2.3.0 read maiden 'Puig' and left 'i " + "Soler' standing as middle 'Doe i', family 'Soler' -- " + "a truncation, and the released words at least stayed " + "out of the surname. With the link joining, the same " + "two words became family 'Doe i Soler': a word of the " + "BIRTH name inside the CURRENT one, which is the class " + "of failure #424 and #533 exist to prevent. The 'y' " + "spelling never had either reading, and the invariant " + "in test_properties.py is that twin. 1.4.0 has no " + "maiden support at all and read middle 'Doe nee Puig " + "i', last 'Soler'", + shape=1), + Case("the_link_stays_in_the_birth_name_after_a_family_comma_too", + "Doe, Jane nee Puig i Soler", + {"given": "Jane", "family": "Doe", "maiden": "Puig i Soler"}, + classification="fix(#397)", + notes="the comma form, and a different reader rather than a " + "second member of one shape: segment 1 is read at the " + "GIVEN slot, so the leak landed in given 'Jane i " + "Soler' rather than in the family. 2.0.0 through 2.3.0 " + "read maiden 'Puig' with middle 'Soler' and suffix " + "'i'. 1.4.0 read middle 'nee Puig Soler', suffix 'i'", + shape=2), + Case("a_link_with_nothing_on_its_right_still_ends_the_clause", + "Jane Doe nee Puig i", + {"given": "Jane", "family": "Doe", "suffix": "i", + "maiden": "Puig"}, + classification="fix(#274)", + notes="the CONTROL for the two rows above and the half that " + "must not move: a link joining nothing is the " + "generation it also spells, so the clause ends at it " + "exactly as it did. The pair is what says the " + "exception is about PLACEMENT and not about the word. " + "Unchanged from 2.0.0 through 2.3.0; 1.4.0 read middle " + "'Doe nee', last 'Puig', suffix 'i', which is the " + "maiden support #274 added", + shape=1), + Case("a_link_before_a_generation_ends_the_clause_too", + "Jane Doe nee Puig i III", + {"given": "Jane", "family": "Doe", "suffix": "i III", + "maiden": "Puig"}, + classification="fix(#436/#437)", + notes="the second control, and the one that pins WHICH bound " + "the exception reads: the right-hand name word is " + "looked for below the trailing run assign's peel takes, " + "so a generation there is outside the clause's own " + "words and the link joins nothing. Widen that bound to " + "the walk's own stop and this row reads maiden 'Puig i " + "III' -- measured. 'i MA' is the acronym half of the " + "same control and is not in this table: 'MA' carries " + "no suffix tag for the piece test to refuse it by, so " + "only the peel bound keeps it out. Unchanged from " + "2.0.0 through 2.3.0; 1.4.0 wrote the suffix 'i, III' " + "with a comma R1 no longer derives, and read last " + "'Puig' for want of maiden support", + shape=1), # ---- #461: a connective initials where it joins nothing -------- # The rule, one sentence for all three groups: a connective # contributes nothing where it is JOINING -- a part holding diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index 39c350a2..a88254e6 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -1645,3 +1645,133 @@ def test_the_count_reaches_a_connective_that_is_particle_vocabulary( # leading, so the joined run loses the letter here. A standing # rules.md#R2 limit this row surfaces, not one it creates. assert out.family_base == "Velasquez Garcia" + + +# --- #397 review: the link inside a MAIDEN CLAUSE ------------------- +# rules.md#M2's link clause. The walk ends the birth name at the first +# suffix WORD after the marker, and the link is one -- so it ended the +# clause there, and what #397 added was to JOIN the words it left +# standing into the current surname. Each branch below has the same +# LINK/PLAIN pair the both-sides tests above use: with the letter +# outside the generational vocabulary the walk never stopped at it in +# the first place, so the pair says which arm does the work. +# +# `ma` is added to BOTH acronym sets where the right-hand neighbour +# has to be an AMBIGUOUS credential: such a word carries no +# `vocab:suffix` tag, so the piece test cannot refuse it and only the +# peel bound keeps it out of the clause. +_LINK_MA_LEX = _LINK_LEX.add(suffix_acronyms={"ma"}, + suffix_acronyms_ambiguous={"ma"}) +_PLAIN_MA_LEX = _PLAIN_LEX.add(suffix_acronyms={"ma"}, + suffix_acronyms_ambiguous={"ma"}) + + +def test_a_link_inside_a_maiden_clause_does_not_end_it() -> None: + # the statement of the rule. A birth-name word on each side of the + # link, so the walk steps over it and the clause takes all three + # words; without the exception the clause is 'Puig' and 'i Soler' + # is left for the joins to build a surname out of. + out = _grouped("Jane Doe née Puig i Soler", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig", "i", "Soler"] + assert _piece_texts(out) == [["Jane", "Doe"]] + + +def test_the_clause_link_arm_is_what_moves_it_not_the_vocabulary( +) -> None: + # the PLAIN twin: with the letter a connective but NOT generational + # vocabulary the walk never stopped at it, so the reading is the + # one it always had. The pair is what says the exception is about + # the overlap. + out = _grouped("Jane Doe née Puig i Soler", lexicon=_PLAIN_LEX) + assert _maiden_texts(out) == ["Puig", "i", "Soler"] + + +def test_the_clause_link_survives_a_family_comma() -> None: + # the same walk under the GIVEN_SLOT reader, which is a different + # branch of the take rather than a second member of one shape: + # before the exception the leak landed in the given part. + out = _grouped("Doe, Jane née Puig i Soler", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig", "i", "Soler"] + assert _piece_texts(out) == [["Doe"], ["Jane"]] + + +def test_a_clause_link_runs_twice_over() -> None: + # every link of the clause is asked, not just the first: the walk + # steps over each one it finds between two birth-name words. + out = _grouped("Jane Doe née Puig i Soler i Vila", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig", "i", "Soler", "i", "Vila"] + + +def test_a_clause_link_with_nothing_on_its_right_still_ends_it( +) -> None: + # the first control. Nothing stands after the link at all, so it + # is joining nothing and is the generation it also spells -- the + # clause ends at it exactly as it did before, and `_name_word_ + # beside` walks off the end. The PLAIN twin keeps the letter, + # which is what says this row is the condition's doing. + out = _grouped("Jane Doe née Puig i", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig"] + plain = _grouped("Jane Doe née Puig i", lexicon=_PLAIN_LEX) + assert _maiden_texts(plain) == ["Puig"] + + +def test_a_generation_on_the_links_right_is_no_name_word() -> None: + # the second control, refused by CLASS: 'jr' is suffix vocabulary, + # so `_name_word_beside` declines it wherever it stands. + out = _grouped("Jane Doe née Puig i jr", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig"] + plain = _grouped("Jane Doe née Puig i jr", lexicon=_PLAIN_LEX) + assert _maiden_texts(plain) == ["Puig", "i"] + + +def test_an_ambiguous_credential_on_the_right_is_refused_by_bound( +) -> None: + # the third control, and the one that pins WHICH right-hand bound + # the exception reads. 'MA' carries no `vocab:suffix` tag -- the + # piece test says nothing about it -- so what keeps it out of the + # clause is `peel_start`, where assign's trailing run begins over + # the pieces as written. Read the walk's own stop instead and this + # row takes 'i MA' into the birth name. + out = _grouped("Jane Doe née Puig i MA", lexicon=_LINK_MA_LEX) + assert _maiden_texts(out) == ["Puig"] + plain = _grouped("Jane Doe née Puig i MA", lexicon=_PLAIN_MA_LEX) + assert _maiden_texts(plain) == ["Puig", "i"] + + +def test_a_suffix_word_that_is_no_connective_still_ends_the_clause( +) -> None: + # the recorded negative control for the CLASS half of the + # exception, the shape 'Juan Garcia Lopez y' is for the join: 'jr' + # stands between two birth-name words and is not a connective at + # all, so the exception is never asked and the clause ends at it + # as it always did. Drop the connective conjunct and this row + # reads maiden 'Puig jr Soler' -- measured. Identical under both + # lexicons, the letter deciding nothing here. + out = _grouped("Jane Doe née Puig jr Soler", lexicon=_LINK_LEX) + assert _maiden_texts(out) == ["Puig"] + assert _piece_texts(out) == [["Jane", "Doe", "jr", "Soler"]] + + +def test_the_marker_is_not_the_name_word_on_the_links_left() -> None: + # the fourth control, and the one the clause's own `lo` bound + # carries: the marker announces the name and is no word of it, so + # a link standing first inside the clause joins nothing there. The + # walk then stops at its very first piece and the pass declines + # altogether, leaving the marker an ordinary word (rules.md#M2). + out = _grouped("Jane Doe née i Soler", lexicon=_LINK_LEX) + assert _maiden_texts(out) == [] + assert _piece_texts(out) == [["Jane", "Doe", "née i Soler"]] + plain = _grouped("Jane Doe née i Soler", lexicon=_PLAIN_LEX) + assert _maiden_texts(plain) == ["i", "Soler"] + + +def test_a_marker_with_nothing_after_it_declines_before_the_bound( +) -> None: + # the clause's `lo` is the first piece after the marker run, and + # with nothing behind the marker there is no such piece: the pass + # declines here rather than indexing for a bound it would never + # read. Unchanged behavior, and the row exists because the early + # return is what makes it unchanged. + out = _grouped("Jane Doe née", lexicon=_LINK_LEX) + assert _maiden_texts(out) == [] + assert _piece_texts(out) == [["Jane", "Doe", "née"]] diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index d51a4173..a8257621 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1063,6 +1063,29 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": ("Smith, MD PhD", "John Smith, MD PhD", "Smith, MD, PhD", "Smith, MD - PhD - FACS", "John Doe, MD - PhD - FACS"), + # #397's maiden-clause rules, at every baseline. The key is a + # substring of BOTH spellings -- the bare 'fix(#397)' one at the + # 2.x baselines and the 'fix(#274/#397)' comma name at 1.4.0 -- + # and every probe must miss both. The wall the comments argue + # for: the link with nothing to its right, the link before a + # generation and before a credential (each the generation it also + # spells, and the clause ends at it as it always did), the two + # spellings where the letter is an INITIAL and the clause keeps + # everything, the delimited form the writer settled, and the 'y' + # twin, which never had either reading at any baseline. + "a link inside a maiden clause stays in the birth name": + ("Jane Doe nee Puig i", "Jane Doe nee Puig i III", + "Jane Doe nee Puig i MA", "Jane Doe nee Puig I Soler", + "JANE DOE NEE PUIG I SOLER", "Jane Doe (nee Puig i Soler)", + "Jane Doe nee Puig y Soler"), + # The 1.4.0 rule for the control, whose diff there carries the + # marker's consumption and the run's rendering and not this + # change at all. It must not reach the names the link fix DOES + # move, nor the older generation run whose rendering the + # fix(#436/#437) rule at the head of that file owns. + "a clause the generational link ended": + ("Jane Doe nee Puig i Soler", "Doe, Jane nee Puig i Soler", + "Jane Doe nee Puig i", "Josep Lluis Carod i III"), "fix(#436/#437) the glued honorific and the roman numeral are one post-nominal run": ("田中さん V.", "田中さん 様.", "田中さん, Dr."), "fix(#436/#437) the glued honorific and the polite address are one post-nominal run": @@ -2625,7 +2648,13 @@ class _LatinCopy(NamedTuple): "The Rt Hon Kenneth Clarke QC MP, HMG", "Washington Jr\\. MD, Franklin", "abdul Smith Jr Ma", "abdul Smith Jr V"}), - frozenset({"JOHN DOE PHD MD", "Jane Doe nee Smith PhD MA", + # 2026-09-20, #397 review: one more name joins the 2.x set and not + # the 1.4.0 one, for the reason 'Jane Doe nee Smith PhD MA' did -- + # 'Jane Doe nee Puig i III' is a comma-written generation run + # those baselines share with the tree's spaced one, while at 1.4.0 + # its ROLES move too and a rule of that ledger's own owns it. + frozenset({"JOHN DOE PHD MD", "Jane Doe nee Puig i III", + "Jane Doe nee Smith PhD MA", "John Doe MD PhD", "John Smith MD PhD", "John Smith Mc V", "Josep Lluis Carod i III", "Josep Lluis Carod i V", @@ -2634,6 +2663,13 @@ class _LatinCopy(NamedTuple): "The Rt Hon Kenneth Clarke QC MP, HMG", "Washington Jr\\. MD, Franklin", "abdul Smith Jr Ma", "abdul Smith Jr V"}), + # #397's maiden-clause rule, one corpus name per alternative -- a + # list of names, not a copy of any wordlist. What selects the two + # is the PLACEMENT of the link inside a clause, which no + # vocabulary decides; a member spelled as the shape (a bare letter + # after a maiden marker) would reach every clause name in the + # corpora and pre-excuse the readings the walk must refuse. + frozenset({"Doe, Jane nee Puig i Soler", "Jane Doe nee Puig i Soler"}), # #397's join and its one-case report, one corpus name per # alternative -- lists of names, not copies of any wordlist. The # join's subject is a SHAPE the vocabulary participates in at one @@ -3199,7 +3235,7 @@ def _claim(rule: dict) -> _Claim: # carrying a marker. Read name by name against the regex; no # role joined the list. "fix(#274) maiden markers consumed": - _Claim(73, ('family', 'maiden', 'middle'), 'a2e495d071dd', None), + _Claim(77, ('family', 'maiden', 'middle'), 'd36e74b1f60d', None), # 2026-09-19, #533: 5 -> 6, the same one new corpus name # '田中 太郎 旧姓 佐藤 MA' as the CJK rule above. "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip": @@ -3300,7 +3336,7 @@ def _claim(rule: dict) -> _Claim: # comma form of the swallowed generation. Reach again, and # verified name by name. "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(360, ('given', 'suffix', 'title'), 'f0e6eed75c6a', None), + _Claim(361, ('given', 'suffix', 'title'), '2e17dee05bcf', None), "fix(comma-family) a comma followed only by titles keeps the given/family split": _Claim(2, ('family', 'given'), "5bd9c6d96c38", None), "fix(comma-family) a comma followed only by titles keeps the given/family split, the C1 example": @@ -3355,7 +3391,7 @@ def _claim(rule: dict) -> _Claim: # 2026-09-20, #397 review: 359 -> 360, the same one new comma # name as the rule above and for the same reason. "fix(comma-precomma-family) pre-comma run reads as family, not given": - _Claim(360, ('family', 'given'), 'f0e6eed75c6a', None), + _Claim(361, ('family', 'given'), '2e17dee05bcf', None), # 2026-09-20, #397: retitled in place, reach and digest # unchanged -- the rule keeps 'Carod i', which the landing # leaves byte-identical. @@ -3824,6 +3860,12 @@ def _claim(rule: dict) -> _Claim: ('DEFAULT',)), "fix(#461) a connective holding its part alone contributes an initial": _Claim(15, ('_initials',), "4d436d1ebeca", ('DEFAULT',)), + "fix(#274/#397) a link inside a maiden clause stays in the birth name, after a family comma": + _Claim(1, ('maiden', 'middle', 'suffix'), 'ad2442b1b12a', + None), + "fix(#274/#436/#437) a clause the generational link ended, and the run it left renders with spaces": + _Claim(1, ('family', 'maiden', 'middle', 'suffix'), + '7d64445a7252', None), }, "expected_since_2.0.0.toml": { # #436/#437's Latin alternation, first in every ledger. @@ -3837,7 +3879,7 @@ def _claim(rule: dict) -> _Claim: # 46651750 -- and each of which arrived with that change's own # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(14, ('suffix',), "4f5e63e709ae", None), + _Claim(15, ('suffix',), "b610147f490b", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -4329,6 +4371,9 @@ def _claim(rule: dict) -> _Claim: _Claim(16, ('_initials',), "075dc34f9e95", ('DEFAULT',)), "fix(#461) a connective with a name word beside it stops contributing an initial": _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), + "fix(#397) a link inside a maiden clause stays in the birth name": + _Claim(2, ('family', 'maiden', 'middle', 'suffix'), + '32405b182f4e', ('DEFAULT',)), }, # The 2.3 cycle's first rule, and a facade-only render fix: every # role is identical, so `_initials` alone. Reach and digest as in @@ -4346,7 +4391,7 @@ def _claim(rule: dict) -> _Claim: # 46651750 -- and each of which arrived with that change's own # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(14, ('suffix',), "4f5e63e709ae", None), + _Claim(15, ('suffix',), "b610147f490b", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -4614,6 +4659,9 @@ def _claim(rule: dict) -> _Claim: _Claim(17, ('_initials',), "797473971e75", ('DEFAULT',)), "fix(#461) a connective with a name word beside it stops contributing an initial": _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), + "fix(#397) a link inside a maiden clause stays in the birth name": + _Claim(2, ('family', 'maiden', 'middle', 'suffix'), + '32405b182f4e', ('DEFAULT',)), }, "expected_since_2.1.0.toml": { # #436/#437's Latin alternation, first in every ledger. @@ -4627,7 +4675,7 @@ def _claim(rule: dict) -> _Claim: # 46651750 -- and each of which arrived with that change's own # case rows. Roles unmoved, `suffix` alone as before. "fix(#436/#437) a space-separated post-nominal run renders with spaces, not commas": - _Claim(14, ('suffix',), "4f5e63e709ae", None), + _Claim(15, ('suffix',), "b610147f490b", None), # #449's six rules, second in every 2.x ledger. The # alternation reaches twenty-two corpus names and # `_ambiguities` alone: no role moves anywhere in this change, @@ -5090,6 +5138,9 @@ def _claim(rule: dict) -> _Claim: _Claim(16, ('_initials',), "075dc34f9e95", ('DEFAULT',)), "fix(#461) a connective with a name word beside it stops contributing an initial": _Claim(13, ('_initials',), "3cc41f4bfc21", ('DEFAULT',)), + "fix(#397) a link inside a maiden clause stays in the birth name": + _Claim(2, ('family', 'maiden', 'middle', 'suffix'), + '32405b182f4e', ('DEFAULT',)), }, "expected_since_2.3.0.toml": { # #383/#479's three rules, the first this ledger carries. The @@ -5242,6 +5293,9 @@ def _claim(rule: dict) -> _Claim: _Claim(17, ('_initials',), "797473971e75", ('DEFAULT',)), "fix(#461) a connective with a name word beside it stops contributing an initial": _Claim(22, ('_initials',), "e73827447b4e", ('DEFAULT',)), + "fix(#397) a link inside a maiden clause stays in the birth name": + _Claim(2, ('family', 'maiden', 'middle', 'suffix'), + '32405b182f4e', ('DEFAULT',)), }, } diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 3fb21815..0a24cfec 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -1691,3 +1691,201 @@ def test_a_letter_that_did_not_join_repairs_as_the_off_switch_does() -> None: assert not failures, ( f"{len(failures)} repair(s) moved for a letter that joined " f"nothing:\n" + "\n".join(failures[:10])) + + +# --- #397 review: the MAIDEN-CLAUSE LINK grid, and its two ---------- +# invariants +# A third grid, kept apart from the two above for the reason the +# off-switch one is: every row here is parsed TWICE, once as written +# and once with the link spelled `y`, and the texts that earn the +# second parse are the clause shapes -- which neither grid above +# generates, both putting their connective among the name's own +# words. What this one adds: a link INSIDE a maiden clause, in every +# position a clause can hold one, under every marker the library +# ships and behind every trailing run the walk stops at. +# +# The `y` spelling is the ORACLE and not a second subject. It is the +# same sentence with the ambiguity removed: `y` is connective +# vocabulary and no generation, so nothing in the walk can read it as +# the end of the clause, and whatever it does is what the `i` +# spelling has to do wherever P3 says the letter is joining. + +#: The clause bodies, `@` standing where the link goes: a link +#: between two words, with a word run on either side of it, doubled, +#: and the two shapes where it joins NOTHING -- nothing on its right, +#: nothing on its left but the marker. The last two are the +#: controls the guard below must refuse, and +#: `test_the_maiden_link_grid_can_fail` counts them out. +_LINK_BODIES: tuple[tuple[str, ...], ...] = ( + ("Puig", "@", "Soler"), + ("Puig", "@", "Soler", "Roig"), + ("Puig", "Soler", "@", "Roig"), + ("Puig", "@", "Soler", "@", "Roig"), + ("Puig", "@"), + ("@", "Soler"), +) +#: Nothing, the two generations and the two credential classes -- the +#: four trailing runs M2's walk stops at, plus the empty one. +_LINK_TAILS: tuple[tuple[str, ...], ...] = ( + (), ("III",), ("Jr.",), ("MA",), ("PhD",)) +#: One to three words, and the last carries a link of its OWN, which +#: stays spelled `i` in both parses: the head's reading is not what +#: this grid is about, and holding it fixed is what makes a moved +#: field the clause's doing. +_LINK_HEADS: tuple[tuple[str, ...], ...] = ( + ("Jane",), ("Doe", "Jane"), ("Jane", "Doe"), + ("Jane", "M.", "Doe"), ("Carod", "i", "Rovira")) + +_LINK_RE = re.compile(r"(? dict[str, str]: + """Every clause shape, mapped to its `y`-spelled twin. + + The two spellings differ in one CHARACTER per link, so their + token spans line up exactly -- which is what lets the second + invariant below delimit the clause with the twin's own maiden + tokens and then read the first parse at those offsets. + """ + out: dict[str, str] = {} + markers = sorted(Lexicon.default().maiden_markers) + for head, marker, body, tail, comma in itertools.product( + _LINK_HEADS, markers, _LINK_BODIES, _LINK_TAILS, + (False, True)): + if comma and len(head) < 2: + continue + pair = [] + for link in ("i", "y"): + words = [link if w == "@" else w for w in body] + rest = list(head[1:] if comma else head) + [marker] \ + + words + list(tail) + pair.append(f"{head[0]}, " + " ".join(rest) if comma + else " ".join(rest)) + out.setdefault(pair[0], pair[1]) + return out + + +_LINK_TWIN = _link_texts() +_MAIDEN_LINK_GRID = _rows( + list(_LINK_TWIN), + (("default", Lexicon.default(), frozenset()),)) + + +def _link_joins_inside_the_clause(maiden: str) -> bool: + """Whether the ORACLE parse put a link inside the birth name with + a birth-name word on each side of it -- rules.md#P3's both-sides + condition, read off the `y` spelling's own maiden field. + + The guard, and it is narrower than "the twin keeps the link" for + a measured reason: 'Jane Doe nee Puig y' keeps its `y` in maiden + 'Puig y', and 'Jane Doe nee Puig i' reads maiden 'Puig' with + suffix 'i' -- deliberately, the link there joining nothing and + being the generation it also spells (rules.md#M2). So a guard + asking only whether the twin kept the letter would demand the two + agree where the rules say they must not. `Puig y III` and `y + Soler` are the same shape from the other two sides. + """ + words = maiden.split() + at = [k for k, w in enumerate(words) if w == "y"] + return bool(at) and all( + any(w != "y" for w in words[:k]) + and any(w != "y" for w in words[k + 1:]) for k in at) + + +@functools.cache +def _maiden_link_findings() -> dict[str, list[str]]: + """One walk of the maiden-link grid; two invariants' answers. + + Both parses of a row are taken once here and handed to both + predicates, which is all this walk does -- the ONE PARSE, MANY + CHECKS rule the two grids above follow. + """ + out: dict[str, list[str]] = {k: [] for k in ("INV8", "INV9")} + for text, parser, label in _MAIDEN_LINK_GRID: + twin = parser.parse(_LINK_TWIN[text]) + if not _link_joins_inside_the_clause(twin.maiden): + continue + on = parser.parse(text) + want = {k: _LINK_RE.sub("i", v) for k, v in twin.as_dict().items()} + if on.as_dict() != want: + out["INV8"].append(f"[{label}] {text!r}: {on.as_dict()} != " + f"y-twin {want}") + spans = [tok.span for tok in twin.tokens + if tok.role is Role.MAIDEN and tok.span is not None] + lo = min(s.start for s in spans) + hi = max(s.end for s in spans) + for tok in on.tokens: + if (tok.span is not None and lo <= tok.span.start + and tok.span.end <= hi and tok.role in _NAME_ROLES): + out["INV9"].append( + f"[{label}] {text!r}: {tok.text!r} of the birth " + f"name reads as {tok.role.value}") + return out + + +def test_the_maiden_link_grid_can_fail() -> None: + """The reachability probe, the shape every grid in this file + carries. Dated recorded control, measured 2026-09-20. + + The third count is the one to watch: the guard above refuses the + two joining-nothing bodies outright, so a grid whose every row + were one of those would pass both invariants in silence. 1,350 of + the first 2,000 rows are guarded in, and 10,540 of all 15,810 -- + two thirds, which is the four admitted bodies of six. + """ + assert len(_MAIDEN_LINK_GRID) == 15810, len(_MAIDEN_LINK_GRID) + assert len(_LINK_TWIN) == 4590, len(_LINK_TWIN) + guarded = sum( + 1 for text, parser, _ in _MAIDEN_LINK_GRID[:2000] + if _link_joins_inside_the_clause(parser.parse(_LINK_TWIN[text]).maiden)) + assert guarded > 800, guarded + + +def test_a_clause_link_reads_as_its_y_twin_does() -> None: + """INV8 (#397 review). THE y TWIN. Where the `y` spelling puts a + link inside the birth name with a birth-name word on each side, + the `i` spelling gives the same seven fields, letter for letter + apart from the link itself. + + `y` is connective vocabulary and nothing else, so its reading is + the one the maiden walk was never able to get wrong; `i` is that + same connective AND the roman numeral, and the walk used to end + the birth name at it. The pair is the whole statement of + rules.md#M2's link clause, and it needs no expected values of its + own. + + Mutation-checked, 2026-09-20: it fails on all 10,540 guarded rows + at 0fbcaa0b -- this branch's tip before the fix -- and on the same + 10,540 at the parent 46651750, the walk having truncated the + birth name at the link since long before #397 reached it. That it + is EVERY guarded row and not a subset is the finding: no clause + shape holding a joining link read as its twin did. + """ + failures = _maiden_link_findings()["INV8"] + assert not failures, ( + f"{len(failures)} parse(s) read a clause link differently " + f"from its y twin:\n" + "\n".join(failures[:10])) + + +def test_no_birth_name_word_reads_as_a_word_of_the_current_name() -> None: + """INV9 (#397 review). The failure class #424 and #533 exist to + prevent, stated for the link: no token standing inside the birth + name -- as the `y` twin's own maiden tokens delimit it -- is + roled GIVEN, MIDDLE or FAMILY. + + Weaker than INV8 and kept beside it because it is the one that + names the HARM. A truncation moves fields too, and this stays + green for it; what it refuses is a word of one person's birth + name being handed to the surname they carry now. + + Mutation-checked, 2026-09-20: it fails on 28,985 tokens at + 0fbcaa0b and on 22,185 at the parent 46651750, over the same + 10,540 guarded rows. The GAP between those two is what this + branch added: at the parent the released words are a truncation + that left them in `middle` and `family`, and with the link + joining they became words of the current surname itself. + """ + failures = _maiden_link_findings()["INV9"] + assert not failures, ( + f"{len(failures)} birth-name word(s) read as a word of the " + f"current name:\n" + "\n".join(failures[:10])) diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 99facfb5..10137f29 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -96,6 +96,8 @@ "Jane Doe (nee Smith) MA" "Jane Doe nee MA" "Jane Doe nee MA Smith" +"Jane Doe nee Puig i" +"Jane Doe nee Puig i Soler" "Jane Doe nee Smith DO DO" "Jane Doe nee Smith MA" "Jane Doe nee Smith MA Prof." diff --git a/tools/differential/corpus_shapes.jsonl b/tools/differential/corpus_shapes.jsonl index 09cdca72..b74c1eb8 100644 --- a/tools/differential/corpus_shapes.jsonl +++ b/tools/differential/corpus_shapes.jsonl @@ -27,6 +27,9 @@ {"name": "Jane Doe geb. Smith MA", "shape": 1} {"name": "Jane Doe nee MA", "shape": 1} {"name": "Jane Doe nee MA Smith", "shape": 1} +{"name": "Jane Doe nee Puig i", "shape": 1} +{"name": "Jane Doe nee Puig i III", "shape": 1} +{"name": "Jane Doe nee Puig i Soler", "shape": 1} {"name": "Jane Doe nee Smith 2", "shape": 1} {"name": "Jane Doe nee Smith MA", "shape": 1} {"name": "Jane Doe nee Smith MA JD", "shape": 1} @@ -125,6 +128,7 @@ {"name": "Doe, J. ba", "shape": 2} {"name": "Doe, J. nee MA ba", "shape": 2} {"name": "Doe, Jane Q. nee Smith MA", "shape": 2} +{"name": "Doe, Jane nee Puig i Soler", "shape": 2} {"name": "Doe, Jane nee Smith DO", "shape": 2} {"name": "Doe, Jane nee Smith Do", "shape": 2} {"name": "Doe, Jane nee Smith MA", "shape": 2} diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 2f5a7781..dca0797a 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -4282,3 +4282,52 @@ issue = "fix(#434/#533) a marker PHRASE takes the maiden name, and its clause en name_regex = "^Maria Kowalska z domu Nowak MA$" fields = ["family", "maiden", "middle"] + +# #397 review: A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME +# (2026-09-20). Two rules, because at this baseline two OTHER changes +# ride along in the same diffs and one rule has to explain the whole +# of each: the marker leaving the name (#274) and R1's space-joined +# rendering of a suffix run v1 wrote with a comma (#436/#437). +# +# The third new name, 'Jane Doe nee Puig i Soler', needs no rule here: +# its diff at this baseline is {family, maiden, middle}, which the +# fix(#274) alternation above already explains -- v1 has no maiden +# support at all, so the clause leaving the name IS the whole diff and +# the link's placement inside it changes none of the three field +# NAMES. The fourth, 'Jane Doe nee Puig i', is explained there too. +# +# rules.md#M2 states the exception; decisions.md#M2's 2026-09-20 +# bullet carries the measurement. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#274/#397) a link inside a maiden clause stays in the birth name, after a family comma" +# 'Doe, Jane nee Puig i Soler'. v1 read middle 'nee Puig Soler' with +# suffix 'i' -- the marker an ordinary word and the link a generation. +# The tree reads given 'Jane', family 'Doe', maiden 'Puig i Soler': +# the marker consumed (#274) and the link kept inside the clause it +# stands in (#397). `suffix` is in the list because v1 had one and the +# tree does not, which is the link's half. +# +# Literal, one name. The shape would be "a maiden marker and a bare +# letter", which reaches every clause name in the corpora. +name_regex = "^Doe, Jane nee Puig i Soler$" +fields = ["maiden", "middle", "suffix"] + +[[change]] +issue = "fix(#274/#436/#437) a clause the generational link ended, and the run it left renders with spaces" +# 'Jane Doe nee Puig i III'. Two changes meet in the v1 diff: the +# marker leaving the name (#274, maiden 'Puig' where v1 read last +# 'Puig' and middle 'Doe nee') and the two-word generation run 'i III' +# that v1 rendered 'i, III' (#436/#437). The link fix moves NOTHING on +# it and that is what the name is in the corpus for -- measured +# against the parent 46651750, the parse is byte-identical before and +# after this bundle, the link having a generation to its right rather +# than a name word. +# +# Separate from the fix(#274/#436/#437) rule above it rather than a +# widening of that anchor: that one is about a clause an unambiguous +# CREDENTIAL ended, and a rule naming only credentials would +# attribute this stop to the wrong reading. +name_regex = "^Jane Doe nee Puig i III$" +fields = ["family", "maiden", "middle", "suffix"] diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 4a28af15..113e2af9 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -73,7 +73,16 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # generation marker and not a name word to its right. What grew is # the CORPUS -- the three arrived with #397's own case rows -- and # the run rendering is what the diffs are about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397 review: one more name joins the alternation and +# the rule does not change. 'Jane Doe nee Puig i III' is the same +# two-word generation run inside a MAIDEN clause's tail: this +# baseline renders 'i, III' and the tree renders 'i III', which is +# the whole of its diff here. The link fix moves nothing on it -- +# measured against the parent 46651750, the parse is byte-identical +# before and after (given 'Jane', family 'Doe', suffix 'i III', +# maiden 'Puig'), the link having a generation and not a name word +# to its right. What grew is the CORPUS. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Puig i III|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -3188,3 +3197,45 @@ issue = "fix(#461) a connective with a name word beside it stops contributing an name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" fields = ["_initials"] orders = ["DEFAULT"] + +# #397 review: A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME +# (2026-09-20). LAST in the file, narrow-first: this rule declares +# {family, maiden, middle, suffix} and every rule above it that could +# reach either name declares a strict subset, so an earlier position +# would be an order-decided contest the run refuses (#382). +# +# rules.md#M2 states the exception; decisions.md#M2's 2026-09-20 +# bullet carries the measurement and why it is fixed in #397's own PR +# rather than filed. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) a link inside a maiden clause stays in the birth name" +# Two names, one sentence, two shapes -- `fields` is the union. +# +# The maiden walk has always ended the birth name at the first suffix +# WORD after the marker, and the Catalan link is also the roman +# numeral, so this baseline read maiden 'Puig' and left 'i Soler' +# outside the clause: 'Jane Doe nee Puig i Soler' middle 'Doe i', +# family 'Soler'; 'Doe, Jane nee Puig i Soler' middle 'Soler', suffix +# 'i'. That truncation PREDATES this bundle. What #397 added is +# worse and is why the stop is fixed here: with the link joining, the +# leaked words merged into the CURRENT surname -- measured on the +# parent 46651750 and on this branch's 0fbcaa0b, family 'Doe i Soler' +# and given 'Jane i Soler' respectively -- a word of the birth name +# inside the current one. +# +# The 'y' spelling is the control and never moved at any baseline: +# 'Jane Doe nee Puig y Soler' reads maiden 'Puig y Soler' here and in +# the tree, 'y' being connective vocabulary and no generation. That +# twin is the property invariant in tests/v2/test_properties.py. +# +# Literal-anchored to the two. The shape is "a one-letter connective +# inside a maiden clause", which would stand ready to excuse every +# future regression at this walk -- and the walk's whole design is +# that some such letters must NOT be taken. _MUST_NOT_MATCH carries +# the wall: the i-last control, the generation behind it, and the +# spellings where the letter is an initial. +name_regex = "^(?:Doe, Jane nee Puig i Soler|Jane Doe nee Puig i Soler)$" +fields = ["family", "maiden", "middle", "suffix"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 783d6fee..fc2b30aa 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -97,7 +97,16 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # generation marker and not a name word to its right. What grew is # the CORPUS -- the three arrived with #397's own case rows -- and # the run rendering is what the diffs are about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397 review: one more name joins the alternation and +# the rule does not change. 'Jane Doe nee Puig i III' is the same +# two-word generation run inside a MAIDEN clause's tail: this +# baseline renders 'i, III' and the tree renders 'i III', which is +# the whole of its diff here. The link fix moves nothing on it -- +# measured against the parent 46651750, the parse is byte-identical +# before and after (given 'Jane', family 'Doe', suffix 'i III', +# maiden 'Puig'), the link having a generation and not a name word +# to its right. What grew is the CORPUS. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Puig i III|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -3099,3 +3108,45 @@ issue = "fix(#461) a connective with a name word beside it stops contributing an name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" fields = ["_initials"] orders = ["DEFAULT"] + +# #397 review: A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME +# (2026-09-20). LAST in the file, narrow-first: this rule declares +# {family, maiden, middle, suffix} and every rule above it that could +# reach either name declares a strict subset, so an earlier position +# would be an order-decided contest the run refuses (#382). +# +# rules.md#M2 states the exception; decisions.md#M2's 2026-09-20 +# bullet carries the measurement and why it is fixed in #397's own PR +# rather than filed. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) a link inside a maiden clause stays in the birth name" +# Two names, one sentence, two shapes -- `fields` is the union. +# +# The maiden walk has always ended the birth name at the first suffix +# WORD after the marker, and the Catalan link is also the roman +# numeral, so this baseline read maiden 'Puig' and left 'i Soler' +# outside the clause: 'Jane Doe nee Puig i Soler' middle 'Doe i', +# family 'Soler'; 'Doe, Jane nee Puig i Soler' middle 'Soler', suffix +# 'i'. That truncation PREDATES this bundle. What #397 added is +# worse and is why the stop is fixed here: with the link joining, the +# leaked words merged into the CURRENT surname -- measured on the +# parent 46651750 and on this branch's 0fbcaa0b, family 'Doe i Soler' +# and given 'Jane i Soler' respectively -- a word of the birth name +# inside the current one. +# +# The 'y' spelling is the control and never moved at any baseline: +# 'Jane Doe nee Puig y Soler' reads maiden 'Puig y Soler' here and in +# the tree, 'y' being connective vocabulary and no generation. That +# twin is the property invariant in tests/v2/test_properties.py. +# +# Literal-anchored to the two. The shape is "a one-letter connective +# inside a maiden clause", which would stand ready to excuse every +# future regression at this walk -- and the walk's whole design is +# that some such letters must NOT be taken. _MUST_NOT_MATCH carries +# the wall: the i-last control, the generation behind it, and the +# spellings where the letter is an initial. +name_regex = "^(?:Doe, Jane nee Puig i Soler|Jane Doe nee Puig i Soler)$" +fields = ["family", "maiden", "middle", "suffix"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index 8d7506d5..96a20b85 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -86,7 +86,16 @@ issue = "fix(#436/#437) a space-separated post-nominal run renders with spaces, # generation marker and not a name word to its right. What grew is # the CORPUS -- the three arrived with #397's own case rows -- and # the run rendering is what the diffs are about. -name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" +# 2026-09-20, #397 review: one more name joins the alternation and +# the rule does not change. 'Jane Doe nee Puig i III' is the same +# two-word generation run inside a MAIDEN clause's tail: this +# baseline renders 'i, III' and the tree renders 'i III', which is +# the whole of its diff here. The link fix moves nothing on it -- +# measured against the parent 46651750, the parse is byte-identical +# before and after (given 'Jane', family 'Doe', suffix 'i III', +# maiden 'Puig'), the link having a generation and not a name word +# to its right. What grew is the CORPUS. +name_regex = "^(?:JOHN DOE PHD MD|Jane Doe nee Puig i III|Jane Doe nee Smith PhD MA|John Doe MD PhD|John Smith MD PhD|John Smith Mc V|Josep Lluis Carod i III|Josep Lluis Carod i V|Kenneth Clarke QC MP|Rovira, Josep Carod i Jr\\.|Smith, John PhD I\\.|The Rt Hon Kenneth Clarke QC MP, HMG|Washington Jr\\. MD, Franklin|abdul Smith Jr Ma|abdul Smith Jr V)$" fields = ["suffix"] # The six #449 rules go SECOND, not first: the rule above @@ -1558,3 +1567,45 @@ issue = "fix(#461) a connective with a name word beside it stops contributing an name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry Robert Johns|John and Jane Aznar y Lopez|John and Jane Smith|Jose e Maria Santos|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" fields = ["_initials"] orders = ["DEFAULT"] + +# #397 review: A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME +# (2026-09-20). LAST in the file, narrow-first: this rule declares +# {family, maiden, middle, suffix} and every rule above it that could +# reach either name declares a strict subset, so an earlier position +# would be an order-decided contest the run refuses (#382). +# +# rules.md#M2 states the exception; decisions.md#M2's 2026-09-20 +# bullet carries the measurement and why it is fixed in #397's own PR +# rather than filed. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) a link inside a maiden clause stays in the birth name" +# Two names, one sentence, two shapes -- `fields` is the union. +# +# The maiden walk has always ended the birth name at the first suffix +# WORD after the marker, and the Catalan link is also the roman +# numeral, so this baseline read maiden 'Puig' and left 'i Soler' +# outside the clause: 'Jane Doe nee Puig i Soler' middle 'Doe i', +# family 'Soler'; 'Doe, Jane nee Puig i Soler' middle 'Soler', suffix +# 'i'. That truncation PREDATES this bundle. What #397 added is +# worse and is why the stop is fixed here: with the link joining, the +# leaked words merged into the CURRENT surname -- measured on the +# parent 46651750 and on this branch's 0fbcaa0b, family 'Doe i Soler' +# and given 'Jane i Soler' respectively -- a word of the birth name +# inside the current one. +# +# The 'y' spelling is the control and never moved at any baseline: +# 'Jane Doe nee Puig y Soler' reads maiden 'Puig y Soler' here and in +# the tree, 'y' being connective vocabulary and no generation. That +# twin is the property invariant in tests/v2/test_properties.py. +# +# Literal-anchored to the two. The shape is "a one-letter connective +# inside a maiden clause", which would stand ready to excuse every +# future regression at this walk -- and the walk's whole design is +# that some such letters must NOT be taken. _MUST_NOT_MATCH carries +# the wall: the i-last control, the generation behind it, and the +# spellings where the letter is an initial. +name_regex = "^(?:Doe, Jane nee Puig i Soler|Jane Doe nee Puig i Soler)$" +fields = ["family", "maiden", "middle", "suffix"] +orders = ["DEFAULT"] diff --git a/tools/differential/expected_since_2.3.0.toml b/tools/differential/expected_since_2.3.0.toml index 705f19cf..7df220dd 100644 --- a/tools/differential/expected_since_2.3.0.toml +++ b/tools/differential/expected_since_2.3.0.toml @@ -884,3 +884,45 @@ issue = "fix(#461) a connective with a name word beside it stops contributing an name_regex = "^(?:Assoc Dean of Chemistry Robert Johns|Dean of Chemistry|Dean of Chemistry Robert Johns|Duke of Edinburgh|Duke of Wellington|John & Jane|John and Jane Aznar y Lopez|John and Jane Smith|John of the Doe|Jose e Maria Santos|Juan & Garcia|Juan and Garcia|Juan y Eva Garcia|Mr\\. and Mrs\\. John and Jane Smith|Rob And Beth Edmunds|and Jon Dough|part1 of The part2 of the part3 and part4|part1 of and The part2 of the part3 And part4|the and Jon Dough|ХОСЕ И МАРИЯ САНТОС|Хосе И Мария Сантос|хосе и мария сантос)$" fields = ["_initials"] orders = ["DEFAULT"] + +# #397 review: A LINK INSIDE A MAIDEN CLAUSE STAYS IN THE BIRTH NAME +# (2026-09-20). LAST in the file, narrow-first: this rule declares +# {family, maiden, middle, suffix} and every rule above it that could +# reach either name declares a strict subset, so an earlier position +# would be an order-decided contest the run refuses (#382). +# +# rules.md#M2 states the exception; decisions.md#M2's 2026-09-20 +# bullet carries the measurement and why it is fixed in #397's own PR +# rather than filed. +# --------------------------------------------------------------- + +[[change]] +issue = "fix(#397) a link inside a maiden clause stays in the birth name" +# Two names, one sentence, two shapes -- `fields` is the union. +# +# The maiden walk has always ended the birth name at the first suffix +# WORD after the marker, and the Catalan link is also the roman +# numeral, so this baseline read maiden 'Puig' and left 'i Soler' +# outside the clause: 'Jane Doe nee Puig i Soler' middle 'Doe i', +# family 'Soler'; 'Doe, Jane nee Puig i Soler' middle 'Soler', suffix +# 'i'. That truncation PREDATES this bundle. What #397 added is +# worse and is why the stop is fixed here: with the link joining, the +# leaked words merged into the CURRENT surname -- measured on the +# parent 46651750 and on this branch's 0fbcaa0b, family 'Doe i Soler' +# and given 'Jane i Soler' respectively -- a word of the birth name +# inside the current one. +# +# The 'y' spelling is the control and never moved at any baseline: +# 'Jane Doe nee Puig y Soler' reads maiden 'Puig y Soler' here and in +# the tree, 'y' being connective vocabulary and no generation. That +# twin is the property invariant in tests/v2/test_properties.py. +# +# Literal-anchored to the two. The shape is "a one-letter connective +# inside a maiden clause", which would stand ready to excuse every +# future regression at this walk -- and the walk's whole design is +# that some such letters must NOT be taken. _MUST_NOT_MATCH carries +# the wall: the i-last control, the generation behind it, and the +# spellings where the letter is an initial. +name_regex = "^(?:Doe, Jane nee Puig i Soler|Jane Doe nee Puig i Soler)$" +fields = ["family", "maiden", "middle", "suffix"] +orders = ["DEFAULT"] From b9ed14294ce55e308844157dc4702ed3632fd638 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 12:58:43 -0700 Subject: [PATCH 09/11] =?UTF-8?q?fix(#397/#461):=20review=20round=20?= =?UTF-8?q?=E2=80=94=20a=20plain=20connective=20in=20the=20suffix=20field,?= =?UTF-8?q?=20a=20credential=20behind=20a=20title,=20and=20a=20report=20th?= =?UTF-8?q?at=20contradicted=20its=20reading?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A. Case repair's generation guard was the suffix ROLE alone, and the role says only where a word landed. A plain connective lands in the suffix field whenever it is a comma form's third part, so parse("Smith, John, and").capitalized(force=True) gave 'John Smith And' where 1.4.0, 2.0.0-2.3.0 and the parent 46651750 all give 'John Smith and' -- the same for ', y', ', e', ', und', ', of' and 'Doe, Jane, and Jr.' -- and a spliced suffix='de y' gave 'De, Y' against R4's own Accepted paragraph. The guard is now the ROLE and the `vocab:suffix` TAG, the two decisions the parse already made. Every name f76963b7 was written for is unmoved ('John Quincy Smith i' -> I, 'Carod i' -> I, 'Josep Lluis Carod i III', and 'Josep Lluis Carod v III' under add(conjunctions={"v"})). ACCEPTED: a spliced suffix="i" now repairs to 'i', following 'y', because 'i' is connective vocabulary here; the parsed name is unaffected. B. `_name_word_beside`'s right bound came from `trailing_start`, the peel over the pieces as WRITTEN, which a trailing TITLE hides the suffix run from. 'John Quincy Adams i MA Prof.' read family 'Adams i MA', suffix empty, no report, initials 'J. Q. A. M.', where 'John Quincy Adams i MA' reads family 'Adams', suffix 'i MA' and reports the acronym. New `_pieces.trailing_start_past_titles` runs the peel and the H5 chain to the fixed point assign already uses, once per segment and only where such a connective was found. 27 rows on 9 names move over the 43,925-name review grid, every one onto the off-switch reading. Frame-neutral on names with no generational connective (John Smith 163, Smith, John 185, Juan Garcia y Lopez 285, Jane Doe nee Smith 233, all unchanged). NOT the maiden twin: the clause's blindness to a trailing title is pre-existing and identical at the parent for the `y` spelling, so it is recorded under #535 rather than fixed here, as is the particle chain's same-shaped bound. C. classify's `conjunction-or-initial` names a fork between a connective and an INITIAL; where the parse rolls that letter SUFFIX it took neither branch. 'JOHN QUINCY SMITH I' carried the report beside a `suffix-or-name` saying the same token reads as a generation. Withdrawn in `_assemble`, the one site that sees final roles and the report list together. 9,806 rows on 1,637 grid names lose exactly one report and nothing else; on the corpus the only name touched is 'JOHN QUINCY SMITH I', so the 2.x three-name ledger rule is intact -- all three role the letter MIDDLE and all three still report. D. The stage's class filter carried a `len(tok.text) != 1`, narrowing P3's clause to one-letter connectives. Dropped: byte-identical over 359,053 parses (the 351,400-parse grid under eight configurations, 2,175 case-table parses under three orders, 5,478 corpus parses under six), and 'John Quincy Smith og' under add(conjunctions={"og"}, suffix_words={"og"}) now reads suffix 'og' as the rule says. The three-word carve-out stays single-letter. The `frozen` comment no longer claims more than freezing does -- a neighbour's join still absorbs the piece ('Josep Carod Rovira Puig y i' -> family 'Puig y i', the parent's reading), pinned as a control. Tests. INV7b over a 21-row suffix-field grid (36 failures at dc3bdf9c, 0 here), INV10 over the connective grid (8,204 at dc3bdf9c, 5,078 narrow and 3,126 wide, 0 here), INV6b over the off-switch grid at no extra parse (944 under a lower-case-unmarking mutant), and the reach-trim guard, which checks every dropped row against the row it collapses onto and that every declared variant earns a row. The off-switch grid gains two trailing-title suffix runs: 66,752 rows over 16,576 texts, and INV6/INV1-strengthened now fail on 154/105 at dc3bdf9c where they passed. Four case rows, unit rows for the `og` class, the trailing-title bound, the withdrawal on both roles, the ALL-CAPS Catalan recipe, and the v1 off switch through `Constants`. Property module 33.8s, coverage 100% on every changed module. E, verified, no code change. `Jane Doe nee MA i Soler` loses its report because the link no longer ends the clause; the `y` twin agrees on every field and report, on all 24 grid names. The join fires on 6 of 1,568 corpus-union-cases names and one is not a Catalan link ('Henry i of England'). The no-parse paths diverge on 34 of those 1,568 against 6 at the parent. add(particles={"y"}) moves a suffix word into a name part on 130 grid names beyond the default lexicon and loses a title on 4, 71 of the 130 still reporting. F. Re-measured every count this round moved: INV6's control 2,360 -> 1,480, INV1-strengthened's 960 -> 639, INV7's 11,341 -> 7,860 (6,843 core / 1,017 v1), the blast radius 6/3/50 -> 12/3/56 (22 gain, 34 lose) with the restricted parent-vintage 2/3/46 unmoved, and the grid's own 12,880 names x eight parsers -> 66,752 rows over 16,576 texts. Three impossible 2026-09-21 dates corrected. The truncated rules.md#P3 citation in `_group.py` now quotes the whole clause and names the caller that applies it. The off-switch claim in the release note and in decisions.md now carries all three exemptions, not one. AGENTS.md says the design docs that quote a grid's counts move with it. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/design/decisions.md | 29 +- docs/design/mechanisms.md | 4 +- docs/design/rules.md | 18 +- docs/release_log.rst | 2 +- nameparser/_pipeline/_assemble.py | 33 +- nameparser/_pipeline/_group.py | 85 +++- nameparser/_pipeline/_pieces.py | 38 ++ nameparser/_render.py | 37 +- tests/v2/cases.py | 78 +++ tests/v2/pipeline/test_assemble.py | 66 +++ tests/v2/pipeline/test_group.py | 109 +++++ tests/v2/test_facade.py | 45 ++ tests/v2/test_ledger_guards.py | 6 +- tests/v2/test_parser.py | 32 ++ tests/v2/test_properties.py | 653 ++++++++++++++++++++++++-- tests/v2/test_render.py | 62 +++ tools/differential/corpus_rules.jsonl | 3 + 18 files changed, 1207 insertions(+), 95 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 52174566..dcc42bd2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Three committed contributor docs carry the parser's normative rules and their re **Guard tests** SHOULD carry a recorded negative control — the answer with the guard off, stored as data (the _EXCLUSION_EFFECT shape; see mechanisms.md's Verification shapes). -**Invariant tests over a generated grid share ONE parsed grid, and a grid pairs a text only with configurations that can read it differently.** Both rules are in `tests/v2/test_properties.py` (`_connective_findings`, `_off_switch_findings`, `_rows`) and both were bought at a price: four tests each walking the same 170,100-row grid, and a grid that was a full cross product of texts × lexicons × policies, together took that module from 11s to 115s and CI's build jobs from ~5 minutes to 17–25 (under coverage every parse costs several times more, and every later PR pays it). A new invariant over an existing grid joins that grid's walk instead of opening its own; a new grid states its measured runtime where it is built. Trim a grid by SHAPE — a variant lexicon reaches only the texts holding the word it adds — never by sampling, and prove the dropped rows were duplicates by signature before dropping them. Mutation counts in the tests' docstrings are counts over the grid, so re-measure them whenever the grid moves. +**Invariant tests over a generated grid share ONE parsed grid, and a grid pairs a text only with configurations that can read it differently.** Both rules are in `tests/v2/test_properties.py` (`_connective_findings`, `_off_switch_findings`, `_rows`) and both were bought at a price: four tests each walking the same 170,100-row grid, and a grid that was a full cross product of texts × lexicons × policies, together took that module from 11s to 115s and CI's build jobs from ~5 minutes to 17–25 (under coverage every parse costs several times more, and every later PR pays it). A new invariant over an existing grid joins that grid's walk instead of opening its own; a new grid states its measured runtime where it is built. Trim a grid by SHAPE — a variant lexicon reaches only the texts holding the word it adds — never by sampling, and prove the dropped rows were duplicates by signature before dropping them. Mutation counts in the tests' docstrings are counts over the grid, so re-measure them whenever the grid moves -- and so do the DESIGN DOCS that quote them. A recorded control reaches `docs/design/decisions.md` and `docs/design/mechanisms.md` as well as the docstring, nothing checks those copies, and a grid change silently makes them wrong about a number they present as measured: the #397 second review moved four such figures at once (2026-09-20). Move them in the same commit, and where the older number is worth keeping, label it with the population it was measured over rather than deleting it. ## Commands diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 4ad2f923..d6a00234 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -13,6 +13,12 @@ Entry conventions: - **`Open:`** — unresolved questions as issue links with one-line handles. The ISSUE is canonical; this block never restates it. Two keyings: under a rule ID for questions about the rule, and — like `Excluded:` — keyed to a VOCABULARY SET for contested memberships, the category neither `deviates:` nor `Excluded:` covers: the rule is right and a word's set membership is the question (the shape: `rai` in the suffix acronyms and `swami` absent from the given-name titles, both since answered — see #suffix-acronym-collisions and #indic-honorifics). Place the block beside the set's `Excluded:` entries so a wordlist editor meets both. - **Weighing entries** for contested questions: the options considered, each option's intended effect, and the accepted costs of the option chosen. The costs accepted here are the artifacts rules.md lists under the rule's `Accepted:` consequences; the two link by rule ID. +### A1 — the ambiguity report + +- 2026-09-20 #397 (second review) — A REPORT MUST NOT CONTRADICT THE READING BESIDE IT, and the shape could not arise before this cycle. `classify`'s `conjunction-or-initial` fork offers exactly two readings and its detail says which it took ("it is read as an initial"). Where the rest of the parse then roles that letter SUFFIX, the fork resolved to NEITHER branch: `JOHN QUINCY SMITH I` carried the connective-or-initial beside a `suffix-or-name` saying the same token reads as a generational suffix, and `john smith i`, `HENRY I`, `SMITH, JOHN I`, `JOSEP CAROD I ROVIRA III` and `abdul e i` were the same. `i` is the first word that is both a MARKED connective and suffix vocabulary, so no parse before #397 could reach it — which is why the rule is written down now rather than having been. + DECIDED: the report is WITHDRAWN where the role is SUFFIX or TITLE, in `_pipeline/_assemble.py`, the one place that sees final roles and the ambiguity list together — and beside a withdrawal that was already there, for a report whose referents did not survive assembly. That is the narrow reading of mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE rather than an exception to it: classify still owns the fork and still takes it where it is taken; what goes is a report whose subject the parse went on to read as something else. Emitting later was the alternative and was rejected as more invasive for the same answer — the fork's inputs (the case class, the own-words span) live in classify and moving the emitter moves them. + MEASURED: over the review's 43,925-name grid under eight configurations, 9,806 rows on 1,637 distinct names lose exactly one `conjunction-or-initial` and nothing else — no report of any other kind moves and none is gained. On the deduped corpus-union-cases population the only name affected is `JOHN QUINCY SMITH I`, a row this round added, so the ledger's three-name `fix(#397) the Catalan link reports a connective-or-initial in a one-case name` rule is untouched: all three of its names role the letter MIDDLE and all three still report. The invariant is `tests/v2/test_properties.py::test_no_report_contradicts_the_reading_beside_it`, which states it twice — no connective-or-initial on a token roled SUFFIX or TITLE, and no token named by two reports whose details assert different readings — and fails on 8,204 rows at `dc3bdf9c`, 0 here. + ### A2 — the empty name - 2026-07 (v2 core, PR #288) — v1 kept parse(".") as first="."; 2.0 empties content-free input instead. The born-empty-ambiguity survival was a review fix in the rc1 arc: an unbalanced-delimiter report must outlive the emptying, or malformed input becomes indistinguishable from blank input. @@ -211,7 +217,7 @@ the fullwidth-colon marker (旧姓：佐藤 arrives as one word; the head-peel q MEASURED, and the pair of numbers is the argument. A new property grid — 4,590 clause texts (five heads including a Catalan one with its own link, all seventeen shipped markers, six clause bodies, five trailing runs, comma and no-comma) against the default policy, both family-first orders and strict comma, 15,810 rows, of which 10,540 hold a link the `y` spelling puts between two birth-name words — carries two invariants in `tests/v2/test_properties.py`. THE y TWIN (`y` is the same connective with the ambiguity removed, so it is an oracle and not a second subject): the `i` parse's seven fields equal the `y` parse's, letter for letter apart from the link. It fails on ALL 10,540 guarded rows at 0fbcaa0b AND at the parent 46651750 — no clause shape holding a joining link read as its twin did — and on 0 here. NO BIRTH-NAME WORD IN THE CURRENT NAME, delimited by the twin's own maiden tokens: 28,985 tokens at 0fbcaa0b against 22,185 at 46651750, and 0 here. The GAP between those two is exactly what this branch added, and it is why the y twin alone would not have told the story: the twin fails identically at both commits while the harm doubled between them. THE GUARD IS NARROWER THAN "the twin keeps the link", by measurement rather than by preference. `Jane Doe nee Puig y` keeps its `y` in maiden 'Puig y' while `Jane Doe nee Puig i` reads maiden 'Puig' with suffix 'i' — deliberately, the link joining nothing there — so a guard asking only whether the twin kept the letter would demand the two agree where the rules say they must not. `Puig y III` and `y Soler` are the same shape from the other two sides, and the grid keeps all three as the bodies the guard has to refuse. INV6's off-switch exemption is NOT widened, and the reason is the one AGENTS.md gives for not widening an exemption a grid cannot reach: measured 2026-09-20, 1,680 rows of that grid carry both a class letter and a marker and 0 of them put the letter after the marker, its generator placing the connective among the name's own words. A clause link has its own grid instead. - TWO SECOND-ORDER MOVEMENTS, both recorded rather than repaired. The two one-case spellings part company, which is P3's own Accepted clause reaching a taken marker rather than a declined one: classify leaves a letter after a marker to the mixed-case rule, so `JANE DOE NEE PUIG I SOLER` never moved (the capital is an initial, and the initial veto kept the walk going all along) while `jane doe nee puig i soler` goes from family 'doe i soler' to maiden 'puig i soler'. And a report is GAINED where a longer clause ends on an ambiguous credential: `Jane Doe nee Puig i Ma` keeps maiden 'Puig i Ma' and says `suffix-or-name` where 0fbcaa0b read family 'Doe i Ma' in silence — the clause's own emitter, reaching a word the truncation had put out of its reach. + TWO SECOND-ORDER MOVEMENTS, both recorded rather than repaired. The two one-case spellings part company, which is P3's own Accepted clause reaching a taken marker rather than a declined one: classify leaves a letter after a marker to the mixed-case rule, so `JANE DOE NEE PUIG I SOLER` never moved (the capital is an initial, and the initial veto kept the walk going all along) while `jane doe nee puig i soler` goes from family 'doe i soler' to maiden 'puig i soler'. And a report is GAINED where a longer clause ends on an ambiguous credential: `Jane Doe nee Puig i Ma` keeps maiden 'Puig i Ma' and says `suffix-or-name` where 0fbcaa0b read family 'Doe i Ma' in silence — the clause's own emitter, reaching a word the truncation had put out of its reach. A THIRD, found by the second review and the mirror of that one: a report is LOST where the clause takes a class member it used to end at. `Jane Doe nee MA i Soler` reads maiden 'MA i Soler' in silence where the parent 46651750 read maiden 'MA' with a `suffix-or-name` on it. That FOLLOWS from the link no longer ending the clause and is not a second decision: the report is the walk's own, raised on the LAST word the clause kept where a trailing rule was asked about it, and with the link joining, 'MA' has a name word behind it and is no longer that word — which is exactly what `Jane Doe nee MA Smith` has always done with the same acronym. The `y` twin is the oracle and it AGREES, in fields and in reports: `Jane Doe nee MA y Soler` reads maiden 'MA y Soler' in silence at the parent and here. Measured 2026-09-20 over the review grid, 24 names lose the report this way and all 24 agree with their `y` twin on every field and every report. ### N3 — the lone-word nickname rule @@ -355,13 +361,18 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf - 2026-09-13 #383/#479 — AMENDS the 2026-08-27 #445 entry under M4 below, which calls `initial` "the `initial` shape tag": since this change the tag is no longer purely a shape claim, because a marked single-letter connective in a one-case name carries it without being initial-SHAPED. rules.md#M4's statement now says "read as an initial" for that reason, and M4's carve-out itself is unchanged — it still reads the tag classify recorded rather than a predicate of its own. - 2026-09-20 #397 — DECIDED AND SHIPPED: `i` joins `conjunctions` and `conjunctions_ambiguous` together, which is what the 2026-09-13 entry above said would happen and which half of it the Open block below called contested. `parse("Josep Carod i Rovira")` gives family "Carod i Rovira" where every release from 1.4.0 through 2.3.0 gave middle "Carod i" with family "Rovira"; `parse("Josep Lluis Carod i Rovira")` gives middle "Lluis" with that same family; and `parse("Carod i Rovira, Josep")` gives family "Carod i Rovira" where they read family "Carod Rovira" and took the link into `suffix` as a generation marker (all three measured 2026-09-20 on the released 1.4.0, 2.0.0, 2.1.0, 2.2.0 and 2.3.0 wheels from a throwaway environment, this tree shadowing a wheel otherwise). - THE OFF SWITCH is `Lexicon.default().remove(conjunctions={"i"})`, and on the v1 surface deleting `i` from `C.conjunctions`; both restore all three readings above byte for byte, measured 2026-09-20. That is a PROPERTY rather than an anecdote, and it is this change's durable statement: a name's fields equal its reading with the letter out of the vocabulary UNLESS a link joined between two name words, or the two readings disagree about the letter being an initial. `tests/v2/test_properties.py::test_a_link_that_joins_nothing_changes_no_field` runs it over a 12,880-name grid times eight parsers and carries the recorded control — 2,360 failing parses at `c8550b64`, the commit the review was written against, and 0 here. + THE OFF SWITCH is `Lexicon.default().remove(conjunctions={"i"})`, and on the v1 surface deleting `i` from `C.conjunctions`; both restore all three readings above byte for byte, measured 2026-09-20. That is a PROPERTY rather than an anecdote, and it is this change's durable statement, and it takes THREE exemptions rather than two once case repair is included — the earlier wording of this sentence named two and the release note named none. A name's FIELDS equal its reading with the letter out of the vocabulary unless (1) a link joined between two name words or (2) the two readings disagree about the letter being an initial; its REPAIR, plain and forced, equals the off switch's unless one of those two holds or (3) the parse placed the letter among the NAME words as a connective, where R4 keeps it lower-case and the off switch — for which it is no connective at all — title-cases it: `parse("Dr. John i Smith").capitalized(force=True)` keeps 'i' against the off switch's 'Dr. John I Smith', and `Carod i Rovira` and `Josep i Rovira` are the same shape. Measured 2026-09-20 over the shipped grid's 28,448 class-letter rows: 1,462 take the join exemption, 9,296 the initial-reading one, and 2,228 more reach the third, which is INV7's alone. `tests/v2/test_properties.py::test_a_link_that_joins_nothing_changes_no_field` runs it over a 16,576-text grid paired with the configurations that can read each text differently — 66,752 rows, of which 28,448 carry a class letter and are parsed twice — and carries the recorded control: 1,480 failing parses at `c8550b64`, the commit the first review was written against, 154 at `dc3bdf9c`, and 0 here. (Quoted over the full cross product of texts × lexicons × policies the earlier numbers were 2,360 and a 12,880-name grid times eight parsers; the shipped grid is trimmed by SHAPE and grew again with the second review, so the test's own docstring is the live figure and this is a dated snapshot.) THE COUNT REACHES THE CLASS (Derek, 2026-09-20), which ANSWERS the C-ii question the 2026-08-21 #418 bullet above left open — "whether a word's membership in one vocabulary should suppress it from a count gating a rule keyed on another". It does not: a single-letter connective counts as a name word toward P3's three-word carve-out whatever else the vocabulary says it is, and `i` is also the roman numeral. The alternative was rejected on its own arithmetic rather than on taste — counting the link as the generation it also spells lets a connective raise the bar for ITS OWN join, so "Josep Carod i Rovira", the shortest linked name and the commonest one, would be the only shape that failed to link while the four-word spellings joined. The arithmetic runs the other way too, which is why the count is scoped to where the word is PLACED TO JOIN: a connective joining nothing would otherwise LOWER the bar for an unrelated one, and "Carod y Rovira i" is that shape — it counted the trailing generation and let the "y" join. It reads given "Carod", middle "y", family "Rovira", suffix "i" here, the same FIELDS the parent gave — its initials moved, but under R3's rule rather than this one. ACCEPTED CONSEQUENCE OF REACHING THE CLASS, and the one row the design's reading tables show it touching: under `Lexicon.default().add(particles={"y"})`, `Juan Velasquez y Garcia` goes from given "Juan" / middle "Velasquez" / family "y Garcia" to given "Juan" / family "Velasquez y Garcia" — the custom lexicon now AGREES with the default one, which is the right direction — and its `family_base` becomes "Velasquez Garcia", a pre-existing R2 limit (`family_base` drops a particle wherever it stands, not only leading) surfaced rather than created here. Measured 2026-09-20 on this branch. - THE BOTH-SIDES SCOPE, AND THE REVIEW THAT FIXED IT. A connective that is also generational vocabulary joins only where a NAME WORD stands on each side of it — a word the rest of the parse reads as a name word rather than as a credential, the trailing numeral or a title, looked for past any run of connectives standing between. The FIRST implementation tested POSITION (an interior piece) rather than the neighbours' CLASS, and swallowed a credential: "Josep Lluis Carod i III" read family "Carod i III" where the "III" is the generation. An independent review found it. It is recorded here because the invariant that ought to have caught it cannot: INV1 inspects the part the join PRODUCED, where the absorbed credential is itself the name word standing on the right, so that name SATISFIES it — the general lesson is a field note in mechanisms.md of this date, and the instrument is `tests/v2/test_properties.py::test_a_trailing_credential_never_joins_into_a_name_part`, which asks the off-switch parse instead and carries its own control (960 failing parses at `c8550b64`, 0 here). + WIDER THAN ONE ROW, and the second review measured how much: the design's reading tables show one name because they hold one of this shape, not because one moves. Over the review's 43,925-name generated grid, `add(particles={"y"})` moves a word the parent roled SUFFIX into a name part on 130 names BEYOND what the default lexicon already moves there — "ABDUL Y ROVIRA MA" goes from given "ABDUL" / family "Y ROVIRA" / suffix "MA" to given "ABDUL Y ROVIRA" / family "MA" — and 71 of those 130 still report `suffix-or-name` on the moved word, so the reading is marked rather than silent. Four more lose a title, all four the same name in its four spellings ("Jr. y Soler Puig MA" and its casings). The direction is the one the row above describes: the custom lexicon converges on the default one, the particle joining what the connective already joined. Measured 2026-09-20. + THE BOTH-SIDES SCOPE, AND THE REVIEW THAT FIXED IT. A connective that is also generational vocabulary joins only where a NAME WORD stands on each side of it — a word the rest of the parse reads as a name word rather than as a credential, the trailing numeral or a title, looked for past any run of connectives standing between. The FIRST implementation tested POSITION (an interior piece) rather than the neighbours' CLASS, and swallowed a credential: "Josep Lluis Carod i III" read family "Carod i III" where the "III" is the generation. An independent review found it. It is recorded here because the invariant that ought to have caught it cannot: INV1 inspects the part the join PRODUCED, where the absorbed credential is itself the name word standing on the right, so that name SATISFIES it — the general lesson is a field note in mechanisms.md of this date, and the instrument is `tests/v2/test_properties.py::test_a_trailing_credential_never_joins_into_a_name_part`, which asks the off-switch parse instead and carries its own control (639 failing parses at `c8550b64` over the shipped grid, 105 at `dc3bdf9c`, 0 here; 960 over the full cross product, a dated snapshot). + THE BOUND THE CONDITION READS, AND THE SECOND REVIEW THAT FIXED IT (2026-09-20). "A name word" is looked for between assign's leading-title run and the start of its trailing suffix run, and that second bound was read by `_pieces.trailing_start`, the peel over the pieces as WRITTEN. A trailing TITLE hides the suffix run from that peel — it takes nothing and answers `len(pieces)` — so the condition was told a credential was a name word and the join swallowed it: `John Quincy Adams i MA Prof.` read family 'Adams i MA' with no report and an `initials()` of "J. Q. A. M.", where `John Quincy Adams i MA`, the same name one title shorter, reads family 'Adams' with suffix 'i MA' and reports the acronym. `John Quincy i MA Prof.`, `Josep Lluis i MA Prof.` and their one-case spellings are the rest of it. THE FIX IS H5's OWN READING, `_pieces.trailing_start_past_titles`, which runs the peel and the title chain to the fixed point `tail_reading` already computes for assign — so the join and the placement agree about where the name ends, which is what H5's sentence says they must. Asked once per segment and only where such a connective was found, so no ordinary name pays it; frame-neutral on `tools/perf/call_count.py` (parse=406.00, facade=443.00). Measured over the review's 43,925-name grid under eight configurations: 27 rows move, on 9 distinct names, every one of them onto the reading the off-switch parser gives. + NOT FIXED AND RECORDED INSTEAD, because it is neither this rule's nor this branch's: the PARTICLE CHAIN reads the same bound the same blind way, so a trailing title lets it run to the end of the segment. `de la ma prof.`, `de la e ma prof.`, `van der berg ma prof.` and `Josep de Carod y Rovira MA Prof.` all put the title inside the family at the parent 46651750 and at 1.4.0, with no letter of this class in them at all. What the branch does is send MORE names down it — once `i` is marked, a one-case `de la i ma` stops ending in a suffix piece and the chain reaches past it — so `de la i ma prof.` and `de la i rovira prof.` now read as their `e` twins always have. 25 names of the review grid are in this family and none of them is a `_name_word_beside` case. It is the #535 boundary and belongs to its own change; `tests/v2/test_properties.py`'s INV1-strengthened carries a LINK-FREE CONTROL so the invariant can tell the two apart, and says so where it is asked. + THE CLASS TEST IS NOT A LENGTH TEST, corrected by the second review (2026-09-20). The stage's own filter carried a `len(tok.text) != 1` beside the two tag tests, narrowing this clause to ONE-LETTER connectives where the rule names a class. Caller-reachable: under `Lexicon.default().add(conjunctions={"og"}, suffix_words={"og"})`, `John Quincy Smith og` read family 'Smith og' with the filter and reads family 'Smith' with suffix 'og' without it — the rule's own answer. DROPPED, measured byte-identical over every oracle in use (fields, reports, `initials()`, `capitalized()` plain and forced, every token's role) across 359,053 parses: the 351,400-parse review grid under eight lexicon/policy/locale configurations, the 2,175-parse sweep of `tests/v2/cases.py` under three orders and the 5,478-parse sweep of the differential corpora under six. It has to be — `i` is the only member of the class in the default vocabulary and in every locale pack, and it is one letter. The THREE-WORD carve-out stays single-letter, because that is what its own sentence says. THE SCOPE'S ALTERNATIVES, MEASURED 2026-09-20 rather than argued, by building each alternative from this tree and sweeping the 1558 non-empty corpus-union-cases names under eight configurations (default, `lenient_comma_suffixes=False`, both family-first orders, and the four locale packs). ABSENT — no both-sides condition at all — differs on five names, and every one of them is a row this bundle added: "John Quincy Smith i" would read family "Smith i", "Josep Lluis Carod i III" family "Carod i III", "Josep Lluis Carod i V" family "Carod i V", "Carod y de Rovira i" given "Carod y de" with family "Rovira i", and "Rovira, Josep Carod i Jr." middle "Carod i Jr.". Nothing that stood in the corpus at the parent separates absent from narrow, so those rows ARE the evidence and pruning one retires it. WIDE — the same condition with the generational-vocabulary test dropped, so it reaches every single-character connective — differs on five as well, and THREE of those were in the corpus already: "Mr. & Mrs. John Smith" would read title "Mr." with given "&" and middle "Mrs. John", "1 & 2, 3 4 5, Mr." would lose "2" out of the family, and "Jane née Jr y Jones" would read middle "née Jr y" with family "Jones". So the class test is not decoration — widening the rule past the link-and-generation class breaks "Mr. & Mrs." — and narrow is both what the design asked for and what measurement supports. (A "0 differences over 1545 names" figure was measured for this scope before the review supplied the class test and before those rows existed; it is superseded here.) - ACCEPTED, THE TWO-WORD SHAPE: "Carod i" keeps given "Carod" with suffix "i", and "Josep Carod i" keeps family "Carod" with suffix "i", because a connective with nothing to its right is connecting nothing. 2.0.0 through 2.3.0 read BOTH of those the same way; 1.4.0 read the two-word one differently, giving family "i" with given "Carod" (measured on the wheels 2026-09-21, correcting an earlier draft of this bullet that claimed 1.4.0 parity for it). That does not move the accepted cost, because the 1.4.0 difference is not this rule's: the two-word shape's routing left 1.4.0's reading in 2.0, which is why the case row carries `fix(suffix-routing)` rather than a connective classification. What this rule decides is that the link does not join there, and 2.0 through 2.3 agree with it. The ledger rule that called this NOT WANTED is rewritten in place rather than deleted — it still explains the name, and the standing is decided now rather than unwanted. + ACCEPTED, THE TWO-WORD SHAPE: "Carod i" keeps given "Carod" with suffix "i", and "Josep Carod i" keeps family "Carod" with suffix "i", because a connective with nothing to its right is connecting nothing. 2.0.0 through 2.3.0 read BOTH of those the same way; 1.4.0 read the two-word one differently, giving family "i" with given "Carod" (measured on the wheels 2026-09-20, correcting an earlier draft of this bullet that claimed 1.4.0 parity for it). That does not move the accepted cost, because the 1.4.0 difference is not this rule's: the two-word shape's routing left 1.4.0's reading in 2.0, which is why the case row carries `fix(suffix-routing)` rather than a connective classification. What this rule decides is that the link does not join there, and 2.0 through 2.3 agree with it. The ledger rule that called this NOT WANTED is rewritten in place rather than deleted — it still explains the name, and the standing is decided now rather than unwanted. ACCEPTED, A CONNECTIVE RUN: "Henry i of England" reads given "Henry i of England" with a `given-or-family` report, where the parent read given "Henry" with family "i of England". The "i" and the "of" merge as a run and the run joins the whole name into one piece — the same thing "Duke of Edinburgh" already did. + ACCEPTED, THE LOWER-CASE REGNAL NUMERAL, and it is the shape that makes the join FIRE with no report at all on input nobody would call Catalan: a mixed-case name with a lowercase `i` between two name words joins like any other link, so "John Paul i Smith" reads family "Paul i Smith" where the parent read middle "Paul i", and "john i Smith Jones" reads given "john i Smith" where it read middle "i Smith". Nothing marks the reading, because nothing is ambiguous by this rule's lights — the letter IS connective vocabulary and it IS joining two name words — and a report there would fire on every Catalan name too (rules.md#A1's hesitating-reader test). The frequency is the argument for accepting it: measured 2026-09-20 over the deduped corpus-union-cases population of 1,568 non-empty names under the three orders, the join fires on SIX names and five of them are the Catalan rows this bundle added; the sixth is "Henry i of England", the paragraph above. A regnal numeral written lowercase between two name words is the shape that would pay, and the corpus holds one. ACCEPTED, THE PARTICLE ON THE RIGHT. A particle is a name word for the both-sides test, so the link joins across one: "Josep de Carod i Rovira" reads family "de Carod i Rovira", which is the Catalan reading, while "Josep Carod i de Rovira" reads middle "Carod i de" with family "Rovira" where the parent read middle "Carod i" with family "de Rovira" — the join takes the particle as its right-hand neighbour and the particle chain then follows the joined piece instead of the tail word. Recorded rather than repaired: which side a particle standing between a link and a surname belongs to is P2's question and not this one. ACCEPTED, AND THE HONEST STATEMENT OF THE ONE-CASE HALF, written out because this branch's own first commit message overclaimed that one-case names keep their reading. `i` is in the MARKED subset, so in a name written wholly in one case it reads as an INITIAL and reports `conjunction-or-initial`, exactly as `e` has since #383/#479. For "JOSEP CAROD I ROVIRA" and "josep carod i rovira" the fields are indeed unchanged and only the report is new. But a one-case name reports wherever a bare `i`/`I` stands among the NAME'S OWN WORDS, which is a great many more names than the Catalan ones — "JOHN I SMITH" and "john i smith" one report each, "JOHN SMITH I" and "john smith i" two, "HENRY I" and "henry i" three. A letter inside a maiden clause is read by the clause's rules and stays silent, exactly as `e` does ("JANE DOE NEE I JONES" reports nothing), which is the own-words scope the 2026-09-13 #383/#479 entry above defines. And in ALL-LOWER names the initial reading MOVES FIELDS wherever the parent read the lower-case letter as the generation. Each such name now reads as its ALL-CAPS twin already did: "rovira, i" gives given "i" where it gave suffix "i" ("ROVIRA, I" already gave given "I"); "john smith i jr" gives middle "smith" with family "i" where it gave family "smith" with suffix "i jr" ("JOHN SMITH I JR" already did); "maier, amy i, jr." gives middle "i" with suffix "jr." where it gave suffix "i, jr." (the corpus name "Maier, Amy I, Jr." reads middle "I" and does not move); and "josep de carod i rovira" gives family "de carod i rovira" where it gave middle "de carod i" with family "rovira". None of those four is a corpus name; all are measured 2026-09-20 on this tree and against the parent's. NO CHANGE TO THE EXCLUDED BLOCK, recorded as a decision rather than left implicit (Derek, 2026-09-20). It now lists "y" against TWO marked letters rather than one, and that is still right: the argument that put "y" outside is about "y" — the commonest Hispanic compound and this library's oldest fixture — while "i" matches "e" exactly, a bare I initial being as common as a bare E. @@ -1279,10 +1290,11 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-20 #461 — DONE, and the clause is settled for every group at once. THE RULE AS TAKEN: a connective contributes nothing where it is JOINING — a part holding another name word for it to join, the part's working particles set aside — and where its part holds nothing else it is joining nothing, so it initials like any other name word and agrees with the base. That picks ONE of the four candidate answers the 2026-08-29 bullet under `decisions.md#R2` listed for "John and Jane Smith" and makes the other three unreachable: `parse("John and Jane Smith").initials()` is "J. J. S.", not the "J. a. J. S." of 2.0 through 2.3, not P3's one-name-word "J. S.", and not 1.4.0's run-together "J a J. S.". THE CRITERION IS NEVER A WORD COUNT, which is what keeps `parse("Jon Dough and")` at "J. D." over base "Dough and" and `parse("Juan Velasquez y Garcia")` at "J. V. G." — a part of two words where one is the connective is still a part where it joins, and a part of three is no different. BOTH VIEWS, ONE MARK. The fact is about a PART, so it is decided once where the parts are settled and recorded on the token (`UNJOINED_CONJUNCTION_TAG`, mechanisms.md#MARK-DONT-STRIP), and the core view and the v1 facade both read it. That is the direct answer to what the first #461 attempt found: a mark honored for some of a part's words and not for one of them is honoring nothing. A SEPARATE marker rather than a widening of `UNJOINED_TAG`, because that one is also read by `family_particles`/`family_base` and by `_cap_word`, and none of those three may move. `Parser.revise` recomputes both marks, the same reason `_remarked` recomputes the first. - BLAST RADIUS, MEASURED 2026-09-20 against the parent commit `46651750` over the deduped corpus-union-cases set under eight configurations, both processes asserting their own `nameparser.__file__`. Over the population AS IT STANDS — 1558 non-empty names, grown by this bundle's own rows — the default order moves 6 names' roles, 3 more names' reports with the roles fixed, and 50 names' `initials()`, 19 gaining a letter and 31 losing one; `HumanName.initials()` moves on THE SAME 50 names with THE SAME values. Family-first moves 5 / 3 / 28 and `FAMILY_FIRST_GIVEN_LAST` 6 / 3 / 58; the 35 names that move their initials under given-last and not under the default order are mostly the ones whose joined run lands in the GIVEN group there, `Juan Velasquez y Garcia` going from "V. y. G. J." to "V. G. J." — family-first now agreeing with given-first on it. Restricted to the 1544 names that stood in the corpus at the parent — the comparable population, since the rows this bundle added are themselves movers — the same sweep reads 2 / 3 / 46 at the default order, 18 gaining and 28 losing, with 1 / 3 / 24 family-first and 2 / 3 / 52 given-last. RECOMPUTE by parsing every name of the deduped `tools/differential/corpus*.jsonl` glob plus `tests/v2/cases.py`'s texts under those eight configurations in two processes, one on this tree and one on the parent's, and diffing the seven role fields, the ambiguity kinds and `initials()` on both surfaces. The gate cannot answer this on its own: its `_initials` pseudo-field sees only names whose roles stayed put, so a count taken from the classified summary is a FLOOR. - TWO OF THE GAINS ARE 1.4.0 PARITY RESTORED, and that is what retires two ledger rules: `JUAN Y GARCIA` gives "J. Y. G." and `محمد و علي` gives "م. و. ع.", which is what 1.4.0 gave for both. The two ranges differ and an earlier draft of this bullet ran them together, so state them apart (measured on all five wheels 2026-09-21): `محمد و علي` gave "م. ع." at 2.0.0, 2.1.0, 2.2.0 AND 2.3.0, while `JUAN Y GARCIA` gave "J. G." at 2.0.0, 2.1.0 and 2.2.0 and went back to 1.4.0's "J. Y. G." at 2.3.0 — the reason being the one the 2026-09-13 #383/#479 entry under P3 records, and the 2.4.0 release note says the same. The readmission is a parity RESTORATION on those two names rather than a deviation, and the given-group half in the other direction is the deviation — see rules.md#R3's Accepted clause, which states both. + BLAST RADIUS, MEASURED 2026-09-20 against the parent commit `46651750` over the deduped corpus-union-cases set under eight configurations, both processes asserting their own `nameparser.__file__`. RE-MEASURED 2026-09-20 at the end of the second review round, the population having grown again with its own rows. Over the population AS IT STANDS — 1568 non-empty names — the default order moves 12 names' roles, 3 more names' reports with the roles fixed, and 56 names' `initials()`, 22 gaining a letter and 34 losing one; `HumanName.initials()` moves on THE SAME 56 names with THE SAME values. Family-first moves 11 / 3 / 34 and `FAMILY_FIRST_GIVEN_LAST` 12 / 3 / 64. The twelve role movers at the default order are "Carod i Rovira, Josep", "Doe, Jane nee Puig i Soler", "Henry i of England", "Jane Doe nee Puig i Ma", "Jane Doe nee Puig i Soler", "Josep Carod i Rovira", "Josep Carod i Rovira III", "Josep Carod i de Rovira", "Josep Lluis Carod i Rovira", "john smith i jr", "maier, amy i, jr." and "rovira, i" — seven of the twelve are rows this bundle added, which is why the restricted figure below is the comparable one. Restricted to the 1543 names that stood in the corpus at the parent, the same sweep reads 2 / 3 / 46 at the default order, 18 gaining and 28 losing, with 1 / 3 / 24 family-first and 2 / 3 / 52 given-last — unmoved by the second review, whose report change touches only "JOHN QUINCY SMITH I", a name the round itself added. (The 1558-name sweep that stood here read 6 / 3 / 50 with 19 gaining and 31 losing, and 5 / 3 / 28 and 6 / 3 / 58 for the two family-first orders; it is a dated snapshot over a smaller population.) RECOMPUTE by parsing every name of the deduped `tools/differential/corpus*.jsonl` glob plus `tests/v2/cases.py`'s texts under those eight configurations in two processes, one on this tree and one on the parent's, and diffing the seven role fields, the ambiguity kinds and `initials()` on both surfaces. The gate cannot answer this on its own: its `_initials` pseudo-field sees only names whose roles stayed put, so a count taken from the classified summary is a FLOOR. + TWO OF THE GAINS ARE 1.4.0 PARITY RESTORED, and that is what retires two ledger rules: `JUAN Y GARCIA` gives "J. Y. G." and `محمد و علي` gives "م. و. ع.", which is what 1.4.0 gave for both. The two ranges differ and an earlier draft of this bullet ran them together, so state them apart (measured on all five wheels 2026-09-20): `محمد و علي` gave "م. ع." at 2.0.0, 2.1.0, 2.2.0 AND 2.3.0, while `JUAN Y GARCIA` gave "J. G." at 2.0.0, 2.1.0 and 2.2.0 and went back to 1.4.0's "J. Y. G." at 2.3.0 — the reason being the one the 2026-09-13 #383/#479 entry under P3 records, and the 2.4.0 release note says the same. The readmission is a parity RESTORATION on those two names rather than a deviation, and the given-group half in the other direction is the deviation — see rules.md#R3's Accepted clause, which states both. THE FRAME BUDGET WENT DOWN: `tools/perf/call_count.py` reads `py3.11 parse=406.00 facade=443.00` where the parent read `412.00 / 449.00`. The two marks now come off ONE walk over the roles instead of two generator expressions, and on 3.11 a genexp costs a frame for its creation and one per resume, so the reference name's two non-empty parts pay exactly six. `_CALL_BASELINE[(3,11)]` is `{"parse": 410, "facade": 447}` with `_BAND = 0.02`, so the bands are [401.8, 418.2] and [438.06, 455.94] and both figures land inside: NO baseline edit and no `decisions.md#parse-cost` decision is owed. PEP 709 inlined comprehensions and not generator expressions, so 3.12 through 3.15 should drop by the same six. THE NO-PARSE PATHS KEEP THE OLDER READING, the same degradation the 2026-09-13 #528 bullet above accepted and for the same reason: a token the parse never saw carries no mark, so the view falls back to the vocabulary. Measured 2026-09-20, `HumanName("JUAN GARCIA Y LOPEZ").initials()` is "J. G. L." live and "J. G. Y. L." after `pickle`, `copy.copy` or `copy.deepcopy`; a name built from keyword fields carries no tags either, and so does one whose `*_list` property a subclass overrides. The remedy is the v1 one — assign `full_name` again — since the facade has no `revise`. + HOW FAR IT REACHES, over a STATED population rather than an example, because this bundle MULTIPLIES it and the earlier wording did not say by how much: over the deduped `tools/differential/corpus*.jsonl` glob plus `tests/v2/cases.py`'s texts — 1,568 non-empty names — the v1 `initials()` of a pickled, copied or deep-copied name differs from the live parse's on 34 names here against 6 at the parent 46651750, measured 2026-09-20 in two processes each asserting its own `nameparser.__file__`. #461's lone-connective half is what multiplies it: every part whose only word is the connective initials live and drops after the round trip, so `HumanName("Juan de y").initials()` is "J. y." live and "J." after `copy.deepcopy`. RECOMPUTE by building that population and comparing `initials()` before and after each of the three state paths. A larger figure measured over a generated grid says nothing this does not — the population is what the number is about. ### R4 — case repair reads the unjoined mark @@ -1305,12 +1317,13 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-08-29 — WHY THE BOUNDARY WENT UNNOTICED UNTIL #407, which is where a future reader should look for it. For an ALL-PARTICLE part the other three tag-driven views give the same answer through `replace()` and `revise()` alike: measured over `de la`, `van der`, `do`, `de` and `van de la`, all five agree on `family_particles=''`, on a `family_base` holding the whole part, and on initials from every word. They converge because an UNTAGGED part and a MARKED all-particle part reach the same place by different routes — untagged, no word is recognized as a particle; marked, none is ACTING as one — and all three views only ask which words are particles. Case repair is the one view that asks a second question, since it must also decide whether to lowercase, so it is where the two routes first come apart. The mirror case confirms the reading: on a MIXED part the convergence is the other way round — `de la vega` and `van der berg` diverge in all three views between `replace()` and `revise()` (`replace()` reports particles `''` and base `'de la vega'` where `revise()` reports `'de la'` and `'vega'`) and AGREE on case repair, R4's all-particle clause not reaching them. So before #407 the distinction was invisible on exactly the shape the clause is about, and visible only on shapes the clause does not govern. -- 2026-09-20/21 #461/#397 — THE GROUNDING MOVED, AND THEN A DEFECT THE GROUNDING EXPOSED. R4's sentence rested on R3 by name ("the carve-out R3 states for initials"), and R3's carve-out is conditional now while R4's is not, so the cross-reference is CUT and replaced by this rule's own reason: a connective that initials because it joins nothing is still not written the way a name is written. Measured 2026-09-20, plain and forced, over `juan y`, `john and jane smith`, `duke of edinburgh`, `juan de y`, `juan y garcia`, `JUAN Y GARCIA`, `josep carod i rovira`, `JOSEP CAROD I ROVIRA` and — under `add(particles={"y"})` — `Anh y Van`: all nine are byte-identical to the same call at the parent, re-measured after the 2026-09-21 repair below and still byte-identical, and `tests/test_capitalization.py` is green. +- 2026-09-20 #461/#397 — THE GROUNDING MOVED, AND THEN A DEFECT THE GROUNDING EXPOSED. R4's sentence rested on R3 by name ("the carve-out R3 states for initials"), and R3's carve-out is conditional now while R4's is not, so the cross-reference is CUT and replaced by this rule's own reason: a connective that initials because it joins nothing is still not written the way a name is written. Measured 2026-09-20, plain and forced, over `juan y`, `john and jane smith`, `duke of edinburgh`, `juan de y`, `juan y garcia`, `JUAN Y GARCIA`, `josep carod i rovira`, `JOSEP CAROD I ROVIRA` and — under `add(particles={"y"})` — `Anh y Van`: all nine are byte-identical to the same call at the parent, re-measured after the repair below and still byte-identical, and `tests/test_capitalization.py` is green. WHAT THE PLAIN CALL DOES, stated over a population rather than over nine names, because "capitalized() does not move" was drafted for this bullet and is false as written. Swept 2026-09-20 over the 1558 non-empty corpus-union-cases names under eight configurations against the parent commit, `capitalized()` moves on SIX distinct names and every one of them is a name whose ROLES moved under the join — it renders different fields rather than treating a word differently. The correct statement is therefore: where only `initials()` moves, `capitalized()` is byte-identical; where the roles move, it follows them. WHAT THE FORCED CALL DID, AND THE DEFECT THAT WAS FOUND WRITING THIS RECORD. The sweep above was run for the PLAIN claim and turned up a second answer: `capitalized(force=True)` moved on ten further names whose roles, initials and plain repair all stayed put, every one a lower-case `i` — `parse("Carod i")` forced gave "Carod i" where 1.4.0 and 2.3.0 gave "Carod I", and so did "John Quincy Smith i", "Josep Carod i", "Lluis Carod i", "Josep Lluis Carod i III", "Josep Lluis Carod i V", "Carod y de Rovira i", "Rovira, Josep Carod i Jr.", "Carod i Rovira" and "Josep i Rovira". Nine of those were a DEFECT and are REPAIRED here; the remaining two are the rule. WHY IT WAS A DEFECT, and the line R4 now draws. In those names the parse read the letter as the GENERATION it also spells and roled the token SUFFIX — but classify had tagged it `conjunction`, and repair's conjunction conjunct read the tag without asking the role, so a generation was lower-cased as a connective. Derek's principle decides it: a connective with nothing to join is not acting as a connective. Repair now asks the ROLE the parse decided rather than the tag alone, and a suffix-roled token is repaired as the suffix it was read as. `parse("John Quincy Smith i")` forced gives "John Quincy Smith I" again on BOTH surfaces, the facade repairing through the same helper. WHAT REMAINS AND IS NOT A DEFECT: `Carod i Rovira` and `Josep i Rovira`, where the letter is a lone MIDDLE word the parse tagged `conjunction` — three-word names, so P3's carve-out leaves the letter a name word and nothing joined. Forced repair keeps it lower-case, which is R4's own rule and exactly the answer the same shape has always had for `y`: `Carod y Rovira` and `Josep y Rovira` give "Carod y Rovira" and "Josep y Rovira" too. Those two names are the whole of what #397 moves on this view. - MEASURED, and the numbers are the fix's own control. Over the same eight configurations the forced-only set is ten names before the fix and two after; the invariant `tests/v2/test_properties.py::test_a_letter_that_did_not_join_repairs_as_the_off_switch_does` — a letter that became no connective of this name repairs as the off-switch parse repairs it, plain and forced, on both surfaces — fails on 11,341 repairs at `e540d4c5` (10,424 core, 917 v1) and on 0 here, and removing the role test alone fails it on the same 11,341. The differential still cannot see any of it, case being no compared surface (the 2026-08-29 bullets above settle why), which is why this needed an invariant rather than a ledger rule. + AND THEN THE REPAIR ITSELF WAS TOO WIDE, found by the second review round (2026-09-20). The guard shipped as the suffix ROLE alone, and the role says only where a word LANDED. A plain connective can land in the suffix field without being generational vocabulary at all — the third part of a comma form is read as the suffix run whatever its words are — so `parse("Smith, John, and").capitalized(force=True)` gave 'John Smith And' where 1.4.0, 2.0.0 through 2.3.0 and the parent 46651750 all give 'John Smith and', and the same for `, y`, `, e`, `, und`, `, of` and `Doe, Jane, and Jr.`. A field SPLICED in went the same way and against R4's own Accepted paragraph, which says a field nobody read asks the vocabulary: `HumanName(first="John", last="Smith", suffix="de y")` repaired forced to 'John Smith De, Y' against 'John Smith De, y' on both wheels. THE GUARD IS NOW THE PAIR — the suffix role AND the `vocab:suffix` tag — because the clause turns on the GENERATION and the role alone cannot see it; classify's tag is the vocabulary half of the decision and reading both re-derives neither. Invisible to everything already watching: no name of the off-switch grid carries the shape (its oracle is a letter of the class, and none of these words is one) and no corpus name carries it either, which is why INV7b was written over a grid of its own. ACCEPTED, and the one reading this moves rather than restores: a field spliced to `suffix="i"` now repairs to 'John Smith i' where the parent gave 'John Smith I'. `i` is connective vocabulary on this branch and was not at the parent, so the spliced field follows `y` now — which is R4's Accepted paragraph working as written, the PARSED name being unaffected because it has a reading and the reading is the generation. + MEASURED, and the numbers are the fix's own control. Over the same eight configurations the forced-only set is ten names before the fix and two after; the invariant `tests/v2/test_properties.py::test_a_letter_that_did_not_join_repairs_as_the_off_switch_does` — a letter that became no connective of this name repairs as the off-switch parse repairs it, plain and forced, on both surfaces — fails on 7,860 repairs at `e540d4c5` (6,843 core, 1,017 v1) and on 0 here, and removing the generation conjunct outright fails it on 8,054. (11,341 / 10,424 / 917 was the same control over the full cross product before the grid was trimmed by shape and then grown by the second review; it is a dated snapshot and the test's docstring is the live figure.) The differential still cannot see any of it, case being no compared surface (the 2026-08-29 bullets above settle why), which is why this needed an invariant rather than a ledger rule. ### R5 — the case-repair gate diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 7512d12f..ef077e24 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -59,7 +59,7 @@ Problem shape. Two stages need the same answer about the same input, and the one ## RENDER-HONORS-THE-PARSE — the parse decides it, the views honor it -Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a connective the parse placed among the NAME words with nothing to join, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). Narrowed twice, the second time by a defect the record itself turned up (2026-09-21): where the parse read such a word as the GENERATION it also spells, repair was reading classify's `conjunction` tag without asking the role and lower-casing a suffix, so `parse("John Quincy Smith i")` forced gave 'John Quincy Smith i'. A view honors what the parse DECIDED, and the role is that decision where the tag is only what the word could have been -- the same distinction this entry draws between re-deriving an answer and reading the record, one level up. The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. +Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which was filed and open here and is CLOSED (2026-09-20, #461). How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is now CLOSED (see the paragraph that closes it below): `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings gave `A. V.` and `J.` on that date, the second being a rules.md#R3 example line the doc runner asserts — and BOTH of those default readings have since moved, which the closing paragraph below carries. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). CLOSED 2026-09-20 (#461), and what closed it was the RULE rather than the code: rules.md#R3 no longer excludes a connective unconditionally, so the readmission this entry called an override is what the rule now says, decided once over the whole part and recorded on the token as UNJOINED_CONJUNCTION_TAG for both initials views to read. The scoped values above STAY TRUE and are re-measured on this date: under `Lexicon.default().add(particles={'y'})`, `parse("Anh y Van")` initials `A. y. V.` and `parse("Juan de y")` `J. d. y.`, exactly as on 2026-08-29. What changed is the pair of DEFAULT-vocabulary readings quoted beside them, and both of them: `parse("Juan de y")` gives `J. y.` where it gave `J.` — the rules.md#R3 example line moved with it — and `parse("Anh y Van")` gives `A. y. V.` where it gave `A. V.`, its middle `y` holding that part alone. On THAT string the two lexicons now agree, which is the shape of the fix: the readmission was never about a caller's overlapping vocabulary, it was about a part with nothing left to join. `Juan de y` still differs between them and legitimately so — under the default vocabulary `de` is a working particle and `y` alone is the base, while under the overlap the whole part is particles and R2's own mark readmits both words. The disagreement with case repair is not closed and is not meant to be: it is narrowed to a connective the parse placed among the NAME words with nothing to join, and it is R4's own rule now rather than an oversight or a borrowing from R3 (decisions.md#R4, 2026-09-20). Narrowed twice, the second time by a defect the record itself turned up (2026-09-20): where the parse read such a word as the GENERATION it also spells, repair was reading classify's `conjunction` tag without asking the role and lower-casing a suffix, so `parse("John Quincy Smith i")` forced gave 'John Quincy Smith i'. A view honors what the parse DECIDED, and the role is that decision where the tag is only what the word could have been -- the same distinction this entry draws between re-deriving an answer and reading the record, one level up. The direction this entry teaches is unchanged and is why the paragraph is closed rather than deleted: a view that honors a record and then readmits what the record excluded is overriding a decision it never took — the remedy was to settle what the record should say, not to make the view louder. Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. ## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins @@ -195,7 +195,7 @@ Problem shape. A test pins an ordering, a sort, a dedup or a partition, and its - A skip is indistinguishable from "correctly declined": pytest turns an empty parametrize into a skip, and a filter that widens its own skip set cannot fail. After changing any selection shape, verify the guard still REACHES the code it watches — assert the selected set is non-empty, or force-a-decision on its size. - A differential corpus cannot evidence behavior keyed to OUT-of-vocabulary shapes: it holds only names someone wrote down, and an unrecognized word is by definition outside the vocabulary — a green run over the corpus proves nothing about such a rule. - The corpus can be near-blind to a WRITING CONVENTION even where the vocabulary is well covered, and a small honest count then reads as a small blast radius. Measured for rules.md#P6: of 782 corpus names, 245 carry a comma, TWO of those end in a particle, and ONE of the two clears the words-to-spare guard and actually moves — so the Dutch trailing-tussenvoegsel listing the rule exists for is essentially unsampled. Before reporting "N names move", report the size of the population that COULD move; when that is ~1, the number is evidence about the corpus. This note's first wording said "exactly ONE ends in a particle", conflating the population with the movers — the very error it exists to prevent, committed inside the correction. Caught by re-running the count against AGENTS.md's axis 1 rather than by re-reading it. -- An invariant judged on the part a JOIN PRODUCED is satisfied by whatever the join absorbed. #397's both-sides condition was stated as "a name word stands on each side of the link", and the invariant written from that sentence walks the joined part and asks whether a non-connective word stands on each side of the letter — which a swallowed credential answers yes to, being that word. `Josep Lluis Carod i III` read family 'Carod i III' and PASSED it. The merge has already destroyed the evidence, so the question cannot be re-asked downstream at all. Judge on the PRE-join classes, or against a second parse with the rule turned off — the off-switch parse puts the credential in `suffix`, and "no word the off-switch reading calls a suffix lands inside a joined name part" caught it on 960 parses where the first invariant caught none. The same pairing is worth reaching for whenever a rule MERGES tokens. +- An invariant judged on the part a JOIN PRODUCED is satisfied by whatever the join absorbed. #397's both-sides condition was stated as "a name word stands on each side of the link", and the invariant written from that sentence walks the joined part and asks whether a non-connective word stands on each side of the letter — which a swallowed credential answers yes to, being that word. `Josep Lluis Carod i III` read family 'Carod i III' and PASSED it. The merge has already destroyed the evidence, so the question cannot be re-asked downstream at all. Judge on the PRE-join classes, or against a second parse with the rule turned off — the off-switch parse puts the credential in `suffix`, and "no word the off-switch reading calls a suffix lands inside a joined name part" caught it on 639 parses of the shipped grid where the first invariant caught none (960 over the full cross product, a dated snapshot before the grid was trimmed by shape). The same pairing is worth reaching for whenever a rule MERGES tokens. - A detector that re-implements a rule's grouping will get the grouping wrong. Derive the boundary from the same vocabulary the rule reads, not from the half you happen to be thinking about: walking a particle run over the NEVER-GIVEN set alone (the rule chains through ANY particle) split "de la Vega" after "de la" and reported 50 false movers for #364, where the true count is one. Both wrong answers were plausible and printed cleanly. - Guard the whole family, parametrize over it: a defect on one of N parallel entry points hides behind a per-example test — three times in one session (a guard on one class of two, a decode hint on 3 of 5 entry points, a sync roster missing 4 copies) — and a {class}×{field}×{bad-value} parametrization is what caught each. - A growth guard needs calibration, not just existence: benchmark guards that compare n vs 4n catch the quadratic the absolute-time tests are blind to, but calibrate against the WEAKEST signal you must detect and confirm a planted regression fails across repeated runs, not once — a stochastic check "verified" on one sample verifies nothing. diff --git a/docs/design/rules.md b/docs/design/rules.md index 1b9d2cef..bd4ffd26 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1945,8 +1945,15 @@ A1. Rationale: a caller can only act on doubt that is reported. Parsing never fails on any input: where the text's structure or a word's reading is genuinely uncertain, the parse completes on the best reading and carries an ambiguity report naming the - doubt. + doubt. A report names the reading the parse took, so a report + whose fork the rest of the parse then resolved to NEITHER branch + is withdrawn rather than carried beside a reading it + contradicts: a letter both a connective and an initial, read as + the generation it also spells, is neither of the two the + connective-or-initial fork offered, and only that fork's report + goes — the generation's own stands. "Van Johnson" → ambiguities=("particle-or-given",) + "JOHN QUINCY SMITH I" → ambiguities=("suffix-or-name",) "Jane „JD Smith" → ambiguities=("unbalanced-delimiter",) "John Smith, MD, Bart" → ambiguities=("comma-structure",) "John Smith" → ambiguities=() · boundary @@ -1954,7 +1961,7 @@ A1. Rationale: a caller can only act on doubt that is reported. segmenter's own error, which propagates — a user-code error is not a content error. (Needs the optional extra to demonstrate, so no example line.) - implemented: nameparser/_pipeline/_state.py + history: decisions.md#A1 · interacts: P3, S2 · implemented: nameparser/_pipeline/_assemble.py, nameparser/_pipeline/_state.py A2. Rationale: an input with no name content names nobody, and saying so beats inventing fields from punctuation. @@ -2146,6 +2153,11 @@ R4. Rationale: case repair is a display concern, applied only on and not a borrowing from R3: a connective that initials because it joins nothing is still not written the way a name is written, while a generation is written the way a generation is written. + The GENERATION is what that clause turns on and not the field: + a connective the suffix field merely holds, which the suffix + vocabulary does not know, was read as no generation and keeps its + lowercase there like any other connective — the third part of a + comma form is the shape that puts one there. A name already written the way repair would write it comes back unchanged, measured by repair's own conventions rather than by the bearer's. A spelling written in a @@ -2159,6 +2171,8 @@ R4. Rationale: case repair is a display concern, applied only on "john smith phd" → capitalized="John Smith Ph.D." "John Quincy Smith i" → capitalized_forced="John Quincy Smith I" "Carod i" → capitalized_forced="Carod I" + "Smith, John, and" → capitalized_forced="John Smith and" + "Doe, Jane, and Jr." → capitalized_forced="Jane Doe and Jr." "juan de la vega" → capitalized="Juan de la Vega" · boundary Accepted: the clause reaches a part the parser read. A field spliced in as raw text after the parse carries no reading of its diff --git a/docs/release_log.rst b/docs/release_log.rst index 6af905f1..feb769fb 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -22,7 +22,7 @@ Release Log - **Fix a maiden marker's clause swallowing a trailing credential in silence.** ``HumanName("Jane Doe nee Smith MA")`` gives maiden ``Smith`` with suffix ``MA``, where 2.0 through 2.3 gave maiden ``Smith MA`` and said nothing; 1.4.0 read the ``MA`` as a suffix too. ``Doe, Jane nee Smith MA`` moves with it, and so do the one-case spellings ``JANE DOE NEE SMITH MA`` and ``jane doe nee smith ma``. The words a marker takes now end where a trailing credential begins, which is what the marker's other two stops -- a suffix word, a trailing roman numeral -- have always done. Until this release it was the last trailing position in the library where a word of the ambiguous credential class was read without a report, and it was order-sensitive besides: ``Jane Doe nee Smith MA PhD`` gave maiden ``Smith MA`` while ``Jane Doe nee Smith PhD MA`` gave maiden ``Smith``, so whether the word was read at all depended on which side of the unambiguous credential the writer put it. Both now give maiden ``Smith``, with suffix ``MA PhD`` and ``PhD MA``. The writing still decides, exactly as it does for the same word ending a name with no clause: ``Jane Doe nee Smith Ma`` keeps maiden ``Smith Ma``, and ``Jane Doe nee Yo-Yo Ma`` keeps a two-word birth surname whole. The one member of this class that is also a surname particle keeps the carve-out it has outside a clause -- ``Doe, Jane nee Smith DO`` gives suffix ``DO`` while ``Doe, Jane nee Smith do`` and ``Doe, Jane nee Smith Do`` keep maiden ``Smith do`` and ``Smith Do``, and the comma-less ``Jane Doe nee Smith do`` gives suffix ``do`` as ``John Doe do`` does. Either reading is now reported, and there is no third: a word the clause gives up reads as a post-nominal, or the clause keeps it and says so. A name word behind the credential ends its reach and stays silent -- ``Jane Doe nee MA Smith`` gives maiden ``MA Smith`` and reports nothing -- and this stop never takes the first word after the marker, whatever its writing says: ``Jane Doe nee MA`` keeps maiden ``MA`` and reports, the marker having announced a name where there would otherwise be none, and ``Jane Doe nee MA PhD`` keeps it too. That differs on purpose from what a certain post-nominal gets there, ``Jane Smith nee PhD`` and ``Jane Smith nee V`` leaving the marker standing as an ordinary word as before. Where no trailing rule reads the clause's tail nothing is decided and the clause keeps every word: ``Smith nee Jones MA, Jane`` and ``Smith, John, Jr nee Jones MA`` both keep maiden ``Jones MA``, unchanged and with no ``suffix-or-name`` report. A trailing title is not transparent here and the two spellings disagree -- ``Jane Doe nee Smith MA Prof.`` is unchanged and silent while ``Jane Doe nee Smith Prof. MA`` gives maiden ``Smith Prof.`` with suffix ``MA`` -- which is recorded as a boundary rather than fixed. The clause also keeps a word it cannot promise a credential reading for, which is where three shapes that look like they should move do not. Where the part the word would land in holds no name of its own there is nothing to read it as a credential, so ``Doe, Dr. nee Smith MA`` and ``Jane Doe, Jr nee Smith MA`` both keep maiden ``Smith MA`` and report. Where a join would swallow it first the same applies, and it is the birth name that would lose the word: ``Berg, abdul nee Jones MA`` keeps maiden ``Jones MA`` rather than reading first ``abdul MA``, and ``Berg, Jane van der nee Smith DO`` keeps maiden ``Smith DO`` rather than letting the particle chain carry the ``DO`` into last ``van der DO Berg``. Each of those reads as 2.3.0 read it. Delimiters settle the question outright and always did: ``HumanName("Jane Doe (nee Smith MA)")`` keeps the whole span as the maiden name and reports nothing, the writer having drawn the boundary, while ``Jane Doe (nee Smith) MA`` gives suffix ``MA`` for the word left outside it. One name is a restoration rather than a change: ``John Smith nee Jones R.A.I.`` gives suffix ``R.A.I.`` again, as 2.3.0 read it, this unreleased cycle having moved it into the maiden name when the unlisted-dotted reading above took the word out of the certain-suffix class. See the ``M2`` and ``S2`` entries of ``docs/design/decisions.md`` (closes #533) - - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). Case repair follows the reading: a lower-case ``i`` the parse read as the generation is still title-cased by ``capitalize(force=True)`` (``Carod i`` gives ``Carod I``, as every release did), while one standing among the name words keeps its lower case as ``y`` always has (``Carod i Rovira`` gives ``Carod i Rovira``, where ``Carod I Rovira`` was the pre-2.4 answer). A link inside a maiden clause stays in the birth name, which no release read that way: ``HumanName("Jane Doe nee Puig i Soler")`` gives maiden ``Puig i Soler`` with last ``Doe``, where 2.0 through 2.3 ended the birth name at the link and gave maiden ``Puig`` with middle ``Doe i``, last ``Soler`` -- and the same words would have joined into last ``Doe i Soler`` under the change above, carrying a word of the birth name into the current surname. ``Doe, Jane nee Puig i Soler`` and ``Jane Doe née Kowalska i Nowak`` move with it, as does the all-lower ``jane doe nee puig i soler``; the ``y`` spelling always read this way and is untouched. The link still has to be joining: ``Jane Doe nee Puig i`` keeps maiden ``Puig`` with suffix ``i``, and ``Jane Doe nee Puig i III`` suffix ``i III``. A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions`` and every prior reading comes back. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) + - **Add the Catalan and Polish surname link.** ``parse("Josep Carod i Rovira")`` gives family ``Carod i Rovira``, where every release from 1.4.0 through 2.3.0 gave middle ``Carod i`` with family ``Rovira``; ``Josep Lluis Carod i Rovira`` gives middle ``Lluis`` with that same family; and ``Carod i Rovira, Josep`` gives it too, where they read family ``Carod Rovira`` and took the link into ``suffix`` as a generation marker. ``i`` is connective vocabulary now, the way ``y`` already was, and a connective counts as a name word wherever the three-word carve-out counts them -- whatever else the vocabulary says the word is, which matters here because ``i`` is also the roman numeral. A connective that is also generational vocabulary joins only where a name word stands on each side of it, so ``John Quincy Smith i`` keeps suffix ``i``, ``Josep Lluis Carod i III`` keeps suffix ``i III``, and the two-word ``Carod i`` keeps its generation reading. Written wholly in one case the letter reads as an initial and says so: ``JOSEP CAROD I ROVIRA`` and ``josep carod i rovira`` keep the fields they had and gain a ``conjunction-or-initial`` report, which a one-case name gains wherever a bare ``i`` or ``I`` stands among the name's own words -- a letter inside a maiden clause is read by the clause's rules and stays silent, as ``e`` already was -- and in an all-lower name that reading can move a field, each such name now reading as its all-caps twin already did (``parse("john smith i jr")`` gives middle ``smith``, family ``i`` and suffix ``jr`` where it gave family ``smith`` and suffix ``i jr``). Case repair follows the reading: a lower-case ``i`` the parse read as the generation is still title-cased by ``capitalize(force=True)`` (``Carod i`` gives ``Carod I``, as every release did), while one standing among the name words keeps its lower case as ``y`` always has (``Carod i Rovira`` gives ``Carod i Rovira``, where ``Carod I Rovira`` was the pre-2.4 answer). A link inside a maiden clause stays in the birth name, which no release read that way: ``HumanName("Jane Doe nee Puig i Soler")`` gives maiden ``Puig i Soler`` with last ``Doe``, where 2.0 through 2.3 ended the birth name at the link and gave maiden ``Puig`` with middle ``Doe i``, last ``Soler`` -- and the same words would have joined into last ``Doe i Soler`` under the change above, carrying a word of the birth name into the current surname. ``Doe, Jane nee Puig i Soler`` and ``Jane Doe née Kowalska i Nowak`` move with it, as does the all-lower ``jane doe nee puig i soler``; the ``y`` spelling always read this way and is untouched. The link still has to be joining: ``Jane Doe nee Puig i`` keeps maiden ``Puig`` with suffix ``i``, and ``Jane Doe nee Puig i III`` suffix ``i III``. A caller with Catalan or Polish data removes the entry from ``conjunctions_ambiguous`` and gets the join in the one-case names too; a caller who wants none of this removes ``i`` from ``conjunctions``, which restores every prior FIELD and every prior report, with two readings it does not restore and cannot: a letter the two vocabularies disagree about being an initial reads as one here and as the generation there, and case repair leaves a connective the parse placed among the NAME words in lower case where the off switch title-cases it -- ``parse("Dr. John i Smith").capitalized(force=True)`` keeps ``i`` where the off switch gives ``Dr. John I Smith``, and ``Carod i Rovira`` and ``Josep i Rovira`` are the same shape. Those two are the whole of what the switch does not undo, and ``tests/v2/test_properties.py`` states them as its invariants' only exemptions. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #397) - **Fix a connective contributing no initial even where it is joining nothing.** ``parse("Juan de y").initials()`` gives ``J. y.``, where every release gave ``J.`` while ``family_base`` said ``y`` -- two views of one parse disagreeing about one token. A connective contributes nothing where it is JOINING, and initials like any other name word where its part holds nothing else for it to join. One rule for all three groups, so ``John and Jane Smith`` gives ``J. J. S.`` where 2.0 through 2.3 gave ``J. a. J. S.`` and 1.4.0 the run-together ``J a J. S.``, ``Duke of Edinburgh`` gives ``D. E.`` where 2.0 through 2.3 gave ``D. o. E.`` and 1.4.0 ``D o E.``, and ``John & Jane`` gives ``J. J.``. The question is asked of the whole part and never of a word count, so ``Jon Dough and`` has base ``Dough and`` and keeps ``J. D.``, and ``Juan Velasquez y Garcia`` keeps ``J. V. G.``. ``HumanName.initials()`` moves with the core -- over the differential corpora the two surfaces move on the same names and give the same values, reading one mark. Two names come back into 1.4.0 parity rather than away from it: ``JUAN Y GARCIA`` and ``محمد و علي`` both give the answer 1.4.0 gave. Parsing got cheaper by the same change -- the marks come off one pass instead of two, six fewer Python frames per name on 3.11. Two limits carried over from the 2.4 facade fix above: case repair still keeps such a connective lower-case, so ``initials()`` and ``capitalize()`` disagree about it on purpose, and a name restored from a pickle or a copy, or built from keyword fields, carries no tags and takes the older reading. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #461) diff --git a/nameparser/_pipeline/_assemble.py b/nameparser/_pipeline/_assemble.py index 6a11c7e0..e3aede20 100644 --- a/nameparser/_pipeline/_assemble.py +++ b/nameparser/_pipeline/_assemble.py @@ -19,7 +19,9 @@ from __future__ import annotations from nameparser._pipeline._state import ParseState -from nameparser._types import Ambiguity, ParsedName, Role, Token +from nameparser._types import ( + Ambiguity, AmbiguityKind, ParsedName, Role, Token, +) # rules.md#A2: "a name with no name content parses to the empty @@ -81,6 +83,35 @@ def assemble(state: ParseState) -> ParsedName: # referents did not lose a contest, they were discarded # wholesale, and the report still describes the input. continue + # rules.md#A1: "a report names the reading the parse took, so + # a report whose fork the rest of the parse then resolved to + # NEITHER branch is withdrawn rather than carried beside a + # reading it contradicts". classify's connective-or-initial + # fork offers exactly two readings and its detail says which + # it took ("it is read as an initial"); a generation and an + # honorific are neither. So where the parse goes on to role + # that letter SUFFIX or TITLE the report is false on its + # face, and 'JOHN QUINCY SMITH I' carried it beside a + # suffix-or-name saying the same token reads as a + # generational suffix (#397 second review). 'i' is the first + # word that is both a marked connective and suffix + # vocabulary, so no parse before this cycle could reach the + # shape. + # + # WITHDRAWN HERE and not emitted later, which is the narrow + # reading of mechanisms.md#AMBIGUITY-AT-THE-DECISION-SITE + # rather than an exception to it: "Emit at the site that + # takes the branch, not where an ambiguous tag sits" governs + # the EMISSION, classify still owns the fork, and what this + # drops is a report whose subject the parse went on to read + # as something else. Here is the one place that knows -- + # roles are final and nothing downstream moves them -- and + # the loop already withdraws a report whose referents did not + # survive, just above. + if (pending.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL + and any(t.role in (Role.SUFFIX, Role.TITLE) + for t in materialized)): + continue ambiguities.append( Ambiguity(pending.kind, pending.detail, materialized)) return ParsedName(original=state.original, diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 717938c2..4c0b8642 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -48,7 +48,7 @@ credential_at_the_given_slot, is_leading_title, is_suffix_piece, is_title_piece, leading_titles, peel_trailing, peel_walk, tail_reading, - trailing_start, + trailing_start, trailing_start_past_titles, ) from nameparser._pipeline._state import ( AMBIGUOUS_ACRONYM_TAG, ParseState, PendingAmbiguity, Structure, @@ -702,7 +702,15 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], # rules.md#P3: "a word the rest of the parse reads as a name word # rather than as a generation, a credential or an honorific, looked -# for past any run of connectives standing between" (#397) +# for past any run of connectives standing between. A connective with +# nothing to its right is connecting nothing, and a word of that +# vocabulary ending a name, or standing before the credential a name +# ends with, is the generation it also spells" (#397) -- the WHOLE +# clause, its second sentence included, because that sentence is what +# this predicate answering `False` means. Reading it as a generation +# is the CALLER's half: `_group_segment`'s `frozen` set is where a +# connective this refuses is placed as the generation, and +# `_link_joins_inside_the_clause` is the maiden walk's. # `Sequence[Sequence[int]]` rather than `Sequence[Piece]`, widened # when the maiden walk became a second caller: this reads a piece and # never edits one, and `_maiden_take` holds its pieces at the wider @@ -923,10 +931,23 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # of it" (#397, restated by its review). `frozen` holds the # TOKEN index of every such connective that has no name word # on one side or the other. It is joining nothing, so it is - # the generation it also spells: it may not merge into a run, - # it may not join, and it counts toward the carve-out total - # the way the generation counted -- which is not at all, a - # suffix piece being no rootname. + # the generation it also spells: no join of its own reaches + # it, it may not merge into a run, and it counts toward the + # carve-out total the way the generation counted -- which is + # not at all, a suffix piece being no rootname. + # + # "No join of its own" is the whole claim, and a NEIGHBOUR's + # join can still absorb it: the two loops below skip a frozen + # piece as the join's SUBJECT and nothing keeps it out of the + # span another connective's join takes. 'Josep Carod Rovira + # Puig y i' freezes the trailing 'i' -- nothing stands on its + # right -- and the 'y' beside it joins across it all the same, + # for family 'Puig y i', which is the parent's reading of the + # same name and 'y i' is what the parent read there too. + # Pinned by test_a_frozen_link_is_still_absorbed_by_a_ + # neighbours_join. Freezing it is not a claim that the word + # cannot move; it is a claim about which joins this loop + # licenses. # # Asked HERE, of the pieces as classify left them, and of the # NEIGHBOURS' class rather than of the connective's position. @@ -942,21 +963,61 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # the chain's trailing run is a length from the end: the # merges below move piece indices and cannot move this one. # - # Nothing but a one-letter connective of the suffix vocabulary - # reaches the body, so a name that has none pays tag lookups - # and no call at all. + # Nothing but a connective of the suffix vocabulary reaches + # the body, so a name that has none pays tag lookups and no + # call at all. + # + # NO LENGTH TEST, and that is the rule's own scope rather than + # an omission: the clause quoted above names a CLASS -- "a + # connective that is also generational vocabulary" -- and says + # nothing about how the word is spelled. A `len(tok.text) != 1` + # stood here and narrowed it to one-letter connectives, + # untested and undocumented, and it was caller-reachable: under + # `Lexicon.default().add(conjunctions={"og"}, suffix_words= + # {"og"})`, 'John Quincy Smith og' read family 'Smith og' with + # the test and reads family 'Smith' plus suffix 'og' without it + # -- which is the answer the rule states (#397 second review). + # Dropping it is byte-identical over every oracle in use -- + # fields, reports, `initials()`, `capitalized()` plain and + # forced, and every token's role -- across 359,053 parses + # measured 2026-09-20: the 351,400-parse review grid under + # eight lexicon/policy/locale configurations, the 2,175-parse + # sweep of tests/v2/cases.py under three orders, and the + # 5,478-parse sweep of the differential corpora under six. + # It has to be: `i` is the ONLY member of the class in the + # default vocabulary and in every locale pack, and it is one + # letter. Pinned by test_a_multi_letter_link_of_the_suffix_ + # vocabulary_joins_by_the_same_rule. + # + # The three-word carve-out below stays single-letter, because + # THAT is what its own sentence says ("a single-letter + # connective in a three-word name"). frozen: set[int] = set() lo = hi = -1 for k, piece in enumerate(pieces): tok = tokens[piece[0]] - if (len(piece) != 1 or len(tok.text) != 1 + if (len(piece) != 1 or "conjunction" not in tok.tags or "vocab:suffix" not in tok.tags): continue if hi < 0: lo = leading_titles(pieces, ptags, tokens) - hi = trailing_start(lo, pieces, ptags, tokens, - one_case=one_case) + # H5's reading and not the peel over the pieces as + # WRITTEN: a title standing behind the suffix run + # hides it from `trailing_start`, which then answers + # `len(pieces)` and hands this loop a credential as + # the name word on the link's right. 'John Quincy + # Adams i MA Prof.' joined to family 'Adams i MA' + # with no report at all, where 'John Quincy Adams i + # MA' -- the same name, one title shorter -- reads + # family 'Adams', suffix 'i MA' and reports the + # acronym (#397 second review). Asked once per + # segment and only where such a connective was found, + # so the cost is the chain's own and no ordinary name + # pays it. + hi = trailing_start_past_titles(lo, pieces, ptags, + tokens, + one_case=one_case) if not (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens) and _name_word_beside(k, 1, lo, hi, pieces, ptags, tokens)): diff --git a/nameparser/_pipeline/_pieces.py b/nameparser/_pipeline/_pieces.py index 2a648317..1c9c2403 100644 --- a/nameparser/_pipeline/_pieces.py +++ b/nameparser/_pipeline/_pieces.py @@ -704,3 +704,41 @@ def tail_reading(rest: list[int], pieces: Sequence[Sequence[int]], # run goes in FRONT of what the pass before it took titled[:0] = rest[kept:peeled.names] rest = rest[:kept] + rest[peeled.names:] + + +# rules.md#H5: "the title is TRANSPARENT to the suffix reading: where +# two or more name words stand, what stands once the chain is taken +# reads exactly as it would read written without the title, plus the +# title" +def trailing_start_past_titles(start: int, + pieces: Sequence[Sequence[int]], + ptags: Sequence[Set[str]], + tokens: Sequence[WorkToken], + *, one_case: bool | None) -> int: + """`trailing_start` read through H5's chain: where assign's + trailing suffix run begins once a trailing TITLE has stopped + hiding it. + + `trailing_start` reads the pieces as WRITTEN, so a title standing + behind the suffix run makes the peel take nothing and the answer + is `len(pieces)` -- the reading assign itself has not had since + H5, because assign runs the peel and the chain to their fixed + point instead (`tail_reading`). A caller using that answer as the + right bound of the NAME is told a credential is a name word: + 'John Quincy Adams i MA Prof.' read family 'Adams i MA' where + 'John Quincy Adams i MA' reads family 'Adams' and suffix 'i MA' + (#397 second review). Every caller that bounds the name wants + this one; `trailing_start` stays for the callers that count a + trailing run of the pieces as they stand. + + The returned index bounds the name from the right, and the + trailing titles the chain spliced out are not under it: they end + the segment, so they stand at or past the first suffix piece + whenever there is one. Where the peel takes nothing even past the + chain this returns `len(pieces)` as `trailing_start` does, and a + trailing title is then inside the bound and refused by the title + test the callers already run beside it. + """ + rest, _titled, peeled = tail_reading(peel_walk(start, ptags), + pieces, ptags, tokens, one_case) + return rest[peeled.names] if peeled.names < len(rest) else len(pieces) diff --git a/nameparser/_render.py b/nameparser/_render.py index 0b95522b..238948f5 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -225,21 +225,32 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # unjoined mark has turned into ordinary name words" -- so a # conjunction keeps conjunction treatment even inside a part the # mark has turned into ordinary name words. - # The ROLE test is the other half of that sentence, and it is why - # a single comparison stands ahead of both conjunction arms: a - # word this vocabulary holds can ALSO be the generation it spells - # ('i' is the Catalan link and the roman numeral), and where the - # parse read the generation the token still carries the + # The GENERATION test is the other half of that sentence, and it + # is why a single conjunct stands ahead of both conjunction arms: + # a word this vocabulary holds can ALSO be the generation it + # spells ('i' is the Catalan link and the roman numeral), and + # where the parse read the generation the token still carries the # `conjunction` tag classify gave it -- so without the test, # `parse("John Quincy Smith i").capitalized(force=True)` gave # 'John Quincy Smith i' where every release through 2.3 gave - # 'John Quincy Smith I' (#397 review). A suffix-roled token is - # repaired as the suffix it was read as, which is the rest of - # R4's sentence: "one the parse read as the generation it also - # spells is not a connective of this name at all". The test is - # the ROLE and not the suffix VOCABULARY, because the role is - # what the parse decided and the vocabulary is only what the word - # could have been (mechanisms.md#RENDER-HONORS-THE-PARSE). + # 'John Quincy Smith I' (#397 review). Such a token is repaired + # as the suffix it was read as, which is the rest of R4's + # sentence: "one the parse read as the generation it also spells + # is not a connective of this name at all". + # BOTH HALVES, and the role alone is not enough -- the role says + # where the word landed and the vocabulary says whether landing + # there made it a generation. A plain connective can land in the + # suffix field without being generational vocabulary at all (a + # third comma part: `Smith, John, and`), and on the role test + # alone every one of them was repaired as a name word -- + # 'John Smith And' where 1.4.0, 2.0 through 2.3 and the parent + # commit all gave 'John Smith and', and 'John Smith De, Y' for a + # field spliced to suffix='de y' where R4's own Accepted + # paragraph says the vocabulary answers and the 'y' keeps its + # lowercase (#397 second review). `vocab:suffix` is classify's + # record of the vocabulary half, so the pair reads two decisions + # the parse already made and re-derives neither + # (mechanisms.md#RENDER-HONORS-THE-PARSE). # No SHIPPED name witnesses the difference: `particles` and # `conjunctions` are disjoint in the default vocabulary and in # every locale pack, so no shipped conjunction can sit in an @@ -296,7 +307,7 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # the same words parsed gave 'E-F' (#463 review). if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY) and UNJOINED_TAG not in tags) - or (role is not Role.SUFFIX + or (not (role is Role.SUFFIX and "vocab:suffix" in tags) and ("conjunction" in tags or (UNCLASSIFIED_TAG in tags and _reads_as_conjunction(word, lex))))): diff --git a/tests/v2/cases.py b/tests/v2/cases.py index eded423d..36b5cd13 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -2812,6 +2812,84 @@ def _check_cjk_shape_purity(self) -> None: "with a comma R1 no longer derives, and read last " "'Puig' for want of maiden support", shape=1), + # ---- #397 second review: the one-case role movers -------------- + # decisions.md#P3's 2026-09-20 bullet describes these four names + # in prose and pinned none of them. They are where the marked + # subset's riskiest movement is -- an ALL-LOWER name whose letter + # the parent read as the GENERATION and this reads as an initial + # -- and each one now reads as its ALL-CAPS twin already did, + # which is the claim the prose makes and + # test_properties.py::test_a_one_case_name_reads_the_same_in_ + # either_case holds over the grid. Every reading below measured + # 2026-09-20 on the released 1.4.0 and 2.3.0 wheels from a + # throwaway environment. + Case("a_one_case_link_after_a_family_comma_is_the_given_name", + "rovira, i", + {"given": "i", "family": "rovira"}, + classification="fix(#397)", + ambiguities=("conjunction-or-initial",), + notes="written wholly in lower case, so the letter reads as " + "an INITIAL and takes the given slot the comma leaves " + "for it. 2.3.0 read family 'rovira' with suffix 'i'; " + "1.4.0 read first 'i', which is this reading. The " + "ALL-CAPS twin 'ROVIRA, I' gave first 'I' at BOTH of " + "them and gives given 'I' here -- the two spellings " + "disagreed at 2.3.0 and agree now. NO SHAPE TAG on " + "any of these four: decisions.md#P3's one-case " + "paragraph says of them that none is a corpus name, " + "and admitting them to the differential's contract " + "corpus is a ledger change of its own rather than " + "this row's business. The cross-version readings " + "above are what the corpus would have bought, " + "measured on the wheels instead"), + Case("a_one_case_link_before_a_generation_is_a_name_word", + "john smith i jr", + {"given": "john", "middle": "smith", "family": "i", + "suffix": "jr"}, + classification="fix(#397)", + ambiguities=("conjunction-or-initial",), + notes="the same movement with a generation behind it: the " + "letter reads as an initial, so the peel stops before " + "it and 'jr' alone is the suffix. 2.3.0 read family " + "'smith' with suffix 'i jr' and 1.4.0 suffix 'i, jr'. " + "The ALL-CAPS twin 'JOHN SMITH I JR' read middle " + "'SMITH', last 'I', suffix 'JR' at both of them -- " + "again the reading this row now gives. No shape " + "tag, for the reason the row above gives"), + Case("a_one_case_link_in_a_comma_suffix_run_is_a_middle_initial", + "maier, amy i, jr.", + {"given": "amy", "middle": "i", "family": "maier", + "suffix": "jr."}, + classification="fix(#397)", + ambiguities=("conjunction-or-initial",), + notes="the third comma shape, and the one whose MIXED-CASE " + "twin is already in this table: 'Maier, Amy I, Jr.' " + "reads middle 'I' and does not move, the writing " + "having decided the letter there. 2.3.0 and 1.4.0 " + "both read suffix 'i, jr.' for the lower-case " + "spelling while 'MAIER, AMY I, JR.' already read " + "middle 'I' -- the disagreement this closes. No " + "shape tag, for the reason the first of these rows " + "gives"), + Case("a_link_inside_a_maiden_clause_keeps_a_bare_credential", + "Jane Doe nee Puig i Ma", + {"given": "Jane", "family": "Doe", "maiden": "Puig i Ma"}, + classification="fix(#397)", + ambiguities=("suffix-or-name",), + notes="the clause's link with a BARE ambiguous acronym " + "behind it, on the shipped vocabulary rather than a " + "built one. 'Ma' is a name word here -- mixed case, " + "so the class member leans to the name -- and a name " + "word on the link's right is what the exception " + "wants, so the clause takes all three words and " + "reports the acronym it kept. 2.3.0 read middle 'Doe " + "i', family 'Ma', maiden 'Puig', handing two words of " + "the birth name to the current one; 1.4.0 read middle " + "'Doe nee', last 'Puig', suffix 'i, Ma'. The " + "ALL-CAPS 'MA' is the other lean and is the control " + "in test_properties.py rather than a row here. No " + "shape tag, for the reason the first of these rows " + "gives"), # ---- #461: a connective initials where it joins nothing -------- # The rule, one sentence for all three groups: a connective # contributes nothing where it is JOINING -- a part holding diff --git a/tests/v2/pipeline/test_assemble.py b/tests/v2/pipeline/test_assemble.py index 9d67fa46..749ef85d 100644 --- a/tests/v2/pipeline/test_assemble.py +++ b/tests/v2/pipeline/test_assemble.py @@ -131,3 +131,69 @@ def test_ambiguity_with_all_indices_dropped_is_omitted() -> None: kinds = [a.kind for a in pn.ambiguities] assert AK.ORDER not in kinds # fully dangled: omitted assert AK.UNBALANCED_DELIMITER in kinds # born empty: kept + + +def test_a_connective_or_initial_report_is_withdrawn_where_the_role_is_a_suffix( +) -> None: + """rules.md#A1, the withdrawal (#397 second review). + + classify's connective-or-initial fork offers a connective and an + initial and its detail says which it took. A generation is + neither, so where the parse goes on to role the letter SUFFIX the + report describes a branch nobody took -- and 'JOHN QUINCY SMITH + I' carried it beside a `suffix-or-name` saying the same token + reads as a generational suffix. 'i' is the first word that is + both a marked connective and suffix vocabulary, so no parse + before this cycle could reach the shape. + + The shipped vocabulary is what these names need, so `parse` is + used rather than the trimmed `_LEX` above. + """ + from nameparser import parse + withdrawn = { + "JOHN QUINCY SMITH I": ["suffix-or-name"], + "john smith i": ["suffix-or-name"], + "SMITH, JOHN I": [], + "HENRY I": ["suffix-or-name", "given-or-family"], + } + for text, kinds in withdrawn.items(): + name = parse(text) + assert name.suffix.lower() == "i", text + assert sorted(a.kind.value for a in name.ambiguities) \ + == sorted(kinds), text + # the contrast that keeps the withdrawal honest: where the parse + # roles the same letter a NAME word the fork DID resolve to one + # of its branches, and the report stands + for text in ("JOSEP CAROD I ROVIRA", "josep carod i rovira", + "JOHN I SMITH", "JOSEP CAROD I ROVIRA III"): + name = parse(text) + assert "conjunction-or-initial" in [ + a.kind.value for a in name.ambiguities], text + # ... and an unmarked-position letter of the same class reports + # nothing at all, which is the third state + assert [a.kind.value for a in parse("Josep Carod i Rovira").ambiguities] \ + == [] + + +def test_the_withdrawal_reads_the_role_and_not_the_word() -> None: + """The same withdrawal asked of a TITLE, the other role a fork + between a connective and an initial cannot have resolved to. + + Nothing shipped builds this state -- a marked letter roled TITLE + -- which is exactly why it is built here: an implementation + keyed on the suffix role alone would pass every row above and + fail this one (AGENTS.md: "Pin the decision, not the vocabulary"). + """ + import dataclasses + from nameparser._pipeline._state import PendingAmbiguity + from nameparser._types import AmbiguityKind as AK + state = run(ParseState(original="Dr. Jane", lexicon=_LEX, + policy=Policy())) + dr = next(i for i, t in enumerate(state.tokens) if t.text == "Dr.") + assert state.tokens[dr].role is Role.TITLE + poisoned = dataclasses.replace( + state, ambiguities=state.ambiguities + ( + PendingAmbiguity(AK.CONJUNCTION_OR_INITIAL, + "read as an initial", (dr,)),)) + assert AK.CONJUNCTION_OR_INITIAL not in [ + a.kind for a in assemble(poisoned).ambiguities] diff --git a/tests/v2/pipeline/test_group.py b/tests/v2/pipeline/test_group.py index a88254e6..bc3d9264 100644 --- a/tests/v2/pipeline/test_group.py +++ b/tests/v2/pipeline/test_group.py @@ -1775,3 +1775,112 @@ def test_a_marker_with_nothing_after_it_declines_before_the_bound( out = _grouped("Jane Doe née", lexicon=_LINK_LEX) assert _maiden_texts(out) == [] assert _piece_texts(out) == [["Jane", "Doe", "née"]] + + +# --- #397 second review: the bound, the class and the frozen piece -- + +def test_a_trailing_title_does_not_hide_the_suffix_run_from_the_join( +) -> None: + # rules.md#H5 read where the JOIN asks its question (#397 second + # review). `trailing_start` reads the peel over the pieces as + # WRITTEN, so a title standing behind the suffix run makes the + # peel take nothing and the answer is `len(pieces)` -- and a + # caller using it as the right bound of the NAME is then told a + # credential is a name word. 'MA' carries no `vocab:suffix` tag, + # so the piece test cannot refuse it either and the join swallowed + # it: family 'Adams i MA', no report, and `initials()` gaining an + # 'M.'. + # + # The pair is the finding, and it is H5's own sentence: the name + # one title shorter has always read the other way. + # the SHIPPED vocabulary reaches this row -- 'i' is a default + # connective and a default suffix word, 'MA' a default ambiguous + # acronym and 'Prof.' a default title -- so this is one of the + # few #397 rows that needs no built lexicon at all. + shipped = Lexicon.default() + titled = _grouped("John Quincy Adams i MA Prof.", lexicon=shipped) + bare = _grouped("John Quincy Adams i MA", lexicon=shipped) + assert _piece_texts(titled) == [ + ["John", "Quincy", "Adams", "i", "MA", "Prof."]] + assert _piece_texts(bare) == [["John", "Quincy", "Adams", "i", "MA"]] + # and end to end, where the fields say what the bound bought. + # The SHIPPED vocabulary reaches this: 'i' is a default + # connective and a default suffix word, and 'MA' a default + # ambiguous acronym, so no built lexicon is needed here and the + # row is a real parse rather than a configured one. + parser = Parser() + name = parser.parse("John Quincy Adams i MA Prof.") + assert name.as_dict() == { + "title": "Prof.", "given": "John", "middle": "Quincy", + "family": "Adams", "suffix": "i MA", "nickname": "", "maiden": ""} + assert name.initials() == "J. Q. A." + assert [a.kind.value for a in name.ambiguities] == ["suffix-or-name"] + # a LOWER-CASE title is the same shape and reaches the same + # bound: the vocabulary lookup is folded, so 'prof.' peels like + # 'Prof.' + lower_title = parser.parse("John Quincy Adams i MA prof.") + assert lower_title.title == "prof." + assert lower_title.suffix == "i MA" + # the ONE-CASE spellings never reached the defect and are pinned + # as the control: written wholly in one case the letter reads as + # an initial (rules.md#P3's marked subset), no join is attempted + # at all, and both spellings already agreed with each other and + # with the untitled name at dc3bdf9c -- which is what says the + # bound and not the marking is what this row is about. + for text, title in (("john quincy adams i ma prof.", "prof."), + ("JOHN QUINCY ADAMS I MA PROF.", "PROF.")): + one_case = parser.parse(text) + assert one_case.title == title + assert one_case.suffix == text.split()[-2] + + +def test_a_multi_letter_link_of_the_suffix_vocabulary_joins_by_the_same_rule( +) -> None: + # rules.md#P3's both-sides clause names a CLASS -- "a connective + # that is also generational vocabulary" -- and says nothing about + # how the word is spelled (#397 second review). A `len(text) != 1` + # filter stood in the stage and narrowed the clause to one-letter + # connectives, untested and undocumented; this is the row that was + # caller-reachable past it. Nothing SHIPPED reaches it -- 'i' is + # the only member of the class in the default vocabulary and in + # every locale pack -- which is why the lexicon is built here. + lex = Lexicon.default().add(conjunctions={"og"}, + suffix_words={"og"}) + parser = Parser(lexicon=lex) + # nothing on its right: it is the generation it also spells + lone = parser.parse("John Quincy Smith og") + assert lone.family == "Smith" + assert lone.suffix == "og" + # a name word on each side: it joins, exactly as a one-letter + # member does + joined = parser.parse("Josep Carod og Rovira") + assert joined.family == "Carod og Rovira" + # the three-word carve-out stays SINGLE-LETTER, which is what its + # own sentence says ("a single-letter connective in a three-word + # name"): 'og' is two letters, so it joins in a THREE-word name + # where the one-letter 'i' of the shipped vocabulary stays a name + # word in the middle. + assert parser.parse("Josep og Carod").given == "Josep og Carod" + assert Parser().parse("Josep i Carod").middle == "i" + # and with nothing on its right the class test decides before the + # carve-out is ever asked, for either spelling + assert parser.parse("Josep Carod og").suffix == "og" + assert Parser().parse("Josep Carod i").suffix == "i" + + +def test_a_frozen_link_is_still_absorbed_by_a_neighbours_join() -> None: + # What freezing a piece claims and what it does not (#397 second + # review). `frozen` keeps a connective from being the SUBJECT of a + # join; it does not keep the word out of the span another + # connective's join takes. The trailing 'i' here has nothing on + # its right and is frozen, and the 'y' beside it joins across it + # all the same. + # + # Not a defect and not a silence: this is the reading the parent + # commit 46651750 gives the same name, the letter being no + # connective there at all, so the row is a CONTROL for the + # comment beside `frozen` rather than a behavior claim of its own. + out = Parser().parse("Josep Carod Rovira Puig y i") + assert out.family == "Puig y i" + assert out.middle == "Carod Rovira" + assert out.suffix == "" diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 8b027f46..e8b38f21 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -1048,3 +1048,48 @@ def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> Non assert str(restored) == restored_cap built.capitalize() assert str(built) == restored_cap + + +def test_the_v1_off_switch_restores_the_pre_397_fields() -> None: + """decisions.md#P3's off switch on the V1 SURFACE (#397 second + review). The bullet says deleting `i` from `C.conjunctions` + restores the parent's readings; the core-side switch is pinned by + the off-switch grid in tests/v2/test_properties.py and this is the + facade half, which has no `Lexicon` of its own to remove from. + + The three role movers of decisions.md#P3's one-case paragraph are + the rows, because they are the ones whose FIELDS move: every + value on the right below is what the released 2.3.0 wheel gives, + measured 2026-09-20. + + A local `Constants`, never the shared `CONSTANTS`: a removal on + the singleton would leak into every later test in the process. + """ + constants = Constants() + constants.conjunctions.remove("i") + for text, on_fields, off_fields in ( + ("rovira, i", + {"first": "i", "last": "rovira", "suffix": ""}, + {"first": "", "last": "rovira", "suffix": "i"}), + ("john smith i jr", + {"first": "john", "middle": "smith", "last": "i", + "suffix": "jr"}, + {"first": "john", "middle": "", "last": "smith", + "suffix": "i jr"}), + ("maier, amy i, jr.", + {"first": "amy", "middle": "i", "last": "maier", + "suffix": "jr."}, + {"first": "amy", "middle": "", "last": "maier", + "suffix": "i, jr."})): + on = HumanName(text) + off = HumanName(text, constants) + for field, value in on_fields.items(): + assert getattr(on, field) == value, (text, field) + for field, value in off_fields.items(): + assert getattr(off, field) == value, (text, field) + # and the join itself, the reading the bullet leads with + joined = HumanName("Josep Carod i Rovira") + assert joined.last == "Carod i Rovira" + unjoined = HumanName("Josep Carod i Rovira", constants) + assert unjoined.middle == "Carod i" + assert unjoined.last == "Rovira" diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index a8257621..44dc074d 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -3336,7 +3336,7 @@ def _claim(rule: dict) -> _Claim: # comma form of the swallowed generation. Reach again, and # verified name by name. "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(361, ('given', 'suffix', 'title'), '2e17dee05bcf', None), + _Claim(363, ('given', 'suffix', 'title'), 'b628b4d25dfb', None), "fix(comma-family) a comma followed only by titles keeps the given/family split": _Claim(2, ('family', 'given'), "5bd9c6d96c38", None), "fix(comma-family) a comma followed only by titles keeps the given/family split, the C1 example": @@ -3391,7 +3391,7 @@ def _claim(rule: dict) -> _Claim: # 2026-09-20, #397 review: 359 -> 360, the same one new comma # name as the rule above and for the same reason. "fix(comma-precomma-family) pre-comma run reads as family, not given": - _Claim(361, ('family', 'given'), '2e17dee05bcf', None), + _Claim(363, ('family', 'given'), 'b628b4d25dfb', None), # 2026-09-20, #397: retitled in place, reach and digest # unchanged -- the rule keeps 'Carod i', which the landing # leaves byte-identical. @@ -3636,7 +3636,7 @@ def _claim(rule: dict) -> _Claim: # unrelated 'y', which is in the corpus because its row # carries a shape tag. Reach again, verified name by name. "fix(initials-per-word) a connective run initials each word (facade, since 2.0.0)": - _Claim(103, ('_initials',), "05db047609d0", ('DEFAULT',)), + _Claim(104, ('_initials',), "bc7dfeba1da5", ('DEFAULT',)), # 2026-09-19, #533: 41 -> 43. Two new corpus names opening # with a bound-given word, 'Berg, abdul MA' and 'Berg, abdul # nee Jones MA' -- the P5 pair this change added to record diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index d1c96e11..4225f94b 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -1848,3 +1848,35 @@ def test_a_phrase_marker_outranks_the_word_it_starts_with() -> None: assert configured.parse("Jane Smith geb von Braun").maiden == "Braun" assert configured.parse("Jane Smith geb Braun").maiden == "Braun" assert parse("Jane Smith geb von Braun").maiden == "von Braun" + + +def test_the_catalan_recipe_unmarks_the_link_for_one_case_names() -> None: + """rules.md#P3's per-caller answer for Catalan and Polish data + (#397 second review), pinned rather than described. + + `i` ships in the MARKED subset, so a name written wholly in one + case reads it as an initial and reports the fork. A caller whose + data is Catalan knows better, and the knob the rule names is + `remove(conjunctions_ambiguous={"i"})` -- which leaves the letter + a connective and takes the one-case fork out of its way. The + prose said so; nothing ran it. + """ + catalan = Parser( + lexicon=Lexicon.default().remove(conjunctions_ambiguous={"i"})) + for text in ("JOSEP CAROD I ROVIRA", "josep carod i rovira"): + name = catalan.parse(text) + assert name.family == text.split(" ", 1)[1] + assert name.ambiguities == () + # the three things the recipe does NOT change, and they are what + # make it safe to hand a Catalan user. The link with nothing on + # its right is still the generation it also spells ... + kept = catalan.parse("JOHN QUINCY SMITH I") + assert kept.suffix == "I" + assert kept.family == "SMITH" + # ... P3's three-word carve-out still leaves a lone letter a name + # word ... + assert catalan.parse("JOHN I SMITH").middle == "I" + # ... and no report is emitted anywhere, the fork the recipe + # removes being the only one these names raised + assert catalan.parse("JOHN I SMITH").ambiguities == () + assert catalan.parse("JOHN QUINCY SMITH I").ambiguities == () diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 0a24cfec..4d69445a 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -25,7 +25,8 @@ from nameparser.config import Constants from nameparser._lexicon import _VOCAB_FIELDS from nameparser._pipeline import run -from nameparser._pipeline._state import ParseState +from nameparser._pipeline._state import (AMBIGUOUS_ACRONYM_TAG, + ParseState) from nameparser._pipeline._vocab import effective_script from nameparser._types import (UNJOINED_CONJUNCTION_TAG, UNJOINED_TAG, AmbiguityKind, ParsedName, Role, Token) @@ -1082,17 +1083,97 @@ def _rows(texts: list[str], return out +def _signature(parser: Parser, text: str) -> tuple[object, ...]: + """Everything the four grids' invariants can read of one parse. + + Used only by the trim's own guard below, which is why it is + exhaustive rather than cheap: it has to be able to see a + difference no invariant in this file happens to ask about. + """ + name = parser.parse(text) + return ( + tuple(sorted(name.as_dict().items())), + tuple((t.text, t.role.value, tuple(sorted(t.tags)), t.span) + for t in name.tokens), + name.initials(), + tuple(name.initials(f"{{{r.value}}}") for r in _NAME_ROLES), + name.family_base, name.family_particles, + str(parser.capitalized(name)), + str(parser.capitalized(name, force=True)), + tuple(sorted((a.kind.value, a.detail, + tuple(t.text for t in a.tokens)) + for a in name.ambiguities)), + ) + + +#: Every (text, lexicon, policy) the trim drops is checked against +#: the row it collapses onto, one in `_TRIM_STRIDE` of them. +_TRIM_STRIDE = 61 + + +def _dropped_rows(texts: list[str], lexicons: _Variants) -> list[ + tuple[str, tuple[str, Lexicon], tuple[str, Policy], + tuple[str, Lexicon], tuple[str, Policy]]]: + """The pairings `_rows` declines, each with the pairing it + collapses onto: the same text under the variant it could not read + differently, replaced by the baseline of whichever dimension + declined. + + Deterministically sampled by stride rather than at random -- a + random slice makes a failure unreproducible, and the shapes here + are generated in a fixed order. + """ + base_lex = lexicons[0] + base_pol = _GRID_POLICIES[0] + out = [] + for i, text in enumerate(texts): + if i % _TRIM_STRIDE: + continue + reach = _reaching(text) + for ln, lex, lex_need in lexicons: + for pn, pol, pol_need in _GRID_POLICIES: + if (lex_need | pol_need) <= reach: + continue + twin_lex = (ln, lex) if lex_need <= reach else base_lex[:2] + twin_pol = (pn, pol) if pol_need <= reach else base_pol[:2] + out.append((text, (ln, lex), (pn, pol), + twin_lex, twin_pol)) + return out + + @functools.cache def _class_letters(lexicon: Lexicon) -> frozenset[str]: """The class rules.md#P3's both-sides condition is about: a - one-letter connective that is ALSO generational vocabulary. + connective that is ALSO generational vocabulary. + + NO LENGTH TEST, matching the rule's own scope and the stage's + (#397 second review, where a `len(w) == 1` came out of both). + Measured no-op over every lexicon these grids build: `i` is the + only member in the default vocabulary and in every locale pack, + and the two variant lexicons add a one-letter connective and a + particle. Cached on the lexicon -- a frozen, hashable value -- because both grids ask this of every row and there are five lexicons between them. """ return frozenset(w for w in lexicon.conjunctions - if len(w) == 1 and w in lexicon.suffix_words) + if w in lexicon.suffix_words) + + +#: The two grids' lexicon variants, module-level so +#: `test_every_declared_variant_earns_a_row` can name them. See +#: `_rows` for what the third element of each declares. +_CONNECTIVE_LEXICONS: _Variants = ( + ("default", Lexicon.default(), frozenset()), + ("conj+v", Lexicon.default().add(conjunctions={"v"}), + frozenset({"v"})), + ("part+y", Lexicon.default().add(particles={"y"}), + frozenset({"y"}))) +_OFF_SWITCH_LEXICONS: _Variants = ( + ("default", Lexicon.default(), frozenset()), + ("conj+v", Lexicon.default().add(conjunctions={"v"}), + frozenset({"v"}))) def _connective_grid() -> list[tuple[str, Parser, str]]: @@ -1105,12 +1186,6 @@ def _connective_grid() -> list[tuple[str, Parser, str]]: [], ["Rovira"], ["de", "Rovira"], ["Rovira", "Puig"]) suffixes: tuple[list[str], ...] = ( [], ["III"], ["Jr."], ["I"], ["MA"]) - lexicons: _Variants = ( - ("default", Lexicon.default(), frozenset()), - ("conj+v", Lexicon.default().add(conjunctions={"v"}), - frozenset({"v"})), - ("part+y", Lexicon.default().add(particles={"y"}), - frozenset({"y"}))) texts: list[str] = [] seen: set[str] = set() for head, mid, conn, tail, suffix, comma in itertools.product( @@ -1126,7 +1201,7 @@ def _connective_grid() -> list[tuple[str, Parser, str]]: if written not in seen: seen.add(written) texts.append(written) - return _rows(texts, lexicons) + return _rows(texts, _CONNECTIVE_LEXICONS) _CONNECTIVE_GRID = _connective_grid() @@ -1160,9 +1235,60 @@ def _predicted_initials(part: tuple[Token, ...], role: Role) -> list[str]: return out +#: The two kinds whose details assert DIFFERENT readings of the same +#: word: one says the letter is read as an initial, the other that it +#: is read as a generational suffix. A token cannot be both, so a +#: token named by both carries a report that contradicts the reading +#: beside it (#397 second review). +_CONTRADICTORY_KINDS = (AmbiguityKind.CONJUNCTION_OR_INITIAL, + AmbiguityKind.SUFFIX_OR_NAME) + + +def _record_contradicting_reports(name: ParsedName, label: str, text: str, + out: list[str]) -> None: + """INV10 (#397 second review). Two statements of one rule, the + second wider than the first and both about the same defect. + + rules.md#A1's own sentence, that a report names the reading the + parse took. The + connective-or-initial fork offers a connective and an initial and + says which it took, so a token the parse roles SUFFIX or TITLE + resolved that fork to NEITHER branch and the report is false on + its face. And no token may be named by two reports whose details + assert different readings of it, which is the same defect stated + without naming a role: 'JOHN QUINCY SMITH I' carried + connective-or-initial beside suffix-or-name, one saying the + letter reads as an initial and the other that it reads as the + generation. + + 'i' is the first word that is both a marked connective and suffix + vocabulary, so this could not arise before this cycle -- which is + why the invariant is worth writing down now rather than having + been written down before. + """ + by_span: dict[tuple[int, int] | None, set[AmbiguityKind]] = {} + for amb in name.ambiguities: + for tok in amb.tokens: + span = None if tok.span is None else (tok.span.start, + tok.span.end) + by_span.setdefault(span, set()).add(amb.kind) + if (amb.kind is AmbiguityKind.CONJUNCTION_OR_INITIAL + and tok.role in (Role.SUFFIX, Role.TITLE)): + out.append( + f"[{label}] {text!r}: {tok.text!r} is roled " + f"{tok.role.value} and reports " + f"{amb.kind.value}") + for span, kinds in by_span.items(): + if set(_CONTRADICTORY_KINDS) <= kinds: + out.append( + f"[{label}] {text!r}: the token at {span} carries both " + f"{_CONTRADICTORY_KINDS[0].value} and " + f"{_CONTRADICTORY_KINDS[1].value}") + + @functools.cache def _connective_findings() -> dict[str, list[str]]: - """One walk of the connective grid; four invariants' answers. + """One walk of the connective grid; FIVE invariants' answers. Each key below is one test's failure list, built with the parse in hand and in grid order, so a test reads exactly what it would @@ -1172,10 +1298,12 @@ def _connective_findings() -> dict[str, list[str]]: is stated in, and this function only asks them all at once. """ out: dict[str, list[str]] = {k: [] for k in - ("INV1", "INV2", "INV3/4", "INV5")} + ("INV1", "INV2", "INV3/4", "INV5", + "INV10")} for text, parser, label in _CONNECTIVE_GRID: letters = _class_letters(parser.lexicon) name = parser.parse(text) + _record_contradicting_reports(name, label, text, out["INV10"]) for role in _NAME_ROLES: part = name.tokens_for(role) # hoisted out of the INV3/INV4 comprehensions below, where @@ -1233,6 +1361,92 @@ def _connective_findings() -> dict[str, list[str]]: return out +@pytest.mark.parametrize("grid_name", ("connective", "off-switch")) +def test_the_reach_trim_drops_only_rows_a_kept_row_repeats( + grid_name: str) -> None: + """The TRIM ITSELF, as a mechanism rather than as a measurement. + + `_rows` pairs a text only with the configurations that can read + it differently, and each variant declares its reach as a set of + WORDS that `_reaching` folds a text into. Sound for the variants + that stand here today -- verified row by row when the trim + landed -- and silently wrong for a variant nobody has written + yet: `_reaching` splits on whitespace and strips edge periods, so + a variant declaring a MULTI-WORD entry ("van der") or a + hyphenated one would be matched by no text at all and would + contribute ZERO rows, leaving a green suite testing one + configuration fewer than it names. + + Two checks, and the second is the one that catches that: every + dropped pairing parses identically to the pairing it collapses + onto, and every declared variant earns at least one row. A + variant that reaches nothing passes the first vacuously. + + Sampled by stride (`_TRIM_STRIDE`), which keeps this at about a + second while covering every shape class: the generators emit + shapes in a fixed order, so a stride walks all of them. + """ + grid, lexicons = ((_CONNECTIVE_GRID, _CONNECTIVE_LEXICONS) + if grid_name == "connective" + else (_OFF_SWITCH_GRID, _OFF_SWITCH_LEXICONS)) + texts: list[str] = [] + seen: set[str] = set() + for text, _parser, _label in grid: + if text not in seen: + seen.add(text) + texts.append(text) + cache: dict[tuple[str, str, str], tuple[object, ...]] = {} + + def sig(text: str, lex: tuple[str, Lexicon], + pol: tuple[str, Policy]) -> tuple[object, ...]: + key = (text, lex[0], pol[0]) + if key not in cache: + cache[key] = _signature( + Parser(lexicon=lex[1], policy=pol[1]), text) + return cache[key] + + dropped = _dropped_rows(texts, lexicons) + assert dropped, "the trim dropped nothing; it has stopped trimming" + problems = [ + f"{text!r} [{lex[0]}/{pol[0]}] differs from its kept twin " + f"[{twin_lex[0]}/{twin_pol[0]}]" + for text, lex, pol, twin_lex, twin_pol in dropped + if sig(text, lex, pol) != sig(text, twin_lex, twin_pol)] + assert not problems, ( + f"{len(problems)} of {len(dropped)} dropped row(s) are not " + f"repeats:\n" + "\n".join(problems[:10])) + + +def test_every_declared_variant_earns_a_row_somewhere() -> None: + """The other half of the trim's guard: a variant whose declared + reach no text can satisfy contributes ZERO rows and the suite + goes on testing one configuration fewer than it names, in + silence. `_reaching` splits a text on whitespace and strips edge + periods, so a multi-word or hyphenated entry is the shape that + would do it. + + Asked of the two grids TOGETHER, because a variant earning rows + in one of them alone is the state today and is recorded rather + than a defect: no text of the connective grid holds a standalone + 'v', so `conj+v` earns its rows in the off-switch grid, whose + suffixes include 'V'. `_rows` says so and calls it + self-maintaining -- give a connective generator a 'v' word and + the rows come back. What no generator can repair is a reach + nothing folds to, and that is what this refuses. + """ + labels = ({label for _t, _p, label in _CONNECTIVE_GRID} + | {label for _t, _p, label in _OFF_SWITCH_GRID} + | {label for _t, _p, label in _MAIDEN_LINK_GRID}) + declared = {f"{ln}/{pn}" + for lexicons in (_CONNECTIVE_LEXICONS, + _OFF_SWITCH_LEXICONS) + for ln, _lex, _ln_need in lexicons + for pn, _pol, _pn_need in _GRID_POLICIES} + missing = sorted(declared - labels) + assert not missing, ( + f"declared variant(s) earning no row in any grid: {missing}") + + def test_the_connective_grid_can_fail() -> None: """The reachability probe every grid in this file carries. @@ -1337,6 +1551,33 @@ def test_initials_emit_exactly_the_predicted_contributors() -> None: f"criterion's contributors:\n" + "\n".join(failures[:10])) +def test_no_report_contradicts_the_reading_beside_it() -> None: + """INV10 (#397 second review). Two forms of one rule, both over + the connective grid's 55,800 rows: no token carries + connective-or-initial while the parse roles it SUFFIX or TITLE, + and no token is named by two reports whose details assert + different readings of it. + + The fork classify takes offers a connective and an initial. A + generation is neither, so where the parse reads one the report + describes a branch nobody took -- which is the narrow half. The + wide half needs no role at all: two reports on one token, one + saying it reads as an initial and the other that it reads as a + generational suffix, cannot both be true whatever the roles say. + + Mutation-checked, 2026-09-20: it fails on 8,204 rows at + dc3bdf9c -- 5,078 the narrow half and 3,126 the wide one -- and + on the same 8,204 at e540d4c5 and c8550b64, the report having + been wrong since the letter joined the marked subset. 0 here, + and deleting the withdrawal in `_pipeline/_assemble.py` restores + all 8,204. + """ + failures = _connective_findings()["INV10"] + assert not failures, ( + f"{len(failures)} report(s) contradict the reading beside " + f"them:\n" + "\n".join(failures[:10])) + + # --- #397 review: the OFF-SWITCH grid, and its two invariants ------- # A second grid, kept apart from the one above rather than folded # into it, and the reason is the cost: both invariants below parse @@ -1361,12 +1602,19 @@ def _off_switch_grid() -> list[tuple[str, Parser, str]]: mids: tuple[list[str], ...] = ([], ["Carod"], ["de", "Carod"]) conns = ("i", "y", "e", "and") tails: tuple[list[str], ...] = ([], ["Rovira"], ["de", "Rovira"]) + # The last two are the #397 SECOND review's addition: a trailing + # TITLE, alone and standing behind a credential. Where the peel + # is read over the pieces as WRITTEN a following title hides the + # suffix run from it, so the bound the join checks its right + # neighbour against said "name word" of a credential and the join + # swallowed it -- 'John Quincy Adams i MA Prof.' read family + # 'Adams i MA' where 'John Quincy Adams i MA' reads family + # 'Adams'. The lower-case spelling of the title comes free with + # the casing loop below, and the leading-title-only head is + # `["Dr."]` above. suffixes: tuple[list[str], ...] = ( - [], ["III"], ["Jr."], ["I"], ["V"], ["MA"], ["i"], ["nee", "Puig"]) - lexicons: _Variants = ( - ("default", Lexicon.default(), frozenset()), - ("conj+v", Lexicon.default().add(conjunctions={"v"}), - frozenset({"v"}))) + [], ["III"], ["Jr."], ["I"], ["V"], ["MA"], ["i"], ["nee", "Puig"], + ["MA", "Prof."], ["Prof."]) texts: list[str] = [] seen: set[str] = set() for head, mid, conn, tail, suffix, comma in itertools.product( @@ -1388,7 +1636,7 @@ def _off_switch_grid() -> list[tuple[str, Parser, str]]: if written not in seen: seen.add(written) texts.append(written) - return _rows(texts, lexicons) + return _rows(texts, _OFF_SWITCH_LEXICONS) _OFF_SWITCH_GRID = _off_switch_grid() @@ -1451,6 +1699,38 @@ def _name_word_beside(toks: list[_Placed], i: int, step: int, return off_role.get(toks[j][1]) in _NAME_ROLES +def _without_the_link(text: str, letters: frozenset[str]) -> str: + """The same name with every class letter deleted -- the LINK-FREE + CONTROL. + + What it is for: a credential can land inside a name part for a + reason that has nothing to do with the link, and one such reason + is older than #397 and untouched by it. The particle chain reads + where the trailing suffix run begins over the pieces as WRITTEN, + so a trailing title hides that run from it and the chain runs to + the end of the segment -- 'Josep de Carod y Rovira MA Prof.' + reads family 'de Carod y Rovira MA Prof.' at the parent 46651750 + and at 1.4.0, with no link of the generational class in it at + all. Deleting the link answers whether the link is what put the + credential there: where the control puts it in a name part too, + it did not. + + A trailing comma rides back onto the word before, so the comma + SHAPE survives the deletion ('Josep Carod i, MA' -> 'Josep + Carod, MA') -- the one structure whose loss would make the + control a different name rather than the same one shorter. + """ + out: list[str] = [] + for word in text.split(): + core = word.rstrip(",") + if core.lower() in letters: + if word.endswith(",") and out: + out[-1] += "," + continue + out.append(word) + return " ".join(out) + + def _initial_spans(name: ParsedName, letters: frozenset[str]) -> set[tuple[int, int]]: return {span for tok, span in _placed(name) @@ -1486,24 +1766,52 @@ def _link_joins_between_name_words(on: ParsedName, off: ParsedName, @functools.cache def _off_switch_findings() -> dict[str, list[str]]: - """One walk of the off-switch grid; three invariants' answers. + """One walk of the off-switch grid; FOUR invariants' answers. Both parses of a row -- as configured, and with the class letters out of the connectives -- are taken once here and handed to all - three predicates, which is the whole of what this walk does. The + four predicates, which is the whole of what this walk does. The exemptions stay each test's own: INV6 takes the join and the - initial reading, INV1-strengthened only the initial reading, INV7 - those two plus R4's placed-connective sentence, exactly as their - docstrings say. + initial reading, INV1-strengthened the initial reading and the + link-free control, INV7 the join and the initial reading plus + R4's placed-connective sentence, exactly as their docstrings say. + + INV6b costs NO parse at all, which is why it joins this walk + rather than opening a grid of its own: the grid already writes + every text in three casings, so the ALL-CAPS twin of a lower-case + row is another row of this same walk and the comparison is a + lookup over what the walk already read. The link-free controls + INV1-strengthened wants are the only parses added here, and only + on a row that would otherwise be recorded as a failure. """ out: dict[str, list[str]] = {k: [] for k in - ("INV6", "INV1-strengthened", "INV7")} + ("INV6", "INV1-strengthened", "INV7", + "INV6b")} v1_off: dict[frozenset[str], Constants] = {} + #: (label, text) -> the roles the configured parse gave, in token + #: order, for INV6b's casing comparison below. + roles_by: dict[tuple[str, str], + tuple[tuple[str, str, bool], ...]] = {} for text, parser, label in _OFF_SWITCH_GRID: letters = _present(text, _class_letters(parser.lexicon)) if not letters: continue on = parser.parse(text) + # INV6b's rows, and its scope: the MARKED subset. Its claim + # is decisions.md#P3's own -- a one-case name reads the + # marked letter as an initial and so reads as its ALL-CAPS + # twin already did -- and a class letter the caller's lexicon + # leaves UNMARKED is the shape P3's Accepted block records + # instead, where a bare Latin capital and its lowercase + # spelling genuinely part. The `conj+v` variant is exactly + # that caller: it adds 'v' to the connectives and not to the + # marked subset, so 'carod y rovira, v' and its ALL-CAPS twin + # disagree, identically at the parent 46651750 (20 rows, + # measured 2026-09-20, every one of them strict-comma). + if letters <= parser.lexicon.conjunctions_ambiguous: + roles_by[(label, text)] = tuple( + (tok.text.lower(), tok.role.value, + AMBIGUOUS_ACRONYM_TAG in tok.tags) for tok in on.tokens) off_parser = _off_switch(parser, letters) off = off_parser.parse(text) # the SEVEN FIELDS, and not comparison_key: a parse carries @@ -1534,10 +1842,37 @@ def _off_switch_findings() -> dict[str, list[str]]: # parse. if "conjunction" in tok.tags or tok.span is None: continue - if off_role.get(tok.span) is Role.SUFFIX: - out["INV1-strengthened"].append( - f"[{label}] {text!r}: {tok.text!r} reads as " - f"the suffix and joined into {role.value}") + if off_role.get(tok.span) is not Role.SUFFIX: + continue + # THE LINK-FREE CONTROL, and the reason it is + # asked here rather than folded into the oracle: + # the particle chain puts a credential into the + # family on its own where a trailing title hides + # the suffix run from it, link or no link, at + # this commit and at the parent and at 1.4.0 + # alike ('Josep de Carod y Rovira MA Prof.'). + # Turning the class letter off ALSO stops that + # chain -- the letter becomes suffix vocabulary + # again and the chain halts at it -- so the + # off-switch parse reads the credential as a + # suffix for a reason this rule is not about. + # Deleting the link asks the question directly. + # Recorded boundary, not a silence: the shape is + # pinned by test_a_trailing_title_still_hides_the + # _suffix_run_from_the_particle_chain below and + # is the #535 family. + # + # Taken only on a row already read as a failure, + # so the green state pays 15 parses over the + # whole grid (measured 2026-09-20). + control = parser.parse( + _without_the_link(text, letters)) + if any(c.text == tok.text and c.role is role + for c in control.tokens_for(role)): + continue + out["INV1-strengthened"].append( + f"[{label}] {text!r}: {tok.text!r} reads as " + f"the suffix and joined into {role.value}") if not (same_fields and not moved and not _placed_as_a_connective(on, letters)): continue @@ -1558,21 +1893,83 @@ def _off_switch_findings() -> dict[str, list[str]]: out["INV7"].append( f"[v1] {text!r} force={force}: {v1_here!r} != " f"off-switch {v1_there!r}") + # INV6b, over what the walk above already read. A text that is + # its own lower-casing and is not its own upper-casing is the + # all-lower spelling of a one-case name, and the grid wrote its + # ALL-CAPS twin too, under the same labels. + for (label, text), roles in roles_by.items(): + if text != text.lower() or text == text.upper(): + continue + twin = roles_by.get((label, text.upper())) + if twin is None or twin == roles: + continue + if _a_case_reading_the_docs_already_own(roles, twin): + continue + out["INV6b"].append( + f"[{label}] {text!r}: {list(roles)} != ALL-CAPS twin " + f"{list(twin)}") return out +def _a_case_reading_the_docs_already_own( + lower: tuple[tuple[str, str, bool], ...], + upper: tuple[tuple[str, str, bool], ...]) -> bool: + """INV6b's TWO exemptions, both of them case readings this + library decided long before #397 and neither of them the marked + subset's. + + ONE: a single-letter suffix word the lower spelling reads as the + suffix and the ALL-CAPS spelling does not. That is the + `suffix_not_acronyms` vs `is_an_initial` tension, Latin-only: + `_vocab.is_initial` matches an ASCII CAPITAL, so the peel's veto + fires on 'I' and 'V' and never on 'i' and 'v'. Measured + 2026-09-20 at the parent 46651750 on letters this branch does not + touch at all -- 'carod y i' reads suffix 'i' where 'CAROD Y I' + reads family 'I', and 'carod y v' against 'CAROD Y V' and 'carod + e i' against 'CAROD E I' are the same pair. + + TWO: a member of the AMBIGUOUS credential class reading + differently in the two spellings. That is decisions.md#S2's own + lean -- an ALL-CAPS bare member of the class reads as the + credential and a lower-case one as a name word -- and it decides + the COMMA STRUCTURE, so its effect reaches every token of the + name: 'i rovira, ma' reads given 'ma' and 'I ROVIRA, MA' suffix + 'MA', and the rest of the row moves with it. Identical at the + parent on all 24 rows it exempts, measured 2026-09-20, and 21 of + those carry no letter of #397's class at all. + + Neither exemption reaches what the branch DID close, which is + what makes INV6b worth running: 'rovira, i', 'john smith i jr' + and 'josep de carod i rovira' each disagreed with its ALL-CAPS + twin at the parent and agrees with it here. + + Compared by token INDEX, the two spellings having the same tokens + in the same order by construction. + """ + return any( + (a[1] == Role.SUFFIX.value and b[1] != Role.SUFFIX.value + and len(a[0]) == 1) + or ((a[2] or b[2]) and a[1] != b[1]) + for a, b in zip(lower, upper)) + + def test_the_off_switch_grid_can_fail() -> None: """The reachability probe, the shape every grid in this file carries. Dated recorded control, measured 2026-09-20. The row count fell from 103,040 to 53,312 with `_rows` (see the - connective probe above); the TEXT count is unmoved, and 2,011 of - the first 4,000 rows carry a class letter, against 1,800 of the - old grid's first 4,000. Over the whole grid 25,088 rows carry one - and are parsed twice; the old grid parsed 46,088 twice, six times - over.""" - assert len(_OFF_SWITCH_GRID) == 53312, len(_OFF_SWITCH_GRID) - assert len({t for t, _, _ in _OFF_SWITCH_GRID}) == 12880 + connective probe above); the TEXT count is unmoved by the trim, + and 2,011 of the first 4,000 rows carried a class letter against + 1,800 of the pre-trim grid's first 4,000. + + RE-MEASURED 2026-09-20 for the #397 second review, which added + the two trailing-title suffix runs: 66,752 rows over 16,576 + texts, 2,109 of the first 4,000 rows carrying a class letter and + 28,448 over the whole grid -- those being the rows parsed twice. + The module's own runtime is the budget these numbers spend, and + it is stated where the walk is built.""" + assert len(_OFF_SWITCH_GRID) == 66752, len(_OFF_SWITCH_GRID) + assert len({t for t, _, _ in _OFF_SWITCH_GRID}) == 16576 reached = sum(1 for text, parser, _ in _OFF_SWITCH_GRID[:4000] if _present(text, _class_letters(parser.lexicon))) assert reached > 1000, reached @@ -1592,11 +1989,15 @@ def test_a_link_that_joins_nothing_changes_no_field() -> None: clause, which the switch decides along with the join and so cannot hold fixed. - Mutation-checked, 2026-09-20: this fails on 1,250 parses at - c8550b64, the commit the review was written against (2,360 over - the full cross product), and on 610 where the both-sides gate - tests POSITION rather than class -- the c8550b64 defect isolated, - which is the one INV1 above cannot see. + Mutation-checked, re-measured 2026-09-20 over the grid as it + now stands (the second review added the two trailing-title + suffix runs, so every count here moved with it): this fails on + 1,480 parses at c8550b64, the commit the first review was + written against, on 764 where the both-sides gate tests POSITION + rather than class -- the c8550b64 defect isolated, which is the + one INV1 above cannot see -- and on 154 at dc3bdf9c, where the + join's right-hand bound was read over the pieces as WRITTEN and + a trailing title hid the suffix run from it. 0 here. """ failures = _off_switch_findings()["INV6"] assert not failures, ( @@ -1618,9 +2019,18 @@ def test_a_trailing_credential_never_joins_into_a_name_part() -> None: is deliberately absent -- a link joining elsewhere in the name never licenses a credential joining here. - Mutation-checked, 2026-09-20: this fails on 534 parses at - c8550b64 (960 over the full cross product), where INV1 fails on - none of them. + THE LINK-FREE CONTROL is its third exemption, added by the + second review and documented at the walk: a credential can land + in a name part for a reason older than #397 and untouched by it, + and deleting the link is how this asks whether the link is what + put it there. + + Mutation-checked, re-measured 2026-09-20 over the grid as it now + stands: this fails on 639 parses at c8550b64, where INV1 fails + on none of them, on the same 639 where the both-sides gate tests + POSITION rather than class, and on 105 at dc3bdf9c -- the + trailing-title bound, the defect the two new suffix runs were + added to see. 0 here. """ failures = _off_switch_findings()["INV1-strengthened"] assert not failures, ( @@ -1677,15 +2087,16 @@ def test_a_letter_that_did_not_join_repairs_as_the_off_switch_does() -> None: not a connective at all. What is left is the generation, and the rule for it is that it repairs as the generation it was read as. - Mutation-checked, re-measured 2026-09-20 on the trimmed grid: - this fails on 7,382 repairs at e540d4c5, where the - suffix-roled letter still took the connective conjunct, and on 0 - here; removing the role test alone fails it on the same 7,382. - (11,341 for both over the full cross product.) The v1 arm runs on - the default-lexicon, default-policy rows, the only ones a - `Constants` can express, and 917 of those failures are its -- the - same 917 as before the trim, those rows never having been - duplicated. + Mutation-checked, re-measured 2026-09-20 over the grid as it + now stands: this fails on 7,860 repairs at e540d4c5, where the + suffix-roled letter still took the connective conjunct, on 8,054 + with the generation conjunct removed outright, and on 0 here and + at dc3bdf9c -- dc3bdf9c PASSES it, which is the point of INV7b + below: the guard there was the suffix ROLE alone, which is too + wide, and too wide in a direction this invariant's subject + cannot reach. The v1 arm runs on the default-lexicon, + default-policy rows, the only ones a `Constants` can express, + and 1,017 of the e540d4c5 failures are its. """ failures = _off_switch_findings()["INV7"] assert not failures, ( @@ -1693,6 +2104,144 @@ def test_a_letter_that_did_not_join_repairs_as_the_off_switch_does() -> None: f"nothing:\n" + "\n".join(failures[:10])) +def test_a_one_case_name_reads_the_same_in_either_case() -> None: + """INV6b (#397 second review). decisions.md#P3 states in prose + that a name written wholly in lower case now reads as its + ALL-CAPS twin already did, and INV6 cannot hold it: the + `_initial_reading_moved` exemption there swallows the one-case + population, 9,296 of the 28,448 rows that carry a class letter, + which is exactly where the marked subset's riskiest movement + lives. + + Every all-lower row's roles, token for token, against the roles + of the same text ALL-CAPS under the same parser. Costs no parse: + the grid writes both spellings, so the twin is another row of the + walk that just ran. + + Scope and exemptions in `_a_case_reading_the_docs_already_own` + and at the `roles_by` write -- the marked subset, less two case + readings the library decided long before this rule. + + Mutation-checked, 2026-09-20: marking the letter for ALL-CAPS + names only -- `and token.text.isupper()` on classify's fork -- + fails this on 944 rows. Five unit tests in + tests/v2/pipeline/test_classify.py fall with it, so this is not + the only guard on that line and is not claimed to be; what it + adds is the SHAPE of the claim. Those five each assert one + spelling's reading against a stored expectation; this asserts + that the two spellings AGREE, over the 6,496 all-lower rows this + invariant's scope keeps -- where INV6 is exempt and where, until + the four rows this round added, no row of tests/v2/cases.py + stood. What it holds is + real: 'rovira, i', 'john smith i jr' and 'josep de carod i + rovira' each disagreed with its ALL-CAPS twin at the parent + 46651750 and agrees with it here. + """ + failures = _off_switch_findings()["INV6b"] + assert not failures, ( + f"{len(failures)} one-case name(s) read differently from " + f"their ALL-CAPS twin:\n" + "\n".join(failures[:10])) + + +# --- #397 second review: the SUFFIX-FIELD CONNECTIVE grid ----------- +# A fourth grid, and a tiny one -- 60 rows against the three above in +# the tens of thousands -- because the shape it is about is one none +# of them can generate. Every grid above puts its connective among +# the name's own words or inside a clause; this one puts a connective +# in the SUFFIX FIELD without its being generational vocabulary, and +# there are exactly two ways to do that: a third comma part, whose +# words assign reads as the suffix run whatever they are, and a field +# spliced in after the parse. +# +# The oracle cannot be the off switch, and that is the reason for the +# separate grid rather than a second reason for it: no `i` stands in +# any of these names, so removing `i` from the connectives changes +# nothing and the comparison would be vacuous -- the silence a +# reachability probe exists to catch. The oracle is the RULE instead, +# the RULE instead, rules.md#R4 read in its own words: a suffix-roled +# connective that the suffix vocabulary does NOT hold was read as no +# generation and keeps its lowercase. +# The literal strings the released wheels give are pinned separately, +# as unit tests in tests/v2/test_render.py. + +_SUFFIX_FIELD_CONNECTIVES = ("and", "y", "e", "und", "of", "&", "и") + + +def _suffix_field_rows() -> list[tuple[str, str]]: + """(description, the connective word) for every way this grid + puts a connective into the suffix field.""" + rows: list[tuple[str, str]] = [] + for word in _SUFFIX_FIELD_CONNECTIVES: + rows += [(f"Smith, John, {word}", word), + (f"Doe, Jane, {word} Jr.", word), + (f"Smith, John, {word} III", word)] + return rows + + +def test_a_connective_the_suffix_field_holds_keeps_its_lowercase() -> None: + """INV7b (#397 second review). R4's clause turns on the + GENERATION and not on the field: a connective the suffix field + merely holds, which the suffix vocabulary does not know, was read + as no generation, so case repair leaves it lowercase -- plain and + forced, on both surfaces. + + INV7 cannot see this. Its oracle is the off switch and its + subject is the class letter, and no name here carries one: `and`, + `y`, `e`, `und`, `of`, `&` and `и` are connectives and nothing + else, which is the whole point -- they are what a role test alone + could not tell apart from the generation. + + Mutation-checked, 2026-09-20: this fails on 36 of its 42 rows + at dc3bdf9c, where `role is not Role.SUFFIX` gated both + conjunction arms, and on 0 here; the six that pass there are the + Cyrillic 'и', which repairs to itself either way. 'Smith, John, + and' repaired forced to 'John Smith And' there, against 'John + Smith and' at 1.4.0, at 2.0.0 through 2.3.0 and at the parent + 46651750. INV7 above cannot see any of this -- it passes at + dc3bdf9c. + """ + parser = Parser() + failures = [] + for text, word in _suffix_field_rows(): + name = parser.parse(text) + if word not in {t.text for t in name.tokens_for(Role.SUFFIX)}: + failures.append(f"{text!r}: {word!r} is not in the suffix") + continue + for force in (False, True): + repaired = str(parser.capitalized(name, force=force)) + v1 = _v1_capitalized(text, None, force) + for label, got in (("core", repaired), ("v1", v1)): + if word not in got.split(): + failures.append( + f"[{label}] {text!r} force={force}: {got!r} does " + f"not keep {word!r} lowercase") + assert not failures, ( + f"{len(failures)} repair(s) capitalized a connective the " + f"suffix field merely holds:\n" + "\n".join(failures[:10])) + + +def test_the_suffix_field_connective_grid_can_fail() -> None: + """The reachability probe, the shape every grid in this file + carries. Dated recorded control, measured 2026-09-20. + + The one that matters here is the SECOND: a row whose connective + the parse did not actually put in the suffix field would pass + INV7b for the wrong reason, and the invariant records that as a + failure rather than skipping it, so this counts the rows that + reach the repair at all. + """ + rows = _suffix_field_rows() + assert len(rows) == 21, len(rows) + parser = Parser() + reached = sum( + 1 for text, word in rows + if word in {t.text for t in parser.parse(text).tokens_for( + Role.SUFFIX)}) + assert reached == 21, reached + assert not _class_letters(Lexicon.default()) & { + w for _t, w in rows}, "a class letter would make INV7 the oracle" + + # --- #397 review: the MAIDEN-CLAUSE LINK grid, and its two ---------- # invariants # A third grid, kept apart from the two above for the reason the diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 1eecb79b..87b84e77 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -291,6 +291,68 @@ def test_repair_capitalizes_a_generation_the_connective_also_spells( force=True)) == "Josep y Rovira" +def test_repair_keeps_a_plain_connective_the_suffix_field_holds_lower( +) -> None: + """rules.md#R4, the GENERATION half read narrowly (#397 second + review). The clause turns on the generation and not on the field: + the test is the suffix ROLE **and** the suffix VOCABULARY, and a + role test alone capitalized every plain connective that merely + LANDS in the suffix field. + + The third part of a comma form is the shape that puts one there + -- assign reads those words as the suffix run whatever they are + -- and none of them is generational vocabulary, so none of them + was read as a generation and all of them keep the lowercase they + have always had. Expected strings measured 2026-09-20 on the + released 1.4.0 and 2.3.0 wheels from a throwaway environment, and + they agree with each other and with the parent commit 46651750. + + 'Smith, John, и' is the one row where the two wheels part, and + for a reason older than this rule: Cyrillic 'и' is a 2.x + conjunction and not a 1.4.0 one, so 1.4.0 gives 'John Smith И' + and 2.3.0 gives 'John Smith и'. The tree follows 2.3.0, as + _render.py's own note on that word says it must. + """ + for text, plain, forced in ( + ("Smith, John, and", "John Smith and", "John Smith and"), + ("Smith, John, y", "John Smith y", "John Smith y"), + ("Smith, John, e", "John Smith e", "John Smith e"), + ("Smith, John, und", "John Smith und", "John Smith und"), + ("Smith, John, of", "John Smith of", "John Smith of"), + ("Smith, John, и", "John Smith и", "John Smith и"), + ("Doe, Jane, and Jr.", "Jane Doe and Jr.", + "Jane Doe and Jr."), + ("Smith, John, and III", "John Smith and III", + "John Smith and III")): + name = parse(text) + assert name.suffix.split()[0] == text.split(", ")[-1].split()[0] + assert str(name.capitalized()) == plain, text + assert str(name.capitalized(force=True)) == forced, text + v1 = HumanName(text) + v1.capitalize(force=True) + assert str(v1) == forced, text + # A SPLICED field is the other way in, and R4's Accepted + # paragraph is explicit about it: text nobody read gets the + # vocabulary's answer, so a suffix set to 'de y' keeps its 'y' + # exactly as a family set to 'de y' does. 1.4.0 and 2.3.0 both + # give 'John Smith De, y'. + spliced = HumanName(first="John", last="Smith", suffix="de y") + spliced.capitalize(force=True) + assert str(spliced) == "John Smith De, y" + # ... and the vocabulary's answer is what a spliced 'i' gets too, + # which is a CHANGE from the parent and the shape to know about: + # 'i' is connective vocabulary here and was not there, so the + # spliced field follows 'y' now where it used to follow the + # numeral. The PARSED name is unaffected -- it has a reading, and + # the reading is the generation (above). + spliced_i = HumanName(first="John", last="Smith", suffix="i") + spliced_i.capitalize(force=True) + assert str(spliced_i) == "John Smith i" + spliced_y = HumanName(first="John", last="Smith", suffix="y") + spliced_y.capitalize(force=True) + assert str(spliced_y) == "John Smith y" + + def test_initials_order_folded_words_first_like_the_family_field() -> None: """#408: the view and the field must read one parse the same way. diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 10137f29..ea049949 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -38,6 +38,7 @@ "Davis Royce, Ed" "Del Toro" "Doe, Dr. nee Smith MA" +"Doe, Jane, and Jr." "Doe, John DO" "Doe, John DO Ed" "Doe, John Do" @@ -83,6 +84,7 @@ "J.R. Smith" "JACK MA" "JOHN PROF. MA" +"JOHN QUINCY SMITH I" "JOHN SMITH MA" "JUAN GARCIA Y LOPEZ" "Jack MA" @@ -273,6 +275,7 @@ "Smith, John Prof." "Smith, John V" "Smith, John V." +"Smith, John, and" "Smith, Jr." "Smith, MA" "Smith, MD PhD" From 6048eb5df89f9efd163abbe5c3a4902aa288bd5b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 13:28:16 -0700 Subject: [PATCH 10/11] perf(#397): a run of connectives is asked about once, not once per member `_name_word_beside` answers P3's both-sides condition by walking the run of connectives beside a link to find the first name word past it, and both of its callers ask it once per MEMBER of that run -- so a name holding a run of n links walked it n times and the stage went quadratic in the run's length. Found while reporting the review round in b9ed1429; `parse()` takes arbitrary strings and that PR's body claims the join loop is linear, so it has to be. Every member of one contiguous run has the SAME nearest non-connective piece on each side, so `_run_neighbours` computes both in two linear passes over the segment and `_name_word_beside` reads an index instead of walking to one. Built on the gate the two bounds already share -- only where a generational connective was found, or (through the memo cell the clause predicate fills on first use) where a clause's walk reached a suffix piece -- so a name with no such connective is untouched. The out-of-range sentinels (-1 and len(pieces)) are what "the run ran out" arrives as, and the bound test already refused them. Milliseconds, minimum of 7 runs, py3.11: shape n=100 200 400 800 1600 "Josep " + "i "*n + "Rovira" parent 46651750 0.5 1.0 2.1 4.1 8.3 b9ed1429 1.4 4.2 15.9 58.6 214.1 here 0.6 1.3 2.5 5.1 10.2 "Josep Carod " + "i y "*n + "Rovira" parent 46651750 1.1 2.2 4.4 8.8 17.6 b9ed1429 2.6 9.0 32.8 113.7 420.3 here 1.2 2.4 4.7 9.4 19.1 "Jane Doe nee Puig " + "i "*n + "Soler" parent 46651750 0.6 1.1 2.1 4.3 8.5 b9ed1429 1.3 4.0 15.2 59.3 213.4 here 0.6 1.1 2.1 4.2 8.4 "Josep " + "Carod i "*n + "Rovira" (runs of one; linear already) parent 46651750 1.1 2.2 4.4 8.8 17.6 b9ed1429 1.2 2.4 4.9 9.7 19.5 here 1.2 2.4 4.8 9.7 19.3 b9ed1429 grows 3.65-3.89x per doubling on the first three; here they read 1.97-2.03x, at the parent's absolute cost. BYTE-IDENTICAL, 1,748,904 parses a side and the same SHA-256: 145,742 names -- the differential corpora, the case table, all three property grids, and a generated run-heavy set (runs of 1-6 connectives mixing i/y/e/and/of, at the head, middle and tail, with suffix and title tails, comma and no-comma, and inside maiden clauses, in three casings) -- under three lexicons (default, add(conjunctions={"v"}), remove(conjunctions_ambiguous={"i"})) and four policies (default, both family-first orders, strict comma). GUARDED TWICE, because one instrument cannot reach both callers. tests/v2/test_benchmark.py gains a thirteenth `_SHAPES` row, `link_run` ("i Und " -- mixed case, since a one-case name reads the letter as an initial and reaches nothing), which reads 4.02-4.19 here and 13.5-14.2 at b9ed1429 across three repeated runs, against a 6.0 bound. The clause caller needs a PREFIX its table cannot express -- "nee i Und " repeated never reaches the clause rule at all, measuring the identical 3.92/4.07/4.12 on both trees -- so it gets `test_a_clause_link_run_does_not_cost_quadratically`, which builds its own input, asserts the clause still holds the run, and counts frames: 876 -> 2,652 here (3.03x) against 1,125 -> 6,741 (5.99x), bound 4.5. THE PRICE, recorded rather than hidden: answering for the whole segment costs a short name one call plus `_is_conj_piece` for the pieces the walk never reached. `Josep Carod i Rovira` 304 -> 307 frames and `Jane Doe nee Puig i Soler` 307 -> 312, against 6,741 -> 2,652 for that second name with a run of 64. `tools/perf/call_count.py` is unmoved (parse=406.00 facade=443.00) and so are `John Smith` 163, `Smith, John` 185, `Juan Garcia y Lopez` 285 and `Jane Doe nee Smith` 233. Mutation-checked, each arm by a named test: both sides reading the left array fails test_a_connective_with_nothing_to_its_right_does_not_ join, both reading the right fails test_a_leading_title_on_the_left_is_ no_name_word, either pass losing its `not` fails test_a_connective_ piece_counts_toward_the_carve_outs_total, and rebuilding per call instead of once fails the new clause guard. SWAPPING the two arrays outright is an equivalent mutant and nothing fails -- both callers AND the two sides, so the conjunction cannot see which array answers which. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/design/decisions.md | 1 + nameparser/_pipeline/_group.py | 146 +++++++++++++++++++++++++++++---- tests/v2/test_benchmark.py | 91 ++++++++++++++++++++ 4 files changed, 224 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dcc42bd2..e4f01427 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -382,7 +382,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. **Anything built on `_normalize` must converge too** — `_fold_words` runs `_normalize` per word and DROPS the words that fold away (`_title_key` is that list space-joined, and `_run_addresses_by_given` reads the list itself, so its last-word arm is the last word of the FOLDED key by construction); keeping the empty slot stored `'lt .'` as `'lt '`, a key match-time can never rebuild (so the entry is silently inert) and `__setstate__` rejects on the next round-trip as "not written by this version". -**Perf regressions are caught by the scaling test, not the absolute-time ones** — `tests/v2/test_benchmark.py::test_parse_cost_grows_no_worse_than_linearly` times a repeated unit at n vs 4n over twelve shapes (one per pipeline inner loop) and bounds the ratio; the `_thousand_names` tests use constant-size, delimiter-free input and are structurally blind to a complexity regression. Two rules when touching it: calibrate `_MAX_RATIO` against the WEAKEST quadratic's signal (a mixed quadratic surfaces far below the textbook 16×, so the operating point `_BASE` matters more than the bound), and confirm a planted regression fails it across REPEATED runs — one failure is a coin-flip on a timing test. The twelve shapes cover different dimensions (segment count only via `commas`, intra-piece accumulation only via `particles`/`conjunctions`, non-ASCII input only via `honorifics` — the other eleven are pure ASCII, so `script_segment` returns at its bail and the CJK stages go unmeasured, and M2's clause view only via `maiden_clause`, whose unit has to END on a class member: `MA nee ` holds the same two words, the peel stops at the trailing marker, and the shape reaches nothing); measure before pruning one. **A shape the CLOCK cannot reach needs a FRAME-count guard instead**, which is the second scaling test in that file (`test_a_trailing_credential_run_does_not_cost_exponentially`, #531): where the defect is an exponential rather than a quadratic, the input length that separates the curves on a timing test does not finish, so the guard counts frames over 8 units against 16 and bounds THAT ratio. One pair does not see every curve, and the fix round for #531 measured why: at 2× the input the per-member LINEAR work swamps a quadratic (2.08× for a genuine one against 1.73× clean), so that pair guards the exponential alone and a second, longer pair — 16 against 64, where the same quadratic reads 7.42× against 3.53× clean — is what can see one. Assert them in that order: an exponential never returns from the longer run, so the cheap pair has to have failed first. Frame counts do not move under load, so this shape needs no repeated-run calibration — but it does need the same reachability assertion `_POLICY_SHAPES` rows carry, since the walk under measurement runs only while every unit still reads as a credential. A stage gated on an opt-in `Policy` field needs a `_POLICY_SHAPES` entry instead, since bare `parse()` never enters it — and that table's rows carry a **reachability probe** run before the measurement, because a precedence change can quietly stop the shape reaching the stage and leave a green test measuring a no-op (`_POLICY_SHAPES` is also asserted non-empty: an empty `parametrize` is a skip, not a failure, so deleting its last row would retire the guard silently). +**Perf regressions are caught by the scaling test, not the absolute-time ones** — `tests/v2/test_benchmark.py::test_parse_cost_grows_no_worse_than_linearly` times a repeated unit at n vs 4n over thirteen shapes (one per pipeline inner loop) and bounds the ratio; the `_thousand_names` tests use constant-size, delimiter-free input and are structurally blind to a complexity regression. Two rules when touching it: calibrate `_MAX_RATIO` against the WEAKEST quadratic's signal (a mixed quadratic surfaces far below the textbook 16×, so the operating point `_BASE` matters more than the bound), and confirm a planted regression fails it across REPEATED runs — one failure is a coin-flip on a timing test. The thirteen shapes cover different dimensions (segment count only via `commas`, intra-piece accumulation only via `particles`/`conjunctions`, non-ASCII input only via `honorifics` — the other twelve are pure ASCII, so `script_segment` returns at its bail and the CJK stages go unmeasured, M2's clause view only via `maiden_clause`, whose unit has to END on a class member: `MA nee ` holds the same two words, the peel stops at the trailing marker, and the shape reaches nothing — and connective RUN LENGTH only via `link_run`, whose unit has to be MIXED CASE and hold a connective of the generational class: `i und ` reads the letter as an initial and reaches nothing where `i Und ` reaches everything); measure before pruning one. **A shape whose input needs a PREFIX cannot be a `_SHAPES` row at all**, since that table repeats a unit and nothing else — a maiden clause needs a name word and a marker before the run it is about, and `"nee i Und " * n` reaches the clause rule not at all, measuring the identical ratio on a broken tree and a fixed one. That one is `test_a_clause_link_run_does_not_cost_quadratically`, which builds its own input and counts FRAMES. **A shape the CLOCK cannot reach needs a FRAME-count guard instead**, which is the second scaling test in that file (`test_a_trailing_credential_run_does_not_cost_exponentially`, #531): where the defect is an exponential rather than a quadratic, the input length that separates the curves on a timing test does not finish, so the guard counts frames over 8 units against 16 and bounds THAT ratio. One pair does not see every curve, and the fix round for #531 measured why: at 2× the input the per-member LINEAR work swamps a quadratic (2.08× for a genuine one against 1.73× clean), so that pair guards the exponential alone and a second, longer pair — 16 against 64, where the same quadratic reads 7.42× against 3.53× clean — is what can see one. Assert them in that order: an exponential never returns from the longer run, so the cheap pair has to have failed first. Frame counts do not move under load, so this shape needs no repeated-run calibration — but it does need the same reachability assertion `_POLICY_SHAPES` rows carry, since the walk under measurement runs only while every unit still reads as a credential. A stage gated on an opt-in `Policy` field needs a `_POLICY_SHAPES` entry instead, since bare `parse()` never enters it — and that table's rows carry a **reachability probe** run before the measurement, because a precedence change can quietly stop the shape reaching the stage and leave a green test measuring a no-op (`_POLICY_SHAPES` is also asserted non-empty: an empty `parametrize` is a skip, not a failure, so deleting its last row would retire the guard silently). **Expected-failure tests use `@pytest.mark.xfail`** — the conftest parametrized fixture breaks `@unittest.expectedFailure`; always use `@pytest.mark.xfail` instead. diff --git a/docs/design/decisions.md b/docs/design/decisions.md index d6a00234..c41f5af4 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -367,6 +367,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf WIDER THAN ONE ROW, and the second review measured how much: the design's reading tables show one name because they hold one of this shape, not because one moves. Over the review's 43,925-name generated grid, `add(particles={"y"})` moves a word the parent roled SUFFIX into a name part on 130 names BEYOND what the default lexicon already moves there — "ABDUL Y ROVIRA MA" goes from given "ABDUL" / family "Y ROVIRA" / suffix "MA" to given "ABDUL Y ROVIRA" / family "MA" — and 71 of those 130 still report `suffix-or-name` on the moved word, so the reading is marked rather than silent. Four more lose a title, all four the same name in its four spellings ("Jr. y Soler Puig MA" and its casings). The direction is the one the row above describes: the custom lexicon converges on the default one, the particle joining what the connective already joined. Measured 2026-09-20. THE BOTH-SIDES SCOPE, AND THE REVIEW THAT FIXED IT. A connective that is also generational vocabulary joins only where a NAME WORD stands on each side of it — a word the rest of the parse reads as a name word rather than as a credential, the trailing numeral or a title, looked for past any run of connectives standing between. The FIRST implementation tested POSITION (an interior piece) rather than the neighbours' CLASS, and swallowed a credential: "Josep Lluis Carod i III" read family "Carod i III" where the "III" is the generation. An independent review found it. It is recorded here because the invariant that ought to have caught it cannot: INV1 inspects the part the join PRODUCED, where the absorbed credential is itself the name word standing on the right, so that name SATISFIES it — the general lesson is a field note in mechanisms.md of this date, and the instrument is `tests/v2/test_properties.py::test_a_trailing_credential_never_joins_into_a_name_part`, which asks the off-switch parse instead and carries its own control (639 failing parses at `c8550b64` over the shipped grid, 105 at `dc3bdf9c`, 0 here; 960 over the full cross product, a dated snapshot). THE BOUND THE CONDITION READS, AND THE SECOND REVIEW THAT FIXED IT (2026-09-20). "A name word" is looked for between assign's leading-title run and the start of its trailing suffix run, and that second bound was read by `_pieces.trailing_start`, the peel over the pieces as WRITTEN. A trailing TITLE hides the suffix run from that peel — it takes nothing and answers `len(pieces)` — so the condition was told a credential was a name word and the join swallowed it: `John Quincy Adams i MA Prof.` read family 'Adams i MA' with no report and an `initials()` of "J. Q. A. M.", where `John Quincy Adams i MA`, the same name one title shorter, reads family 'Adams' with suffix 'i MA' and reports the acronym. `John Quincy i MA Prof.`, `Josep Lluis i MA Prof.` and their one-case spellings are the rest of it. THE FIX IS H5's OWN READING, `_pieces.trailing_start_past_titles`, which runs the peel and the title chain to the fixed point `tail_reading` already computes for assign — so the join and the placement agree about where the name ends, which is what H5's sentence says they must. Asked once per segment and only where such a connective was found, so no ordinary name pays it; frame-neutral on `tools/perf/call_count.py` (parse=406.00, facade=443.00). Measured over the review's 43,925-name grid under eight configurations: 27 rows move, on 9 distinct names, every one of them onto the reading the off-switch parser gives. + AND THE CONDITION IS ASKED ONCE PER RUN, not once per member, which is a THIRD fix in the same clause and was a defect the second review left standing (found in its own report, fixed 2026-09-20). `_name_word_beside` walked the run of connectives beside a link to find the first name word past it, and both callers ask it per member, so a name holding a run of n links walked it n times and the stage went quadratic in the run's length: `"Josep " + "i " * n + "Rovira"` grew 3.8x per doubling against the 2.0x every other shape holds — 59ms at n=800 and 214ms at 1,600, where the parent 46651750 reads 8.3ms — and `"Jane Doe nee Puig " + "i " * n + "Soler"` the same, the clause's own caller. Every member of one contiguous run has the SAME nearest name word on each side, so `_group._run_neighbours` answers it once for the whole segment in two linear passes, built on the same gate as the two bounds. All four shapes now read 1.97-2.03x per doubling to n=1,600. THE PRICE IS A CONSTANT ON SHORT NAMES and is recorded rather than hidden: one call plus `_is_conj_piece` for the pieces the walk never reached, so `Josep Carod i Rovira` goes 304 -> 307 frames and `Jane Doe nee Puig i Soler` 307 -> 312, against 6,741 -> 2,652 for that second name with a run of 64. A name with no generational connective is untouched (`John Smith` 163, `Smith, John` 185, `Juan Garcia y Lopez` 285, `Jane Doe nee Smith` 233, all unmoved). Byte-identical over 1,748,904 parses — 145,742 names, the corpus, the case table, all three property grids and a run-heavy generated set, under three lexicons and four policies — same digest on both trees. Guarded twice, because one instrument cannot reach both callers: `tests/v2/test_benchmark.py`'s thirteenth `_SHAPES` row (`link_run`, `"i Und "`) times the join's caller, and `test_a_clause_link_run_does_not_cost_quadratically` counts frames for the clause's, which no repeated unit can express. NOT FIXED AND RECORDED INSTEAD, because it is neither this rule's nor this branch's: the PARTICLE CHAIN reads the same bound the same blind way, so a trailing title lets it run to the end of the segment. `de la ma prof.`, `de la e ma prof.`, `van der berg ma prof.` and `Josep de Carod y Rovira MA Prof.` all put the title inside the family at the parent 46651750 and at 1.4.0, with no letter of this class in them at all. What the branch does is send MORE names down it — once `i` is marked, a one-case `de la i ma` stops ending in a suffix piece and the chain reaches past it — so `de la i ma prof.` and `de la i rovira prof.` now read as their `e` twins always have. 25 names of the review grid are in this family and none of them is a `_name_word_beside` case. It is the #535 boundary and belongs to its own change; `tests/v2/test_properties.py`'s INV1-strengthened carries a LINK-FREE CONTROL so the invariant can tell the two apart, and says so where it is asked. THE CLASS TEST IS NOT A LENGTH TEST, corrected by the second review (2026-09-20). The stage's own filter carried a `len(tok.text) != 1` beside the two tag tests, narrowing this clause to ONE-LETTER connectives where the rule names a class. Caller-reachable: under `Lexicon.default().add(conjunctions={"og"}, suffix_words={"og"})`, `John Quincy Smith og` read family 'Smith og' with the filter and reads family 'Smith' with suffix 'og' without it — the rule's own answer. DROPPED, measured byte-identical over every oracle in use (fields, reports, `initials()`, `capitalized()` plain and forced, every token's role) across 359,053 parses: the 351,400-parse review grid under eight lexicon/policy/locale configurations, the 2,175-parse sweep of `tests/v2/cases.py` under three orders and the 5,478-parse sweep of the differential corpora under six. It has to be — `i` is the only member of the class in the default vocabulary and in every locale pack, and it is one letter. The THREE-WORD carve-out stays single-letter, because that is what its own sentence says. THE SCOPE'S ALTERNATIVES, MEASURED 2026-09-20 rather than argued, by building each alternative from this tree and sweeping the 1558 non-empty corpus-union-cases names under eight configurations (default, `lenient_comma_suffixes=False`, both family-first orders, and the four locale packs). ABSENT — no both-sides condition at all — differs on five names, and every one of them is a row this bundle added: "John Quincy Smith i" would read family "Smith i", "Josep Lluis Carod i III" family "Carod i III", "Josep Lluis Carod i V" family "Carod i V", "Carod y de Rovira i" given "Carod y de" with family "Rovira i", and "Rovira, Josep Carod i Jr." middle "Carod i Jr.". Nothing that stood in the corpus at the parent separates absent from narrow, so those rows ARE the evidence and pruning one retires it. WIDE — the same condition with the generational-vocabulary test dropped, so it reaches every single-character connective — differs on five as well, and THREE of those were in the corpus already: "Mr. & Mrs. John Smith" would read title "Mr." with given "&" and middle "Mrs. John", "1 & 2, 3 4 5, Mr." would lose "2" out of the family, and "Jane née Jr y Jones" would read middle "née Jr y" with family "Jones". So the class test is not decoration — widening the rule past the link-and-generation class breaks "Mr. & Mrs." — and narrow is both what the design asked for and what measurement supports. (A "0 differences over 1545 names" figure was measured for this scope before the review supplied the class test and before those rows existed; it is superseded here.) diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 4c0b8642..19f11873 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -308,17 +308,29 @@ def _join_takes_the_member(view: Sequence[Sequence[int]], def _link_joins_inside_the_clause(k: int, lo: int, hi: int, pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], - tokens: Sequence[WorkToken]) -> bool: + tokens: Sequence[WorkToken], + beside: list[_Beside]) -> bool: """Whether the suffix piece at `k` is a connective PLACED TO JOIN between two name words of the clause `lo`..`hi`. Defined here, beside its one caller, and forward-referencing the two predicates it is built out of: `_is_conj_piece` and `_name_word_beside` are the JOIN's, further down this module, and - moving them up to meet this would say they belonged to the clause.""" + moving them up to meet this would say they belonged to the clause. + + `beside` is the caller's memo cell, EMPTY until the first piece + that gets this far -- which is a suffix piece inside a clause, so + almost no name reaches it -- and filled here rather than at the + call site so the laziness costs no frame of its own. The walk + does not touch `pieces` or `ptags`, so one fill serves every + piece of it (#397 second review, the run fix).""" + if not beside: + beside.append(_run_neighbours(pieces, ptags, tokens)) return (_is_conj_piece(pieces[k], ptags[k], tokens) - and _name_word_beside(k, -1, lo, hi, pieces, ptags, tokens) - and _name_word_beside(k, 1, lo, hi, pieces, ptags, tokens)) + and _name_word_beside(k, -1, lo, hi, pieces, ptags, tokens, + beside[0]) + and _name_word_beside(k, 1, lo, hi, pieces, ptags, tokens, + beside[0])) def _maiden_take(pieces: Sequence[Sequence[int]], @@ -609,11 +621,17 @@ def _maiden_take(pieces: Sequence[Sequence[int]], # either ('PhD née - i Jones'). lo = seen[m + run] j = m + run + # The link exception's memo cell, filled on first use inside the + # predicate: an ordinary clause never reaches it, and a clause + # holding a RUN of links asks about the run once instead of once + # per member ('Jane Doe nee Puig i i i ... Soler'). + beside: list[_Beside] = [] while (j < len(seen) and seen[j] < trailing and (not is_suffix_piece(pieces[seen[j]], ptags[seen[j]], tokens) or _link_joins_inside_the_clause(seen[j], lo, peel_start, - pieces, ptags, tokens))): + pieces, ptags, tokens, + beside))): j += 1 # j == m + run means nothing followed the marker but a suffix, so # the pass declines and the marker stays ordinary words @@ -664,6 +682,85 @@ def _maiden_take(pieces: Sequence[Sequence[int]], return seen[m:m + run], seen[m + run:j] +#: What `_name_word_beside` reads instead of walking: two arrays over +#: the segment's pieces, giving for each index the nearest piece on +#: its left and on its right that is NOT a connective -- `-1` and +#: `len(pieces)` where the run reaches the end. Built by +#: `_run_neighbours`, and an ALIAS rather than a NamedTuple because a +#: NamedTuple's __new__ is a frame of its own (decisions.md#parse-cost). +_Beside = tuple[list[int], list[int]] + + +def _run_neighbours(pieces: Sequence[Sequence[int]], + ptags: Sequence[Set[str]], + tokens: Sequence[WorkToken]) -> _Beside: + """The nearest non-connective piece on each side of every index. + + EVERY MEMBER OF ONE RUN HAS THE SAME ANSWER, which is the whole + of the fix: `_name_word_beside` used to walk the run itself, so a + name holding a run of n connectives walked it n times and the + stage went quadratic in the run's length -- measured 2026-09-20, + `"Josep " + "i " * n + "Rovira"` grew 3.8x per doubling against + the 2.0x every other shape holds, 59ms at n=800. Two linear + passes answer the same question once for the whole segment. + `tests/v2/test_benchmark.py`'s `link_run` shape is the guard. + + `_is_conj_piece` is asked ONCE per piece, into a list the second + pass then reads: asking it in both passes would double the calls, + and it is the only per-piece call this builder makes. Recorded in + the FIRST pass rather than by a comprehension of its own -- which + is a code object on 3.11 and a bytecode saving here rather than a + frame one, measured: the profile hook emits no call event for it. + + WHAT IT COSTS A SHORT NAME, because answering for the whole + segment is not free where the walk would have stopped at once: + this call, plus `_is_conj_piece` for the pieces the walk never + reached. Measured 2026-09-20 against b9ed1429 -- `Josep Carod i + Rovira` 304 -> 307 frames (one call and two more `_is_conj_piece` + over its four pieces) and `Jane Doe nee Puig i Soler` 307 -> 312. + An O(1) rise per link-bearing name against an unbounded saving: + the same name with a run of 64 links goes 6,741 -> 2,652. + `tools/perf/call_count.py` is unmoved (parse=406.00, + facade=443.00) -- its reference name carries no link -- and so are + `John Smith`, `Smith, John`, `Juan Garcia y Lopez` and `Jane Doe + nee Smith`, none of which reaches this at all. + + Called only where a generational connective was found in the + segment (the `frozen` loop) or where a clause's walk reached a + suffix piece (`_link_joins_inside_the_clause`), so no ordinary + name pays for it at all. + + Mutation-checked 2026-09-20, each arm by a NAMED test: reading + the left array on both sides fails + test_a_connective_with_nothing_to_its_right_does_not_join, the + right array on both sides fails + test_a_leading_title_on_the_left_is_no_name_word, and dropping + either pass's `not` fails + test_a_connective_piece_counts_toward_the_carve_outs_total. + SWAPPING the two arrays outright is an EQUIVALENT mutant and no + test fails: both callers ask for a name word on each side and + AND the two answers, so which array answers which side is not a + question the conjunction can see. + """ + n = len(pieces) + left = [-1] * n + right = [n] * n + conj = [False] * n + prev = -1 + for k in range(n): + left[k] = prev + is_conj = _is_conj_piece(pieces[k], ptags[k], tokens) + conj[k] = is_conj + if not is_conj: + prev = k + nxt = n + for k in range(n - 1, -1, -1): + right[k] = nxt + if not conj[k]: + nxt = k + return left, right + + # rules.md#P3: "a recognized connective joins its neighbors into one # name part, connective runs included — except a single-letter # connective in a three-word name, which stays a name word, and a @@ -718,7 +815,8 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], def _name_word_beside(k: int, step: int, lo: int, hi: int, pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], - tokens: Sequence[WorkToken]) -> bool: + tokens: Sequence[WorkToken], + beside: _Beside) -> bool: """Whether such a word stands on the `step` side of the connective piece at `k`. @@ -736,15 +834,23 @@ def _name_word_beside(k: int, step: int, lo: int, hi: int, Rovira'): the peel walks from the end and stops at the first name word, the title run from the front. - The walk steps over connectives because a RUN of them joins as + The answer steps over connectives because a RUN of them joins as one ('Carod i y Rovira'), so the word this rule is about is the first one past the run -- and where the run runs out ('Juan i e') - there is no name word on that side at all. + there is no name word on that side at all. `beside` is where that + stepping already happened: `_run_neighbours` walked every run once + for the whole segment, so this reads an index rather than walking + to it. + + A SENTINEL OUT OF RANGE is how "the run ran out" arrives -- -1 on + the left, `len(pieces)` on the right -- and the bound test below + is what turns it into False, exactly as it did when the walk ran + here and stopped at the same place. `lo` is never negative and + `hi` never past `len(pieces)`, so neither sentinel can pass it, + and the two piece tests are never asked about an index that is + not one. """ - j = k + step - while (0 <= j < len(pieces) - and _is_conj_piece(pieces[j], ptags[j], tokens)): - j += step + j = beside[0][k] if step < 0 else beside[1][k] return (lo <= j < hi and not is_suffix_piece(pieces[j], ptags[j], tokens) and not is_title_piece(pieces[j], ptags[j], tokens)) @@ -994,6 +1100,7 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # connective in a three-word name"). frozen: set[int] = set() lo = hi = -1 + beside: _Beside = ([], []) for k, piece in enumerate(pieces): tok = tokens[piece[0]] if (len(piece) != 1 @@ -1014,13 +1121,22 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # acronym (#397 second review). Asked once per # segment and only where such a connective was found, # so the cost is the chain's own and no ordinary name - # pays it. + # pays it -- which is the same gate the run memo + # below is built on, and for the same reason. hi = trailing_start_past_titles(lo, pieces, ptags, tokens, one_case=one_case) - if not (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens) + # ONE ANSWER PER RUN, built beside the two bounds and + # on the same gate: every member of a contiguous run + # of connectives has the same nearest name word on + # each side, and asking per member walked the run + # once per member -- quadratic in its length, 3.8x + # per doubling measured at `b9ed1429`. + beside = _run_neighbours(pieces, ptags, tokens) + if not (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens, + beside) and _name_word_beside(k, 1, lo, hi, pieces, ptags, - tokens)): + tokens, beside)): frozen.add(piece[0]) total = sum(_is_rootname(p, t, tokens) for p, t in zip(pieces, ptags) diff --git a/tests/v2/test_benchmark.py b/tests/v2/test_benchmark.py index 2d0c316a..73509781 100644 --- a/tests/v2/test_benchmark.py +++ b/tests/v2/test_benchmark.py @@ -215,6 +215,20 @@ def test_a_thousand_names_still_parse_in_reasonable_time( # words and reaches nothing: the peel # stops at the trailing marker, so the # ORDER inside the unit is the shape +# connective RUN LENGTH link_run ONLY -- P3's both-sides +# condition walks the run of connectives +# beside a link, and no other unit here +# puts two connectives in a row where one +# of them is GENERATIONAL vocabulary. +# 'and ' is a run too, but no member of it +# is in the class, so the frozen loop's +# body never runs and the walk is never +# entered; the unit needs MIXED CASE as +# well, since a one-case name reads the +# marked letter as an initial and the +# frozen loop declines it on the tag +# ('i und ' reaches nothing, 'i Und ' +# reaches everything) _SHAPES = { "delimiter_pairs": "(a) ", # extract: matched pairs -> masked spans "quote_pairs": '"a" ', # extract: the open==close path @@ -228,6 +242,7 @@ def test_a_thousand_names_still_parse_in_reasonable_time( "honorifics": "씨 ", # script_segment: the peel's site scan "bound_given": "abdul ", # group: the P5 reserve over every piece "maiden_clause": "nee MA ", # group: M2's view over the segment + "link_run": "i Und ", # group: P3's both-sides walk (#397) } _BASE = 800 @@ -273,6 +288,18 @@ def test_a_thousand_names_still_parse_in_reasonable_time( # signal is the strongest of the three quadratics on record and does # not decide the bound; the shape reads 4.05-4.12 at base 800 across # repeated runs, inside the clean column, and neither number moved. +# The thirteenth (link_run, #397 second review) arrived with its own +# quadratic in hand as well, and it is the one this shape was added +# FOR rather than one found by adding it: `_name_word_beside` walked +# the run of connectives beside a link once per MEMBER of that run, +# so at commit b9ed1429 the shape measures 8.48 at base 100, 10.29 at +# 200 and 12.11 at 400 -- outside the bound at every one of them, and +# the second-strongest signal on record. Answered once per run +# (`_group._run_neighbours`) it reads 4.04 at base 200, 4.02 at 400 +# and 4.02-4.19 at 800 across repeated runs, inside the clean column; +# neither number moved. The absolute cost is the shape's own price +# and is paid at the top of the clean range: 11.6ms at base 800 +# against 48.7ms at 3200. _MAX_RATIO = 6.0 @@ -464,3 +491,67 @@ def test_a_trailing_credential_run_does_not_cost_exponentially() -> None: f"input, where this tree measures 3.5x and the per-member memo " f"#531 first shipped measured 7.4x. Something in _assign.py's " f"trailing slot is asking a walk per member again (#531)") + + +# The clause caller of the same walk, and a THIRD instrument for the +# same reason the one above needed a second: `_SHAPES` repeats a unit +# and nothing else, so it cannot express "a name word, a marker, then +# a long run" -- and a maiden clause needs exactly that prefix. +# Measured: `"nee i Und " * n` never reaches the clause link exception +# at all (the take declines, and b9ed1429 and this tree measure the +# identical 3.92/4.07/4.12 on it), which is the silent-no-op a +# reachability probe exists to catch. So this guard builds its own +# input and counts FRAMES, which the walk is made of and which do not +# move under load. +# +# One pair, not two: the defect here is a quadratic and there is no +# exponential to order it against, so the 16-vs-64 pair is the whole +# guard. Measured 2026-09-20 through this file's own `_frames_for`: +# 876 frames at 16 and 2,652 at 64 on this tree (3.03x), against +# 1,125 and 6,741 at b9ed1429 (5.99x), where `_name_word_beside` +# walked the run once per member -- identical on three repeated runs +# at each end, frame counts being deterministic. +_CLAUSE_RUN_SMALL = 16 +_CLAUSE_RUN_LARGE = 64 +#: 3.03x measured here against 5.99x at b9ed1429: 4.5 sits ~1.5x over +#: the measurement and ~1.3x under the regression. Frame counts are +#: deterministic for a given tree and interpreter, so both margins are +#: for a future shape change rather than for runner noise. +_CLAUSE_RUN_MAX_RATIO = 4.5 + + +def _clause_run(members: int) -> str: + """A maiden clause whose birth name is a RUN of links. + + Mixed case on purpose: written wholly in one case the letter reads + as an initial (rules.md#P3's marked subset) and the clause's link + exception is never asked, so the guard would measure nothing. + """ + return "Jane Doe nee Puig " + "i " * members + "Soler" + + +def test_a_clause_link_run_does_not_cost_quadratically() -> None: + if sys.getprofile() is not None: + pytest.skip("a profile hook is already installed; this test owns it") + small_text = _clause_run(_CLAUSE_RUN_SMALL) + large_text = _clause_run(_CLAUSE_RUN_LARGE) + # REACHABILITY, the probe every shape in this file carries: the + # walk under measurement runs only while the clause KEEPS the run, + # which is rules.md#M2's link exception. End the clause at the + # first link instead and the guard measures a walk that no longer + # happens, at a comfortable ratio, forever. Asked at both sizes, + # the run length being what this varies. + for text, members in ((small_text, _CLAUSE_RUN_SMALL), + (large_text, _CLAUSE_RUN_LARGE)): + assert parse(text).maiden == " ".join( + ["Puig"] + ["i"] * members + ["Soler"]) + small = _frames_for(small_text) + large = _frames_for(large_text) + ratio = large / small + assert ratio < _CLAUSE_RUN_MAX_RATIO, ( + f"a clause holding {_CLAUSE_RUN_SMALL} links costs {small} frames " + f"and one holding {_CLAUSE_RUN_LARGE} costs {large} -- " + f"{ratio:.1f}x for 4x the input, where this tree measures 3.0x " + f"and the per-member walk at b9ed1429 measured 6.0x. " + f"_group.py's `_name_word_beside` is walking the run per member " + f"again (#397)") From 59d1443c01596c716e319e94e59b5a17f410263d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 20 Sep 2026 14:18:53 -0700 Subject: [PATCH 11/11] refactor(#397/#461): simplify pass over the link rule and the initials mark No behavior change: every reading over 157,188 names (the corpora, the case rows, the property grids and 146,398 generated names) under 24 lexicon x policy configurations, plus the facade and the no-parse paths -- 3,931,700 rows -- is byte-identical to 6048eb5d. Reference frames hold at 406 / 443, all eleven per-name probes are unchanged, and every run shape still doubles at 2x. - _group: `_link_joins_inside_the_clause` asks whether the piece is a connective before it fills the neighbour memo, so a clause ending at an ordinary credential never builds it ('Jane Doe nee Smith PhD' 346 -> 340 frames); `peel_start` moves down beside `lo`, so the clause's two bounds are computed and explained together. - _render: `_cap_word` names its guard (`generation`) and flattens to three arms under one `not generation`; the particle arm asks for a middle or family role, so the wider guard is equivalent. - _post_rules: one `is_conj` local, connectives collected in one place. - Comments across _group, _facade, _types compressed to what the code does, the measurement and the reason. - One comment claim corrected: 'PhD nee - i Jones' does not witness "a core is below `lo`" -- delimiter cores exist only in tail segments, so that dash is an ordinary piece and IS the name word that keeps the link (maiden '- i Jones', measured). Declined, recorded for a later round: folding both sides into one `_name_word_beside` call saves a frame per connective but moves frame figures quoted in test_benchmark.py and decisions.md, which this pass does not touch. Rejected by measurement: a shared two-sided helper (+1 frame per generational connective). Co-Authored-By: Claude Fable 5.1 --- nameparser/_facade.py | 28 ++- nameparser/_pipeline/_group.py | 261 ++++++++++++++-------------- nameparser/_pipeline/_post_rules.py | 20 +-- nameparser/_render.py | 23 ++- nameparser/_types.py | 5 +- 5 files changed, 167 insertions(+), 170 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index c9f219e4..e22e60cc 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -523,9 +523,9 @@ def _token_is_conjunction(self, tok: Token) -> bool: if UNCLASSIFIED_TAG in tok.tags: return _render._reads_as_conjunction(tok.text, self._lexicon) # #461: a connective with nothing in its part to join is not - # acting as one, and the parse decided that and marked the - # token -- the same mark the core's initials() reads, so the - # two views cannot disagree about it. + # acting as one. The parse decided that and marked the token, + # and this reads the same mark the core's initials() reads, so + # the two views cannot disagree about it. return ("conjunction" in tok.tags and UNJOINED_CONJUNCTION_TAG not in tok.tags) @@ -653,18 +653,16 @@ def _process_initial(self, name_part: str, # group that yields nothing for any other reason is still # dropped. # - # That third case survives the parse path no longer, and #461 - # is why: a connective with nothing to join is readmitted, so - # a group of a PARSED name cannot reach it. Any word that is - # neither a particle nor a connective initials, so a group - # reaching this is all particles and connectives; not being - # wholly particles it holds a connective; and that - # connective's part holds nothing but particles and - # connectives for it to join, so it is readmitted and the - # group yields it. Measured 2026-09-20, zero such groups over - # 95,119 names -- every corpus and case text plus the review's - # generated grid -- where "Vega, Santa de y" was the example - # until #461 and now initials 'S. y. V.'. + # No group of a PARSED name reaches that third case any more, + # and #461 is why: any word that is neither a particle nor a + # connective initials, so a group reaching it is all particles + # and connectives; not being wholly particles it holds a + # connective; and that connective's part holds nothing but + # particles and connectives for it to join, so the mark + # readmits it and the group yields it. Measured 2026-09-20, + # zero such groups over 95,119 names -- every corpus and case + # text plus the review's generated grid -- where "Vega, Santa + # de y" was the example until #461 and now initials 'S. y. V.'. # # What still reaches it is the paths with no parse to read, # where a connective is answered from the vocabulary and diff --git a/nameparser/_pipeline/_group.py b/nameparser/_pipeline/_group.py index 19f11873..b1f6e4a8 100644 --- a/nameparser/_pipeline/_group.py +++ b/nameparser/_pipeline/_group.py @@ -292,19 +292,9 @@ def _join_takes_the_member(view: Sequence[Sequence[int]], # reason, and the class test is that rule's own. The take runs BEFORE # every join, so the link is still a piece of its own here and the # question is asked of the pieces as classify left them -- the same -# inputs `_group_segment`'s `frozen` logic gives `_name_word_beside`, +# inputs `_group_segment`'s `frozen` loop gives `_name_word_beside`, # which is why this calls that predicate rather than restating the # class (mechanisms.md#ONE-PREDICATE-PER-QUESTION). -# -# The two halves of "also generational vocabulary" are already settled -# where this is called from: the walk asks `is_suffix_piece` first and -# only consults this when the answer was yes, so what is left to ask -# is the connective half, through the predicate that owns it. A lone -# link is therefore no link at all ('Jane Doe nee Puig i' keeps maiden -# 'Puig' and suffix 'i'), and neither is one standing before the -# generation or the credential a clause ends with ('... nee Puig i -# III', '... i MA'): `hi` is where assign's peel begins, so those -# stand at or past it and `_name_word_beside` refuses them by bound. def _link_joins_inside_the_clause(k: int, lo: int, hi: int, pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], @@ -313,22 +303,35 @@ def _link_joins_inside_the_clause(k: int, lo: int, hi: int, """Whether the suffix piece at `k` is a connective PLACED TO JOIN between two name words of the clause `lo`..`hi`. + The caller asks `is_suffix_piece` first and consults this only + where the answer was yes, so the generational half of "also + generational vocabulary" is settled and the connective half is + what is left to ask. A lone link is therefore no link at all + ('Jane Doe nee Puig i' keeps maiden 'Puig' and suffix 'i'), and + neither is one standing before the generation or the credential a + clause ends with ('... nee Puig i III', '... i MA'): `hi` is where + assign's peel begins, so those stand at or past it and + `_name_word_beside` refuses them by bound. + Defined here, beside its one caller, and forward-referencing the two predicates it is built out of: `_is_conj_piece` and `_name_word_beside` are the JOIN's, further down this module, and moving them up to meet this would say they belonged to the clause. - `beside` is the caller's memo cell, EMPTY until the first piece - that gets this far -- which is a suffix piece inside a clause, so - almost no name reaches it -- and filled here rather than at the - call site so the laziness costs no frame of its own. The walk - does not touch `pieces` or `ptags`, so one fill serves every - piece of it (#397 second review, the run fix).""" + `beside` is the caller's memo cell, filled on the first CONNECTIVE + piece the walk reaches rather than on the first suffix piece, so + a clause ending at an ordinary credential never builds it at all + and a clause holding a RUN of links builds it once ('Jane Doe nee + Puig i i i ... Soler'). Filled here rather than at the call site + so the laziness costs no frame of its own, and one fill serves + the whole walk because that walk mutates neither `pieces` nor + `ptags` (#397 second review, the run fix).""" + if not _is_conj_piece(pieces[k], ptags[k], tokens): + return False if not beside: beside.append(_run_neighbours(pieces, ptags, tokens)) - return (_is_conj_piece(pieces[k], ptags[k], tokens) - and _name_word_beside(k, -1, lo, hi, pieces, ptags, tokens, - beside[0]) + return (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens, + beside[0]) and _name_word_beside(k, 1, lo, hi, pieces, ptags, tokens, beside[0])) @@ -432,20 +435,6 @@ def _maiden_take(pieces: Sequence[Sequence[int]], rest = peel_walk(seen[m], ptags, skip) peeled = peel_trailing(rest, pieces, ptags, tokens, one_case) trailing = rest[-1] if peeled.numeral is not None else len(pieces) - # Where assign's trailing run begins over the pieces as WRITTEN: - # `trailing_start`'s whole answer, read off the peel pair it wraps - # rather than re-running that pair, which is the reading its own - # docstring sends this caller here for. Only the link exception in - # the walk below wants it, as the right bound of the clause -- the - # walk's own stop is `trailing`, and this is never past it: the - # numeral fork's `trailing` is the walk's LAST piece and the - # acronym fork's `stop` is a max over this one, so the two never - # disagree about where the clause ends, only about what the - # exception may reach across. Measured 2026-09-20 with a probe - # here over the whole suite -- 93,408 reaches of this site, - # `peel_start > trailing` 0 of them. - peel_start = (rest[peeled.names] if peeled.names < len(rest) - else len(pieces)) # The fork reads the piece before the numeral, and the take # REMOVES that piece: afterwards assign sees the piece before the # marker there, and if that is initial-shaped the fork will not @@ -610,21 +599,35 @@ def _maiden_take(pieces: Sequence[Sequence[int]], if m + run >= len(seen): return None # rules.md#M2: "a link inside the birth name does not end it" -- - # the clause's OWN bounds, which are not the segment's: `lo` is the - # first piece after the marker run, so the marker is never the name - # word on a link's left, and `peel_start` is where the trailing run - # begins, so the generation or credential a clause ends with is - # never the name word on its right ('... nee Puig i III', '... i - # MA', whose MA carries no `vocab:suffix` tag for the piece test to - # refuse it by). A core between the marker and that first word is - # below `lo` by construction and so cannot pass for the name word - # either ('PhD née - i Jones'). + # the clause's OWN bounds for the link exception, which are not the + # segment's. `lo` is the first piece after the marker run, so the + # marker is never the name word on a link's left, and a delimiter + # core between the marker and that first word is below `lo` by + # construction and cannot pass for one either. A core is the TAIL + # segment's alone (`extra_suffix_delimiters`, empty by default), so + # an ordinary dash is not one and does pass: 'PhD née - i Jones' + # keeps maiden '- i Jones' (measured 2026-09-20). + # `peel_start` is where assign's trailing run begins over + # the pieces as WRITTEN, so the generation or credential a clause + # ends with is never the name word on a link's right ('... nee Puig + # i III', '... i MA', whose MA carries no `vocab:suffix` tag for + # the piece test to refuse it by). + # + # `peel_start` is `trailing_start`'s whole answer, read off the + # peel pair above rather than re-running it, which is the reading + # that function's own docstring sends this caller here for. It is + # never past the walk's own stop, `trailing`: the numeral fork's + # `trailing` is the walk's LAST piece and the acronym fork's `stop` + # is a max over this one, so the two never disagree about where the + # clause ends, only about what the exception may reach across. + # Measured 2026-09-20 with a probe here over the whole suite -- + # 93,408 reaches of this site, `peel_start > trailing` 0 of them. lo = seen[m + run] + peel_start = (rest[peeled.names] if peeled.names < len(rest) + else len(pieces)) j = m + run - # The link exception's memo cell, filled on first use inside the - # predicate: an ordinary clause never reaches it, and a clause - # holding a RUN of links asks about the run once instead of once - # per member ('Jane Doe nee Puig i i i ... Soler'). + # The link exception's memo cell, filled inside the predicate on + # the first connective it is asked about (see its docstring). beside: list[_Beside] = [] while (j < len(seen) and seen[j] < trailing and (not is_suffix_piece(pieces[seen[j]], ptags[seen[j]], @@ -727,20 +730,13 @@ def _run_neighbours(pieces: Sequence[Sequence[int]], Called only where a generational connective was found in the segment (the `frozen` loop) or where a clause's walk reached a - suffix piece (`_link_joins_inside_the_clause`), so no ordinary - name pays for it at all. - - Mutation-checked 2026-09-20, each arm by a NAMED test: reading - the left array on both sides fails - test_a_connective_with_nothing_to_its_right_does_not_join, the - right array on both sides fails - test_a_leading_title_on_the_left_is_no_name_word, and dropping - either pass's `not` fails - test_a_connective_piece_counts_toward_the_carve_outs_total. - SWAPPING the two arrays outright is an EQUIVALENT mutant and no - test fails: both callers ask for a name word on each side and - AND the two answers, so which array answers which side is not a - question the conjunction can see. + CONNECTIVE suffix piece (`_link_joins_inside_the_clause`), so no + ordinary name pays for it at all. + + Dropping either pass's `not` fails + test_a_connective_piece_counts_toward_the_carve_outs_total + (mutation-checked 2026-09-20; how the two arrays are READ is + checked in `_name_word_beside`, which reads them). """ n = len(pieces) left = [-1] * n @@ -779,16 +775,15 @@ def _is_rootname(piece: Sequence[int], ptags: Set[str], return False # rules.md#P3: "A connective counts as a name word wherever this # rule counts them, whatever else the vocabulary says the word is" - # (#397). The order against the `initial` test above decides + # (#397). The order against the `initial` refusal above decides # nothing, and what makes that safe lives in classify rather than # here: for a single letter it writes `initial` or `conjunction` # and never both, so a one-case `I`/`i` arrives with no conjunction - # tag whichever test runs first. Measured -- hoisting this arm - # above the `initial` refusal moves no field, report or initial on - # any corpus name under three name orders, and no test. The - # refusal keeps its place as the older and narrower of the two. - # INLINE rather than a call to _is_conj_piece: this runs once per - # piece of every name (frame budget). + # tag whichever test runs first -- measured, hoisting this arm + # above the refusal moves no field, report or initial on any + # corpus name under three name orders, and no test. INLINE rather + # than a call to _is_conj_piece: this runs once per piece of every + # name (frame budget). if ("conjunction" in ptags or (len(piece) == 1 and "conjunction" in tokens[piece[0]].tags)): return True @@ -840,15 +835,23 @@ def _name_word_beside(k: int, step: int, lo: int, hi: int, there is no name word on that side at all. `beside` is where that stepping already happened: `_run_neighbours` walked every run once for the whole segment, so this reads an index rather than walking - to it. - - A SENTINEL OUT OF RANGE is how "the run ran out" arrives -- -1 on - the left, `len(pieces)` on the right -- and the bound test below - is what turns it into False, exactly as it did when the walk ran + to it. A SENTINEL OUT OF RANGE is how "the run ran out" arrives -- + -1 on the left, `len(pieces)` on the right -- and the bound test + below turns it into False, exactly as the walk did when it ran here and stopped at the same place. `lo` is never negative and - `hi` never past `len(pieces)`, so neither sentinel can pass it, - and the two piece tests are never asked about an index that is - not one. + `hi` never past `len(pieces)`, so neither sentinel can pass the + bound, and the two piece tests are never asked about an index + that is not one. + + Mutation-checked 2026-09-20, each side by a NAMED test: reading + the left array for both sides fails + test_a_connective_with_nothing_to_its_right_does_not_join, and + the right array for both fails + test_a_leading_title_on_the_left_is_no_name_word. SWAPPING the + two arrays outright is an EQUIVALENT mutant and no test fails: + both callers ask for a name word on each side and AND the two + answers, so which array answers which side is not a question the + conjunction can see. """ j = beside[0][k] if step < 0 else beside[1][k] return (lo <= j < hi @@ -1034,13 +1037,15 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), if len(pieces) + additional >= 3: # rules.md#P3: "A connective that is also generational # vocabulary joins only where a name word stands on each side - # of it" (#397, restated by its review). `frozen` holds the - # TOKEN index of every such connective that has no name word - # on one side or the other. It is joining nothing, so it is - # the generation it also spells: no join of its own reaches - # it, it may not merge into a run, and it counts toward the - # carve-out total the way the generation counted -- which is - # not at all, a suffix piece being no rootname. + # of it" (#397). `frozen` holds the TOKEN index of every such + # connective missing that name word on one side or the other. + # It is joining nothing, so it is the generation it also + # spells: no join of its own reaches it, it may not merge into + # a run, and it counts toward the carve-out total the way the + # generation counted -- not at all, a suffix piece being no + # rootname. A TOKEN index for the same reason the chain's + # trailing run is a length from the end: the merges below move + # piece indices and cannot move this one. # # "No join of its own" is the whole claim, and a NEIGHBOUR's # join can still absorb it: the two loops below skip a frozen @@ -1048,51 +1053,41 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # span another connective's join takes. 'Josep Carod Rovira # Puig y i' freezes the trailing 'i' -- nothing stands on its # right -- and the 'y' beside it joins across it all the same, - # for family 'Puig y i', which is the parent's reading of the - # same name and 'y i' is what the parent read there too. - # Pinned by test_a_frozen_link_is_still_absorbed_by_a_ - # neighbours_join. Freezing it is not a claim that the word - # cannot move; it is a claim about which joins this loop - # licenses. + # for family 'Puig y i', which is what the parent read there + # too. Pinned by test_a_frozen_link_is_still_absorbed_by_a_ + # neighbours_join. # # Asked HERE, of the pieces as classify left them, and of the - # NEIGHBOURS' class rather than of the connective's position. - # Position was the first cut and it tested the wrong thing: - # any piece on each side passed it, so a generational suffix - # standing behind the link was swallowed into the name - # ("Josep Lluis Carod i III" read family 'Carod i III'). And - # the question cannot be re-asked further down, because a - # merge answers it: in the part a join produced, the absorbed - # suffix IS the word standing on the right. - # - # A token index rather than a piece index for the same reason - # the chain's trailing run is a length from the end: the - # merges below move piece indices and cannot move this one. + # NEIGHBOURS' class rather than of the connective's position: + # any piece on each side passes a position test, so a + # generational suffix standing behind the link was swallowed + # into the name ("Josep Lluis Carod i III" read family 'Carod + # i III'). It cannot be re-asked further down either, because + # a merge answers it -- in the part a join produced, the + # absorbed suffix IS the word standing on the right. # # Nothing but a connective of the suffix vocabulary reaches # the body, so a name that has none pays tag lookups and no # call at all. # - # NO LENGTH TEST, and that is the rule's own scope rather than - # an omission: the clause quoted above names a CLASS -- "a - # connective that is also generational vocabulary" -- and says - # nothing about how the word is spelled. A `len(tok.text) != 1` - # stood here and narrowed it to one-letter connectives, - # untested and undocumented, and it was caller-reachable: under + # NO LENGTH TEST, which is the rule's own scope rather than an + # omission: the clause quoted above names a CLASS and says + # nothing about how the word is spelled. Narrowing it to + # one-letter connectives was caller-reachable -- under # `Lexicon.default().add(conjunctions={"og"}, suffix_words= - # {"og"})`, 'John Quincy Smith og' read family 'Smith og' with - # the test and reads family 'Smith' plus suffix 'og' without it - # -- which is the answer the rule states (#397 second review). - # Dropping it is byte-identical over every oracle in use -- - # fields, reports, `initials()`, `capitalized()` plain and - # forced, and every token's role -- across 359,053 parses - # measured 2026-09-20: the 351,400-parse review grid under - # eight lexicon/policy/locale configurations, the 2,175-parse - # sweep of tests/v2/cases.py under three orders, and the - # 5,478-parse sweep of the differential corpora under six. - # It has to be: `i` is the ONLY member of the class in the - # default vocabulary and in every locale pack, and it is one - # letter. Pinned by test_a_multi_letter_link_of_the_suffix_ + # {"og"})`, 'John Quincy Smith og' reads family 'Smith' plus + # suffix 'og', the answer the rule states, and read family + # 'Smith og' while a `len(tok.text) != 1` stood here (#397 + # second review). Nothing SHIPPED can witness it, `i` being + # the only member of the class in the default vocabulary and + # in every locale pack and one letter long -- measured rather + # than assumed: byte-identical (fields, reports, `initials()`, + # `capitalized()` plain and forced, every token's role) across + # 359,053 parses on 2026-09-20, the 351,400-parse review grid + # under eight lexicon/policy/locale configurations, the + # 2,175-parse sweep of tests/v2/cases.py under three orders + # and the 5,478-parse sweep of the differential corpora under + # six. Pinned by test_a_multi_letter_link_of_the_suffix_ # vocabulary_joins_by_the_same_rule. # # The three-word carve-out below stays single-letter, because @@ -1107,6 +1102,9 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), or "conjunction" not in tok.tags or "vocab:suffix" not in tok.tags): continue + # The bounds and the run memo, computed once per segment + # and only where such a connective was found, so the cost + # is the link's own and no ordinary name pays any of it. if hi < 0: lo = leading_titles(pieces, ptags, tokens) # H5's reading and not the peel over the pieces as @@ -1114,24 +1112,19 @@ def merge(lo: int, hi: int, add: Set[str] = frozenset(), # hides it from `trailing_start`, which then answers # `len(pieces)` and hands this loop a credential as # the name word on the link's right. 'John Quincy - # Adams i MA Prof.' joined to family 'Adams i MA' - # with no report at all, where 'John Quincy Adams i - # MA' -- the same name, one title shorter -- reads - # family 'Adams', suffix 'i MA' and reports the - # acronym (#397 second review). Asked once per - # segment and only where such a connective was found, - # so the cost is the chain's own and no ordinary name - # pays it -- which is the same gate the run memo - # below is built on, and for the same reason. + # Adams i MA Prof.' joined to family 'Adams i MA' with + # no report at all, where 'John Quincy Adams i MA' -- + # the same name, one title shorter -- reads family + # 'Adams', suffix 'i MA' and reports the acronym + # (#397 second review). hi = trailing_start_past_titles(lo, pieces, ptags, tokens, one_case=one_case) - # ONE ANSWER PER RUN, built beside the two bounds and - # on the same gate: every member of a contiguous run + # ONE ANSWER PER RUN: every member of a contiguous run # of connectives has the same nearest name word on - # each side, and asking per member walked the run - # once per member -- quadratic in its length, 3.8x - # per doubling measured at `b9ed1429`. + # each side, and asking per member walked the run once + # per member -- quadratic in its length, 3.8x per + # doubling measured at `b9ed1429`. beside = _run_neighbours(pieces, ptags, tokens) if not (_name_word_beside(k, -1, lo, hi, pieces, ptags, tokens, beside) diff --git a/nameparser/_pipeline/_post_rules.py b/nameparser/_pipeline/_post_rules.py index 53f9ee3c..b9c5d1fb 100644 --- a/nameparser/_pipeline/_post_rules.py +++ b/nameparser/_pipeline/_post_rules.py @@ -853,25 +853,25 @@ def post_rules(state: ParseState) -> ParseState: # ONE walk decides both marks -- a plain loop, not two generator # expressions: this runs per role on every parse and a genexp # costs a frame of its own (frame budget). An EMPTY role needs no - # guard of its own: every arm below iterates the part, so the walk - # is inert on one, and a `continue` ahead of it survived the whole - # suite and moved neither frame count (measured 2026-09-20, - # py3.11: parse=406.00 facade=443.00 with it and without it). + # `continue` ahead of the walk, which is inert on one: measured + # 2026-09-20, py3.11, one survived the whole suite and moved + # neither frame count (parse=406.00 facade=443.00 either way). + # `bool(part)` says the same thing about the R2 arm as + # `_types._remarked`'s twin of this walk does. for role in (Role.GIVEN, Role.MIDDLE, Role.FAMILY): part = _idx(tokens, role) - all_particle = True + all_particle = bool(part) conj: list[int] = [] others = 0 for i in part: tags = tokens[i].tags + is_conj = "conjunction" in tags + if is_conj: + conj.append(i) if "particle" not in tags: all_particle = False - if "conjunction" in tags: - conj.append(i) - else: + if not is_conj: others += 1 - elif "conjunction" in tags: - conj.append(i) if all_particle: for i in part: tokens[i] = dataclasses.replace( diff --git a/nameparser/_render.py b/nameparser/_render.py index 238948f5..1ae68c43 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -225,12 +225,11 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # unjoined mark has turned into ordinary name words" -- so a # conjunction keeps conjunction treatment even inside a part the # mark has turned into ordinary name words. - # The GENERATION test is the other half of that sentence, and it - # is why a single conjunct stands ahead of both conjunction arms: + # The `generation` guard below is the other half of that sentence: # a word this vocabulary holds can ALSO be the generation it # spells ('i' is the Catalan link and the roman numeral), and # where the parse read the generation the token still carries the - # `conjunction` tag classify gave it -- so without the test, + # `conjunction` tag classify gave it -- so without the guard, # `parse("John Quincy Smith i").capitalized(force=True)` gave # 'John Quincy Smith i' where every release through 2.3 gave # 'John Quincy Smith I' (#397 review). Such a token is repaired @@ -251,6 +250,10 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # record of the vocabulary half, so the pair reads two decisions # the parse already made and re-derives neither # (mechanisms.md#RENDER-HONORS-THE-PARSE). + # It guards the whole test rather than the two conjunction arms + # alone, which reads as the wider claim and is not one: the + # particle arm asks for role MIDDLE or FAMILY, so a SUFFIX-roled + # token can never reach it either way. # No SHIPPED name witnesses the difference: `particles` and # `conjunctions` are disjoint in the default vocabulary and in # every locale pack, so no shipped conjunction can sit in an @@ -305,12 +308,14 @@ def _cap_word(word: str, role: Role, tags: frozenset[str], # whose tags it keeps on purpose, and keying this on `span is None` # overrode them -- `revise(middle='e-f')` repaired to 'e-F' where # the same words parsed gave 'E-F' (#463 review). - if ((normalized in lex.particles and role in (Role.MIDDLE, Role.FAMILY) - and UNJOINED_TAG not in tags) - or (not (role is Role.SUFFIX and "vocab:suffix" in tags) - and ("conjunction" in tags - or (UNCLASSIFIED_TAG in tags - and _reads_as_conjunction(word, lex))))): + generation = role is Role.SUFFIX and "vocab:suffix" in tags + if not generation and ( + (normalized in lex.particles + and role in (Role.MIDDLE, Role.FAMILY) + and UNJOINED_TAG not in tags) + or "conjunction" in tags + or (UNCLASSIFIED_TAG in tags + and _reads_as_conjunction(word, lex))): return word.lower() # v1 cap_word tries the edge-stripped form, then the period-free # form ('Ph.D.' -> 'ph.d' -> 'phd' hits the exceptions map) diff --git a/nameparser/_types.py b/nameparser/_types.py index da868c60..de7d4386 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -757,14 +757,15 @@ def _remarked(tokens: list[Token]) -> tuple[Token, ...]: elif not alone and UNJOINED_TAG in tags: out[i] = dataclasses.replace(out[i], tags=tags - {UNJOINED_TAG}) + # re-read: the arm above may have replaced the token tags = out[i].tags mark = lone_conj and "conjunction" in tags if mark and UNJOINED_CONJUNCTION_TAG not in tags: out[i] = dataclasses.replace( out[i], tags=tags | {UNJOINED_CONJUNCTION_TAG}) elif not mark and UNJOINED_CONJUNCTION_TAG in tags: - out[i] = dataclasses.replace(out[i], - tags=tags - {UNJOINED_CONJUNCTION_TAG}) + out[i] = dataclasses.replace( + out[i], tags=tags - {UNJOINED_CONJUNCTION_TAG}) return tuple(out)