From 919644be50fe55478e965a31270105244ea114fa Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 13 Sep 2026 21:17:16 -0700 Subject: [PATCH 01/11] fix(#528): HumanName.initials() reads the parse's connective tags The v1 facade decided "is this word the connective" from the lexicon plus v1's initial shape on the part's raw text, where the core's initials() reads the tag the parse recorded. #383/#479 gave rules.md#P3 a fork the shape cannot see -- in a name written wholly in one case a marked 'e' is an initial and an unmarked 'y' joins -- so the two views of one parse disagreed about the same letter. _list_for becomes one walk with two views: _list_tokens_for yields each list element's backing tokens, and the string view is built from it, so the element boundaries the walk exists to hold (a "joined" continuation, a folded middle) cannot drift between them. _process_initial keeps its v1 signature for direct callers and gains an optional `tokens`; with tokens each word asks the tag, falling back to _render._reads_as_conjunction -- case repair's own R4 helper -- for a token carrying UNCLASSIFIED_TAG, which is text a parse never read. _is_conjunction goes with its only caller. Measured over the 1174-name corpus glob: six names moved and the two views now agree on every name but 'Ph. D., John', whose divergence is the pre-existing Ph. D. merge. 'john e smith' gives j. e. s. where 1.4.0 through 2.3.0 gave j. s.; 'JUAN Y GARCIA' gives J. G. where they gave J. Y. G. Accepted cost, the R4 shape: for a word backed by a token the answer is fixed at parse time, so a C.conjunctions edit takes effect on the next full_name assignment, as capitalize()'s already did. An unpickled name is spliced text throughout and takes the vocabulary fallback, which capitalize() has done since __setstate__ stamped the tag. And a stated private-method break: a v1-shaped subclass overriding _process_initial(self, name_part, firstname=False) now raises TypeError from initials(), since initials() passes tokens= -- such an override must accept it. Pinned by a test and named in the method. Ledgers: one new _initials rule at 1.4.0 over the four names whose roles do not move -- 'john e smith', 'john e jones', 'jones, john e', 'JUAN Y GARCIA' -- written ahead of fix(initials-per-word)'s connective rule, whose equal `fields` make file order the whole decision on the two names both reach; _CROSS_RULE_WINNERS and _RECORDED_DIFFS pin that. At 2.0.0-2.2.0 fix(#462)'s two-cause paragraph becomes single-cause, the facade half having closed; at 2.3.0 both surfaces now move together. The prose that said the facade does not follow the tags is corrected where the sweep found it, tests/v2/cases.py's two #383/#479 notes included. Five gates at 0 unexplained. Co-Authored-By: Claude Fable 5.1 --- nameparser/_facade.py | 152 +++++++++++---- nameparser/_render.py | 23 ++- tests/test_capitalization.py | 21 +- tests/test_initials.py | 23 ++- tests/v2/cases.py | 25 +-- tests/v2/test_facade.py | 190 ++++++++++++++++++- tests/v2/test_ledger_guards.py | 87 ++++++++- tests/v2/test_render.py | 46 ++--- tools/differential/compare.py | 11 ++ tools/differential/expected_since_1.4.0.toml | 94 +++++++-- tools/differential/expected_since_2.0.0.toml | 52 ++--- tools/differential/expected_since_2.1.0.toml | 27 +-- tools/differential/expected_since_2.2.0.toml | 27 +-- tools/differential/expected_since_2.3.0.toml | 28 ++- 14 files changed, 640 insertions(+), 166 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 3ad9bac4..4c2a6b09 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -345,26 +345,39 @@ def _set_field(self, member: str, value: str | list[str] | None) -> None: self._parsed = self._parsed.replace( **{_V2_FIELD.get(member, member): joined}) - def _list_for(self, member: str) -> list[str]: + def _list_tokens_for(self, member: str) -> list[tuple[Token, ...]]: # A "joined" continuation token ("Ph." + "D.") belongs to its # predecessor's part, matching v1's fix_phd (suffix_list had ONE # "Ph. D." element). ParsedName._text_for heals only the suffix # string view (the ", " join); the facade list view heals for # every role -- a continuation is never its own list element. role = Role(_V2_FIELD.get(member, member)) - parts: list[str] = [] - folded: list[str] = [] + parts: list[list[Token]] = [] + folded: list[list[Token]] = [] for tok in self._parsed.tokens_for(role): if "joined" in tok.tags and parts: - parts[-1] += " " + tok.text + parts[-1].append(tok) elif FOLDED_TAG in tok.tags: # middle_as_family fold: v1 PREPENDED middle_list to # last_list -- keep the list view consistent with the # string view (_text_for orders folded-first too) - folded.append(tok.text) + folded.append([tok]) else: - parts.append(tok.text) - return folded + parts + parts.append([tok]) + return [tuple(group) for group in folded + parts] + + def _list_for(self, member: str) -> list[str]: + # The STRING view of the walk above, which is the v1 `*_list` + # shape. Two views off one walk rather than two walks: the + # initials view needs each element's backing tokens (#528) and + # every other reader needs its text, and a second walk could + # drift from this one on exactly the element boundaries the + # comment above exists to hold. Not on the parse path -- + # measured 2026-09-13, HumanName(name) never reaches it -- so + # the tuple building is off the benchmarked budget + # (tests/v2/test_benchmark.py budgets parse() and HumanName()). + return [" ".join(tok.text for tok in group) + for group in self._list_tokens_for(member)] @property def title(self) -> str: @@ -472,9 +485,37 @@ def _is_particle(self, text: str) -> bool: self._resolve() return _normalize(text) in self._lexicon.particles - def _is_conjunction(self, text: str) -> bool: + def _token_is_conjunction(self, tok: Token) -> bool: + # #528: the PARSE's answer, not the vocabulary's. A token the + # parser classified carries its reading in its tags, which is + # the source the core's initials() has always read + # (mechanisms.md#RENDER-HONORS-THE-PARSE), so a bare capital + # 'Y' in a one-case name contributes no initial where its tag + # says connective, and a one-case 'e' contributes one where its + # tag says initial. Before #528 this was computed from the raw + # word instead -- "in the conjunctions set AND NOT + # _render._INITIAL" (v1's is_conjunction, restored by #462) -- + # a shape test standing in for a tag, which stopped agreeing + # with the parse the moment #383/#479 gave the classifier a + # fork the shape cannot see. + # + # UNCLASSIFIED_TAG is the one case with no parse to honor: the + # words were spliced into a field as raw text, by `hn.middle = + # ...` (ParsedName.replace) or by the v1 pickle load in + # __setstate__. They carry no reading, so the vocabulary + # answers -- the same fallback _render._cap_word takes for the + # same tokens and through the same helper, which is why the + # helper is imported rather than the predicate rewritten + # (decisions.md#R4 for why one question and not two: whether a + # word is a connective is a fact the word can answer alone, + # whether a particle is acting as one is a fact about the part). + # + # _resolve() first, as _is_particle above does: an unpickled + # instance has no _lexicon until resolved. self._resolve() - return _normalize(text) in self._lexicon.conjunctions + if UNCLASSIFIED_TAG in tok.tags: + return _render._reads_as_conjunction(tok.text, self._lexicon) + return "conjunction" in tok.tags def _split_last(self) -> tuple[list[str], list[str]]: # rules.md#R2: "a name part whose every word is particle @@ -511,34 +552,55 @@ def last_base(self) -> str: # -- initials ------------------------------------------------------------- - def _process_initial(self, name_part: str, firstname: bool = False) -> str: + def _process_initial(self, name_part: str, + firstname: bool = False, + tokens: tuple[Token, ...] | None = None) -> str: # after v1 parser.py:427, not verbatim: particles and # conjunctions are filtered from initials unless the part is a - # first name. split() rather than split(" ") because split(" ") - # yields '' between repeated spaces and `part[0]` below would - # raise IndexError on it (#232). v1 stated the reason as - # `*_list` attributes bypassing whitespace normalization, which - # no longer holds -- the `*_list` properties are read-only in - # 2.x, and assignment through `hn.middle = ...` normalizes -- - # but a doubled space anywhere in a part still reaches here. - parts = name_part.split() + # first name. + # + # TWO WAYS IN. `tokens` is the part's backing tokens, which + # _initials_lists always has and passes; `name_part` is v1's + # signature, kept because subclasses and tests call this + # directly with a string (tests/test_initials.py), and there + # the words come from splitting it. `tokens` supersedes + # `name_part` entirely when given -- the words are the tokens' + # own text rather than a re-split of the joined element, so a + # two-word element and its tokens cannot fall out of step. + # STATED BREAK: _initials_lists always calls with `tokens=`, so + # a subclass overriding with v1's two-argument signature + # (name_part, firstname=False) now raises TypeError the first + # time initials() runs, rather than being silently skipped. The + # alternative -- a string wrapper kept over a token core -- + # would make such an override silently ineffective instead, + # which hides the override rather than breaking it loudly. + # split() rather than split(" ") because split(" ") yields '' + # between repeated spaces and `word[0]` below would raise + # IndexError on it (#232). v1 stated the reason as `*_list` + # attributes bypassing whitespace normalization, which no + # longer holds -- the `*_list` properties are read-only in 2.x, + # and assignment through `hn.middle = ...` normalizes -- but a + # doubled space anywhere in a part still reaches here. + # + # Particles are NOT decided per token: _is_particle stays a + # live vocabulary lookup, as _render._cap_word keeps it -- + # rules.md#R4 draws this boundary per question, not per field. + self._resolve() + if tokens is None: + words: tuple[str, ...] = tuple(name_part.split()) + # No parse read this text, so every word takes the same + # fallback _token_is_conjunction takes for a spliced one. + conjunctions: tuple[bool, ...] = tuple( + _render._reads_as_conjunction(word, self._lexicon) + for word in words) + else: + words = tuple(tok.text for tok in tokens) + conjunctions = tuple(self._token_is_conjunction(tok) + for tok in tokens) initials = [] - for part in parts: - # v1 parser.py:771 (1.4.0): is_conjunction was "in the - # conjunctions set AND NOT is_an_initial", so a dotted or - # bare-capital E/Y is the initial it looks like rather - # than the connective. The 2.0 facade dropped that half - # and lost the middle initial of 'Scott E. Werner' (#462). - # _render._INITIAL is v1's `initial` shape, kept in step - # with the pipeline's copy by tests/v2/test_regex_sync.py; - # the facade may import _render but not _pipeline - # (tests/v2/test_layering.py). Scoped here rather than in - # _is_conjunction: this is the only caller, and a future - # one should not inherit a decision made for initials. - conjunction = (self._is_conjunction(part) - and not _render._INITIAL.fullmatch(part)) - if not (self._is_particle(part) or conjunction) or firstname: - initials.append(part[0]) + for word, conjunction in zip(words, conjunctions): + if not (self._is_particle(word) or conjunction) or firstname: + initials.append(word[0]) if len(initials) > 0: return self.initials_separator.join(initials) # Return '' (never empty_attribute_default, which may be None) @@ -556,12 +618,18 @@ def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: strings -- except a part that is wholly PARTICLES, whose words initial as ordinary name words since #404, so the prefix-only middle name "de la" is no longer an example of the dropping. + + Each group is walked as TOKENS rather than as the strings of + the `*_list` view (#528), so every word carries the reading the + parse gave it; the elements are the list view's own, folded + first and continuations merged, because one walk builds both. """ - def group_initials(names: list[str], - firstname: bool = False) -> list[str]: - got = [i for i in (self._process_initial(n, firstname) - for n in names if n) if i] - words = [w for n in names if n for w in n.split()] + def group_initials(groups: list[tuple[Token, ...]], + firstname: bool = False) -> list[str]: + got = [i for i in ( + self._process_initial("", firstname=firstname, tokens=group) + for group in groups) if i] + words = [tok.text for group in groups for tok in group] if got or not words or not all(self._is_particle(w) for w in words): return got @@ -579,9 +647,9 @@ def group_initials(names: list[str], # already applies the same guard to the base, which is why # last_base was never empty here. return [w[0] for w in words] - return (group_initials(self.first_list, True), - group_initials(self.middle_list), - group_initials(self.last_list)) + return (group_initials(self._list_tokens_for("first"), True), + group_initials(self._list_tokens_for("middle")), + group_initials(self._list_tokens_for("last"))) def initials_list(self) -> list[str]: first, middle, last = self._initials_lists() diff --git a/nameparser/_render.py b/nameparser/_render.py index bf3786ca..c418c65e 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -58,7 +58,16 @@ # caller-added CJK conjunction spliced into a field, since no shipped # vocabulary carries one, and it costs nothing there: CJK is caseless, # so the carve-out's lower() and the fall-through's capitalize() return -# the same string, and case repair is now this pattern's only reader. +# the same string. Since #528 this pattern is read by more than case +# repair: through _reads_as_conjunction below, asking it only about +# text no parse read -- case repair's spliced field, and the v1 +# facade's initials view, both where a token carries UNCLASSIFIED_TAG +# (_facade._token_is_conjunction) and where there is no token at all +# (_process_initial's bare-string path, a direct call with no parse +# behind it). For initials the CJK divergence would decide whether +# such a spliced connective contributes a letter rather than which +# case it renders in -- still unreachable from any shipped vocabulary, +# and still not worth the import layering forbids. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") @@ -68,10 +77,14 @@ def _reads_as_conjunction(word: str, lex: Lexicon) -> bool: A token the parse classified carries its reading in its tags and this is not consulted. A token carrying UNCLASSIFIED_TAG was spliced into a field as raw text -- by replace(), or by the - facade's v1 pickle load -- and carries no reading, so case repair - falls back to the vocabulary, which gives the answer the parser - would have given, the initial carve-out included ('E.' assigned to - middle is an initial, not the Italian conjunction). + facade's v1 pickle load -- and carries no reading, so the two + views that hold a vocabulary fall back to this: case repair, and + since #528 the v1 facade's initials view. It gives the answer the + parser would have given, the initial carve-out included ('E.' + assigned to middle is an initial, not the Italian conjunction). + What it cannot give is an answer the parse reached by looking at + the whole NAME -- rules.md#P3's one-case fork is the live example + -- which is why it is the fallback and the tags are the rule. """ return bool(_normalize(word) in lex.conjunctions and not _INITIAL.fullmatch(word)) diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 0ddb85af..4d8c49a4 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -401,14 +401,19 @@ def test_a_restored_pickle_keeps_v1_conjunction_repair(self) -> None: restored.capitalize(force=True) self.m(str(restored), want, restored) - # The ONE name a pickle round trip does change, pinned so it is not - # rediscovered as a bug. #458 moved the conjunction-versus-initial - # decision into the parse, and a pickle carries no tags, so the - # restored name is repaired the way 1.4.0 repaired everything -- - # per word of the text, giving the Italian conjunction inside a - # hyphenated middle name. It is the pickle contract (strings only, - # never a re-parse) meeting the tag read, not a defect in either. - # 1.4.0 gave 'Juan e-F Smith' both ways. + # One of a CLASS of names a pickle round trip changes, pinned so it + # is not rediscovered as a bug. #458 moved the conjunction-versus- + # initial decision into the parse, and a pickle carries no tags, so + # the restored name is repaired the way 1.4.0 repaired everything + # -- per word of the text, giving the Italian conjunction inside a + # hyphenated middle name here. It is the pickle contract (strings + # only, never a re-parse) meeting the tag read, not a defect in + # either. 1.4.0 gave 'Juan e-F Smith' both ways. Since #383/#479 + # the one-case fork widened this class: 'JUAN Y GARCIA' and 'john e + # smith' also diverge on a round trip now, for the same reason and + # through the same fallback, whether the reader is capitalize() or + # (since #528) initials() -- pinned at + # tests/v2/test_facade.py::test_initials_of_an_unpickled_name_ask_the_vocabulary_too. def test_a_pickle_round_trip_loses_the_e_f_reading(self) -> None: direct = HumanName('juan e-f smith') direct.capitalize(force=True) diff --git a/tests/test_initials.py b/tests/test_initials.py index 89d10969..ab4095b6 100644 --- a/tests/test_initials.py +++ b/tests/test_initials.py @@ -260,13 +260,22 @@ def test_initials_drop_a_bare_non_ascii_conjunction_letter(self) -> None: hn = HumanName("Хосе Мария И Сантос") self.m(hn.initials(), "Х. М. С.", hn) - def test_initials_still_drop_a_lowercase_conjunction(self) -> None: - # the boundary #462 leaves alone: a bare lowercase e/y IS the - # connective, and 1.4.0 and 2.x agree -- true of this facade - # surface only since #383/#479: the core's parse().initials() - # now reads a one-case 'e' as an initial instead - # (tests/v2/test_render.py::test_facade_initials_do_not_yet_follow_the_one_case_fork) + def test_initials_follow_the_one_case_fork_on_both_letters(self) -> None: + # Renamed from test_initials_still_drop_a_lowercase_conjunction + # (#528): that name described only 'y''s half, which is the one + # half this change does NOT move -- keeping it would have hidden + # that 'e''s half now moves. The two halves of the fork, on the + # facade. 'y' is not marked + # as reading both ways, so in a name written wholly in one case + # it is the connective and drops -- 1.4.0's answer, unmoved. + # 'e' IS marked, so the same name shape reads it as an initial + # and it contributes: 'j. s.' through 2.3.0, 'j. e. s.' since + # #528 made this view read the parse's tags instead of + # re-deriving from vocabulary and shape. The core's + # parse(...).initials() has given 'j. e. s.' since #383/#479 + # and the two agree now; decisions.md#P3 and decisions.md#R3 + # carry the split and its closing. hn = HumanName("john e smith") - self.m(hn.initials(), "j. s.", hn) + self.m(hn.initials(), "j. e. s.", hn) hn = HumanName("maria y lopez") self.m(hn.initials(), "m. l.", hn) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 1f747cd0..b2b9b782 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1091,11 +1091,13 @@ def _check_cjk_shape_purity(self) -> None: "where 2.3 gave 'j. s.', and capitalize() gives 'John E " "Smith' where 2.3 gave 'John e Smith', pinned in " "tests/v2/test_render.py. Measured 2026-09-13: the v1 " - "facade's HumanName.initials() does not yet follow this " - "fork and still gives 'j. s.' -- " - "test_facade_initials_do_not_yet_follow_the_one_case_fork " - "in tests/v2/test_render.py pins the split; closing it " - "is a follow-up issue's job. This table asserts roles", + "facade's HumanName.initials() did not follow this fork " + "that morning and still gave 'j. s.'; #528 closed the " + "split the same day by making that view read the " + "parse's tags, so both surfaces give 'j. e. s.' now and " + "test_facade_initials_follow_the_one_case_fork in " + "tests/v2/test_render.py pins the agreement where a " + "test pinned the split. This table asserts roles", shape=1), Case("one_case_upper_three_word_e_is_an_initial", "JOHN E SMITH", {"given": "JOHN", "middle": "E", "family": "SMITH"}, @@ -1139,12 +1141,13 @@ def _check_cjk_shape_purity(self) -> None: "ROLE is unchanged. What moves is initials() -- " "ParsedName.initials() gives 'J. G.' where 2.3 gave " "'J. Y. G.', a conjunction contributing none " - "(rules.md#R3) -- while the v1 facade's " - "HumanName.initials() still gives 'J. Y. G.', the same " - "split the 'john e smith' row records, running the " - "other way " - "(test_facade_initials_do_not_yet_follow_the_one_case_fork " - "in tests/v2/test_render.py) -- which is why this row " + "(rules.md#R3) -- and the v1 facade's " + "HumanName.initials() gave 'J. Y. G.' until #528 made " + "it read the same tags on 2026-09-13, the same split " + "the 'john e smith' row records, running the other way " + "and closed the same day " + "(test_facade_initials_follow_the_one_case_fork in " + "tests/v2/test_render.py) -- which is why this row " "needs the lowercase twin below to be readable", shape=1), Case("one_case_lower_y_keeps_the_three_word_carveout", "juan y garcia", diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 17d6a21f..c9316815 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -7,7 +7,7 @@ from nameparser._config_shim import CONSTANTS, Constants from nameparser._facade import HumanName -from nameparser._types import Role +from nameparser._types import UNCLASSIFIED_TAG, Role _DATA_DIR = Path(__file__).parent / "data" @@ -633,3 +633,191 @@ def test_facade_parses_unspaced_korean_by_default() -> None: # lexicon mirrors Lexicon.default() via the shim snapshot) n = HumanName("김민준") assert (n.last, n.first) == ("김", "민준") + + +def test_list_tokens_for_carries_the_list_view_s_own_elements() -> None: + # #528: the initials view needs the TOKEN behind each word, and the + # element boundaries are the whole point -- a "joined" continuation + # belongs to its predecessor's part and a folded middle sorts first. + # One walk builds both views so they cannot drift apart. + # + # Verified at review (2026-09-13): 0 mismatches between + # _list_tokens_for's join and _list_for's own output over the + # full corpus, 1173 names x 7 members = 8211 pairs. That sweep + # can't fail by construction -- _list_for IS DEFINED as that join + # -- so it does not stand as a test by itself. What a per-token + # walk COULD get wrong is the element BOUNDARIES, so this pins the + # three shapes where that could diverge by name: a folded middle + # (reordered ahead of the other parts), a "joined" continuation + # (two tokens sharing one element), and a spliced field + # (UNCLASSIFIED_TAG tokens from replace(), no STABLE tag to walk). + folded_c = Constants() + folded_c.middle_name_as_last = True + spliced = HumanName("john smith") + spliced.middle = "e f" + cases: list[tuple[HumanName, str, list[list[str]]]] = [ + (HumanName("Hassan, Mohamad Ahmad Ali", constants=folded_c), "last", + [["Ahmad"], ["Ali"], ["Hassan"]]), # folded middle, first + (HumanName("Ph. D., John"), "last", + [["Ph.", "D."]]), # "joined" continuation + (spliced, "middle", [["e"], ["f"]]), # spliced field + ] + for n, member, expected_shape in cases: + groups = n._list_tokens_for(member) + assert [[t.text for t in g] for g in groups] == expected_shape, \ + (n.original, member) + assert [" ".join(t.text for t in g) for g in groups] \ + == n._list_for(member), (n.original, member) + + for name in ("Ph. D., John", "Dr. Juan Q. Xavier de la Vega III", + "der, y van", "JUAN GARCIA Y LOPEZ", "Doe, John A."): + n = HumanName(name) + for member in ("title", "first", "middle", "last", "suffix", + "nickname", "maiden"): + groups = n._list_tokens_for(member) + assert all(g for g in groups), (name, member) + + +def test_token_is_conjunction_reads_the_tag_then_the_vocabulary() -> None: + # #528, the two-way decision. A token the parser classified answers + # from its tags -- the source the core's initials() reads + # (mechanisms.md#RENDER-HONORS-THE-PARSE) -- so a one-case 'e' that + # rules.md#P3 tagged `initial` is a name word and a bare capital 'Y' + # that P3 tagged `conjunction` is not, which is the reverse of what + # the vocabulary-and-shape test said about either. + parsed = HumanName("JUAN Y GARCIA") + tags = {t.text: t for t in parsed._parsed.tokens} + assert parsed._token_is_conjunction(tags["Y"]) is True + assert parsed._token_is_conjunction(tags["JUAN"]) is False + + fork = HumanName("john e smith") + e = {t.text: t for t in fork._parsed.tokens}["e"] + assert fork._token_is_conjunction(e) is False + + # A field spliced in as raw text was read by no parse, so there is + # no tag to honor and the vocabulary answers -- the same fallback + # rules.md#R4's case repair takes, through the same helper. + spliced = HumanName("john smith") + spliced.middle = "e" + lower = spliced._parsed.tokens_for(Role.MIDDLE)[0] + assert UNCLASSIFIED_TAG in lower.tags + assert spliced._token_is_conjunction(lower) is True + spliced.middle = "E" + upper = spliced._parsed.tokens_for(Role.MIDDLE)[0] + assert spliced._token_is_conjunction(upper) is False + + +def test_process_initial_direct_call_keeps_the_v1_string_path() -> None: + # tests/test_initials.py calls this with a bare string and no + # tokens, which is v1's shape and stays supported: with no tokens + # there is no parse to honor, so every word takes the spliced-text + # fallback -- the reading this method had for every word before + # #528. These five are measured, not predicted. _process_initial + # joins with initials_separator only (never initials_delimiter -- + # that is applied by the caller in initials()), so the bare-string + # probes with the default Constants come back undotted: 'j s' and + # 'J Y G', not 'j. s.' / 'J. Y. G.'. + hn = HumanName("", initials_separator="-", initials_delimiter=".") + assert hn._process_initial("Van Berg", firstname=True) == "V-B" + hn2 = HumanName("", initials_separator="") + assert hn2._process_initial("Van Berg", firstname=True) == "VB" + hn3 = HumanName("") + assert hn3._process_initial("john e smith") == "j s" + assert hn3._process_initial("JUAN Y GARCIA") == "J Y G" + assert hn3._process_initial("de la") == "" + + +def test_process_initial_with_tokens_reads_the_parse() -> None: + # The same two name parts, this time handed the tokens the parse + # built: the answer flips on both, which is #528 in one assertion. + hn = HumanName("JUAN Y GARCIA") + middle = hn._list_tokens_for("middle")[0] + assert hn._process_initial("", firstname=False, tokens=middle) == "" + fork = HumanName("john e smith") + fork_middle = fork._list_tokens_for("middle")[0] + assert fork._process_initial("", firstname=False, + tokens=fork_middle) == "e" + + +def test_v1_signature_override_raises_from_initials() -> None: + # The stated break (Derek, 2026-09-13): _initials_lists always + # calls _process_initial with `tokens=`, so a subclass overriding + # it with v1's two-argument signature (name_part, firstname=False) + # raises TypeError the moment initials() runs, rather than being + # silently skipped. Accepted over the alternative -- a string + # wrapper kept over a token core -- which would make such an + # override silently ineffective instead: the override would look + # like it works and never actually run. Cited by the docs commit + # as the accepted cost of #528's move to tokens. + class LegacyOverride(HumanName): + # Deliberately v1's narrower signature -- the incompatibility + # IS the break under test, not a typing slip. + def _process_initial(self, name_part: str, # type: ignore[override] + firstname: bool = False) -> str: + return super()._process_initial(name_part, firstname) + + hn = LegacyOverride("John Smith") + with pytest.raises(TypeError, match="tokens"): + hn.initials() + + +def test_initials_of_a_spliced_field_ask_the_vocabulary() -> None: + # A field assigned after the parse is raw text: ParsedName.replace() + # stamps UNCLASSIFIED_TAG on it, which says the words were read by + # nothing rather than read and found plain. There is no tag to + # honor, so this view falls back to the vocabulary for the one + # question a word can answer alone -- the same fallback and the + # same helper as rules.md#R4's case repair. The lowercase spelling + # reads as the connective and drops; the capital is initial-shaped + # and stays, keeping the letter's case as every initial does. + lower = HumanName("john smith") + lower.middle = "e" + assert lower.initials() == "j. s." + upper = HumanName("john smith") + upper.middle = "E" + assert upper.initials() == "j. E. s." + + +def test_initials_freeze_the_connective_answer_at_parse_time() -> None: + # The accepted cost of #528 (Derek, 2026-09-13): for a word backed + # by a token, "is this the connective" is decided when the name is + # parsed, exactly as capitalize()'s answer already was. A + # vocabulary edit after the parse therefore takes effect on the + # next full_name assignment, not on the next initials() call. + # A local Constants, never CONSTANTS: the shared singleton would + # leak the removal into every later test in the process. + constants = Constants() + name = HumanName("juan y garcia", constants=constants) + assert name.initials() == "j. g." + constants.conjunctions.remove("y") + assert name.initials() == "j. g." # frozen at parse time + # capitalize() has behaved this way all along, which is the + # precedent this cost was accepted on + name.capitalize() + assert str(name) == "Juan y Garcia" + name.full_name = "juan y garcia" # re-parse applies it + assert name.initials() == "j. y. g." + + +def test_initials_of_an_unpickled_name_ask_the_vocabulary_too() -> None: + # __setstate__ is the second producer of UNCLASSIFIED_TAG tokens: a + # v1 pickle carries the *_list STRINGS and no tags, so a restored + # name is spliced text throughout and takes the fallback above. + # That makes it disagree with a live parse of the same string on + # the two names rules.md#P3's one-case fork moved -- which + # capitalize() has done since the tag was introduced, for the same + # reason and through the same helper. Pinned rather than left to + # prose; decisions.md#R3 records it. + for name, live_initials, restored_initials, live_cap, restored_cap in ( + ("JUAN Y GARCIA", "J. G.", "J. Y. G.", + "Juan y Garcia", "Juan Y Garcia"), + ("john e smith", "j. e. s.", "j. s.", + "John E Smith", "John e Smith")): + assert HumanName(name).initials() == live_initials + restored = pickle.loads(pickle.dumps(HumanName(name))) + assert restored.initials() == restored_initials + live = HumanName(name) + live.capitalize() + assert str(live) == live_cap + restored.capitalize() + assert str(restored) == restored_cap diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 6a026840..f904ee52 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -889,6 +889,16 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # measured at 1.4.0 that day, neither 'JOSE E MARIA SANTOS' nor # 'Jose E Maria Santos' diffs at all, the facade's initials being # what this baseline compares and #383/#479 moving only the core's. + # + # Re-read 2026-09-13 after #528, which made the facade follow the + # tags: #383/#479 no longer moves the core alone, and the four + # names whose facade view it moved are now claimed by + # `fix(#528) the facade's initials follow the parse's connective + # tags`, written ahead of this rule. Both probes above still do + # not diff -- measured the same day, they are an all-upper and a + # capital-E mixed-case spelling and neither surface moves on + # either -- so the case-sensitivity argument is untouched and so + # is the roster. "a connective run initials": ("Jose E Maria Santos", "JOSE E MARIA SANTOS", "Scott E. Werner", "Amy E Maid"), @@ -906,6 +916,13 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # claim would be the field narrowing (`_initials` against a # measured `_ambiguities`), which is a thinner wall than the # regex and would vanish the moment the report stopped moving. + # + # 2026-09-13, after #528: the FACADE's initials move on this name + # too ('j. s.' -> 'j. e. s.'), so both surfaces do. At the three + # 2.x baselines that changes nothing here -- the measured + # `_ambiguities` diff still keeps `_initials` out of the name's + # diff entirely -- and the probe's job is unchanged: this rule's + # lowercase exclusion is what refuses the claim, and it must. "fix(#462)": ("john e smith", "maria y lopez", "E.T. Smith", "Maier, Amy I, Jr."), # #436/#437's rules are literal-anchored, so _CORPUS_CLAIMS' @@ -1091,6 +1108,28 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: "fix(#383/#479) a bare capital connective in an all-upper name stops initialing": ("Juan Y Garcia", "juan y garcia", "JUAN GARCIA Y LOPEZ", "juan q. xavier velasquez y garcia iii"), + # #528's rule is a literal alternation of four names, so + # _CORPUS_CLAIMS cannot see a widening that reaches only names the + # corpora lack -- these probes are the wall, and the key is the + # FULL issue string for the same reason the #383/#479 keys are. + # + # The first four are MIXED-CASE spellings: mixed case is where the + # writing decides the letter (rules.md#P3), neither surface moved + # there, and a rule reaching one would be absorbing a regression + # in the half of P3 that change did not touch -- 'Scott E. Werner' + # most of all, being the name fix(#462) exists for. 'maria y lopez' + # and 'juan y garcia' are the one-case controls whose lowercase + # 'y' was the connective before and after. 'Ph. D., John' is the + # one corpus name whose two initials views still disagree, for the + # unrelated reason fix(initials-per-word) the Ph. D. merge names; + # this rule must never claim it. 'john e jones, III' is the probe + # worth understanding: the trailing uppercase III makes the whole + # name mixed-case, so its 'e' never entered the one-case fork and + # its initials do not move on either surface. + "fix(#528) the facade's initials follow the parse's connective tags": + ("John E Smith", "Juan Y. Garcia", "Juan y Garcia", + "Scott E. Werner", "Jose E Maria Santos", "maria y lopez", + "Ph. D., John", "john e jones, III", "juan y garcia"), # The third feat(#269) rule, and the only one keyed on a derived # view. Its boundary is the other two: the prefix chain and the # Cyrillic pair are #269 recognitions as well, and both move ROLES, @@ -2059,6 +2098,17 @@ class _LatinCopy(NamedTuple): frozenset({"JOSE E MARIA SANTOS", "JOHN E SMITH", "john e smith", "john e jones", "jones, john e", "e j smith", "e and e"}), + # #528's movers, one corpus name per alternative -- a list of + # names, not a copy of CONJUNCTIONS. The rule's subject is not + # expressible as a shape over the raw string at all: what moved is + # which READING the parse gave a letter, and the same four strings + # would be claimed by a connective-shaped member together with + # every other corpus name carrying a connective ('Juan y Eva + # Garcia', 'juan garcia y lopez', 'Rob And Beth Edmunds'). Four of + # the six names whose HumanName.initials() move, the other two + # moving roles as well so that `_initials` never enters their diff. + frozenset({"john e smith", "john e jones", "jones, john e", + "JUAN Y GARCIA"}), }) def _unjustified_reach(name_regex: str, members: set[str]) -> list[str]: @@ -2750,6 +2800,14 @@ def _claim(rule: dict) -> _Claim: # alternation grew. "feat(#269) a recognized non-Latin connective contributes no initial": _Claim(1, ('_initials',), "770ce7374f32", ('DEFAULT',)), + # #528's literal name list, added 2026-09-13 and sitting ahead + # of the three vocabulary rules below because it shares their + # `_initials` field and two of their corpus names. Four corpus + # names, `_initials` alone: the roles move on none of them, + # which is what leaves the derived view as the whole diff. A + # fifth name here means the alternation grew. + "fix(#528) the facade's initials follow the parse's connective tags": + _Claim(4, ('_initials',), "7cb6b2f5779e", ('DEFAULT',)), # 96 -> 97 on 2026-09-08: 'Prince of Wales Jr' joined the # rules corpus with the 2.3 title-run bundle -- a parity # row, kept as the boundary the peel floor declines -- and @@ -2758,9 +2816,14 @@ def _claim(rule: dict) -> _Claim: # santos', 'John e Smith' and 'juan y garcia' arrived with # #383/#479's case rows and rules.md#P3 examples, each # carrying a lowercase connective this regex reaches. Reach, - # not explanation again: the facade's view does not move for - # any of them at this baseline, which is why #383/#479 has no - # `_initials` rule here at all. + # not explanation again: the facade's view did not move for + # any of them at this baseline, which is why #383/#479 had no + # `_initials` rule here. #528 gave it one on 2026-09-13 -- + # `fix(#528) the facade's initials follow the parse's + # connective tags`, written ahead of this rule because they + # share a field and two corpus names -- and this rule's REACH + # is unmoved by it: the regex did not change, and reach is + # what this number counts. "fix(initials-per-word) a connective run initials each word (facade, since 2.0.0)": _Claim(101, ('_initials',), "e91031622dca", ('DEFAULT',)), "fix(initials-per-word) a bound-given run initials each word (facade, since 2.0.0)": @@ -3771,6 +3834,24 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: "fix(#385/#402) an all-particle name part initials its words (R2)", "van ma van": "fix(#385/#402) an all-particle name part initials its words (R2)", + # The equal-`fields` contest #528 opened: both rules carry + # `fields = ["_initials"]`, so neither is narrower and + # `precedes_narrower` has nothing to declare -- classify() + # hands the name to whichever is written first, and this is + # where that answer is pinned. The connective rule's regex + # reaches both names through their lowercase ' e ' (measured + # 2026-09-13) and describes the 2.0.0 per-word GROUPING change, + # which is not what moved: what moved is the reading of the + # letter, and #528's rule says so. 'jones, john e' and + # 'JUAN Y GARCIA' are not contested -- the connective rule's + # regex wants whitespace after the letter and is case-sensitive + # on it -- so neither has a row. + "john e smith": + "fix(#528) the facade's initials follow the parse's " + "connective tags", + "john e jones": + "fix(#528) the facade's initials follow the parse's " + "connective tags", # the glued/spaced boundary. 'Andersonさん' and '김민준씨' left # suffix-routing for a rule that names them; '김민준 씨.' is # spaced and stays on the spaced rule, which #372 taught to diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 5e5240bc..3021af31 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -680,29 +680,31 @@ def test_capitalized_one_case_connective_that_reads_as_an_initial() -> None: assert str(parse("John e Smith").capitalized()) == "John e Smith" -def test_facade_initials_do_not_yet_follow_the_one_case_fork() -> None: - """The core's initials() follows the parse's tags: 'e' in a - one-case name is an INITIAL (rules.md#P3), so R3's "each given, - middle, and base family word" reaches it and it initials. - HumanName.initials() does not go through the parse at all -- - `_facade._process_initial` re-derives "conjunction" from the - lexicon and the part's raw text/shape, not from the token's tag -- - so it keeps 1.4.0 parity here. The split is recorded at - decisions.md#P3 and closing it is a follow-up issue's job, not - this one's. +def test_facade_initials_follow_the_one_case_fork() -> None: + """Both surfaces read the same letter the same way (#528). + + The core's initials() follows the parse's tags: in a name written + wholly in one case an 'e' is an INITIAL and a 'y' is the connective + (rules.md#P3), so R3's "each given, middle, and base family word" + reaches the first and not the second. Until #528 HumanName.initials() + re-derived that from the lexicon and the part's raw shape instead, + and kept 1.4.0's answer on both letters; it now reads the same tags, + so the two views of one parse agree. decisions.md#R3 records it. """ assert parse("john e smith").initials() == "j. e. s." - assert HumanName("john e smith").initials() == "j. s." - - -def test_y_side_initials_of_the_one_case_fork() -> None: - """The other direction of the split above, pinned on 'y' rather - than 'e': the core follows the fork ('Y' is a plain conjunction in - a one-case name and contributes no initial, rules.md#R3), and the - facade still does not. - """ + assert HumanName("john e smith").initials() == "j. e. s." assert parse("JUAN Y GARCIA").initials() == "J. G." + assert HumanName("JUAN Y GARCIA").initials() == "J. G." assert parse("JUAN GARCIA Y LOPEZ").initials() == "J. G. L." - # the split, the other direction: the facade's HumanName still - # reads a bare capital connective as an initial (1.4.0 parity) - assert HumanName("JUAN Y GARCIA").initials() == "J. Y. G." + assert HumanName("JUAN GARCIA Y LOPEZ").initials() == "J. G. L." + # The mixed-case controls, where the writing decides the letter and + # nothing moved on either surface + assert HumanName("John E Smith").initials() == "J. E. S." + assert HumanName("Juan Y. Garcia").initials() == "J. Y. G." + assert HumanName("maria y lopez").initials() == "m. l." + # The one corpus name where the two views still differ, and it is + # not this rule's: the facade merges 'Ph.' + 'D.' into one list + # element and renders it with no inner delimiter + # (fix(initials-per-word) the Ph. D. merge, decisions.md#phd-merge) + assert HumanName("Ph. D., John").initials() == "J. P D." + assert parse("Ph. D., John").initials() == "J. P. D." diff --git a/tools/differential/compare.py b/tools/differential/compare.py index 76a684bb..c711f874 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -1784,6 +1784,17 @@ class _ShapeMismatch(NamedTuple): "Abu Bakr Al Baghdadi, MD": ("_initials",), "abu bakr al baghdadi": ("_initials",), "Berg, abdul van": ("_initials",), + # #528's two, adjudicated 2026-09-13. Both are contested by + # `fix(#528) the facade's initials follow the parse's + # connective tags` against `fix(initials-per-word) a + # connective run initials each word`: equal `fields`, so + # neither is narrower, `precedes_narrower` has no narrower + # rule to name, and file order is the whole decision. The + # shapes are this run's, not guessed -- the gate reported the + # pair as an unpinned contest and these are the diffs it + # measured. The winners are pinned in _CROSS_RULE_WINNERS. + "john e smith": ("_initials",), + "john e jones": ("_initials",), # #498's fourteen, adjudicated 2026-09-05: every 1.4.0 diff # two or more rules admitted, where the winner beats a loser # by neither narrow-first nesting nor a `precedes_narrower` diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 6d0dbabe..584dc5e7 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -2776,10 +2776,10 @@ fields = ["family", "suffix"] # on parse().initials() -- renders for a parse whose fields did not # move. # -# The order the file has them in: the three literal name LISTS first -# (fix(#385/#402), fix(#360), feat(#269)), then the three VOCABULARY -# rules -- connective, bound-given, particle chain -- then the Ph. D. -# merge. +# The order the file has them in: the four literal name LISTS first +# (fix(#385/#402), fix(#360), feat(#269), fix(#528) -- the last added +# 2026-09-13), then the three VOCABULARY rules -- connective, +# bound-given, particle chain -- then the Ph. D. merge. # Only the bound-given/particle-chain pair is ordered by narrowness, # and by measured reach rather than by reading: 41 corpus names # against 107 (measured 2026-09-02 at this baseline, and recorded in @@ -2793,17 +2793,34 @@ fields = ["family", "suffix"] # `_initials` diff under a declared order could only come from the # CORE, and must not be absorbed by a rule whose prose says "facade". # -# #383/#479 has NO rule in this block, and the absence is a measured -# finding rather than an omission. The change moves what -# `parse(...).initials()` renders for 'john e smith', 'john e jones', -# 'jones, john e' and 'JUAN Y GARCIA' -- but `HumanName.initials()` -# re-derives the connective decision from vocabulary and initial shape -# on the raw text instead of reading the parse's tags, so the FACADE's -# view does not move at all, and the facade is the only surface -# compared below 2.0. Measured 2026-09-13: not one of those four names -# diffs here. The core's movement is classified at the 2.x baselines, -# where the core is compared (decisions.md#P3 records the facade split -# and the follow-up it owes). +# #383/#479 HAD no rule in this block, and #528 gave it one. Recorded +# as a pair because the absence was itself a measured finding and the +# reason it stopped holding is the whole of #528. The change moved +# what `parse(...).initials()` renders for 'john e smith', 'john e +# jones', 'jones, john e' and 'JUAN Y GARCIA' -- and on 2026-09-13 +# `HumanName.initials()` re-derived the connective decision from +# vocabulary and initial shape on the raw text instead of reading the +# parse's tags, so the FACADE's view did not move at all and the +# facade is the only surface compared below 2.0. Measured that day: +# not one of the four diffed here. #528 closed the split; the facade +# reads the tags now, the same four names move here, and +# `fix(#528) the facade's initials follow the parse's connective tags` +# below classifies them. The two role-moving names of the same change +# ('jose e maria santos', 'JUAN GARCIA Y LOPEZ') are NOT among them: +# their roles move against this baseline, so `_initials` never enters +# their diff (compare.py's roles-identical guard) and the role rule at +# the end of this file explains what happened to them. decisions.md#P3 +# records the split and decisions.md#R3 its closing. +# +# THE #528 RULE SITS AHEAD OF THE VOCABULARY RULES, with the literal +# name lists, and the position is load-bearing rather than tidy: the +# connective rule below carries `fields = ["_initials"]` too, its +# regex reaches 'john e smith' and 'john e jones' (measured +# 2026-09-13), and equal `fields` puts the pair outside +# `precedes_narrower` -- which leaves file order deciding, with +# _CROSS_RULE_WINNERS in tests/v2/test_ledger_guards.py pinning the +# answer. Written first, the narrower rule wins, which is this file's +# convention. [[change]] issue = "fix(#385/#402) an all-particle name part initials its words (R2)" @@ -2864,6 +2881,53 @@ name_regex = "^محمد و علي$" fields = ["_initials"] orders = ["DEFAULT"] +[[change]] +issue = "fix(#528) the facade's initials follow the parse's connective tags" +# 'john e smith', 'john e jones', 'jones, john e' and 'JUAN Y GARCIA'. +# `HumanName.initials()` decided "is this word the connective" from +# the lexicon plus v1's initial SHAPE on the part's raw text, where +# the core's initials() reads the tag the parse recorded. #383/#479 +# gave rules.md#P3 a fork the shape cannot see -- in a name written +# wholly in one case a marked 'e' is an initial and an unmarked 'y' +# joins -- so from 2.4's #527 until #528 the two views of one parse +# disagreed about the same letter. They agree now, and the facade's +# answer is the core's: 'j. s.' -> 'j. e. s.' on the three e-names +# (1.4.0's values measured from the wheel, 2026-09-13) and 'J. Y. G.' +# -> 'J. G.' on the capital one. No role moves on any of the four -- +# P3's three-word carve-out is untouched -- which is exactly why the +# derived view is the whole diff here. +# +# LITERAL, four names, and the four are not the change's whole +# population: six corpus names move `HumanName.initials()`, and +# 'jose e maria santos' and 'JUAN GARCIA Y LOPEZ' move their ROLES +# against this baseline as well, so `_initials` never enters their +# diff and the #383/#479 role rule at the end of this file explains +# them. A regex for the class -- "a one-case name whose single-letter +# connective the parse read against 1.4.0's reading" -- is not +# writable over the raw string at all, the reading being the parse's. +# _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the +# probes: the mixed-case spellings, where the writing decides the +# letter and nothing moved on either surface; 'maria y lopez', whose +# lowercase 'y' was the connective before and after; and +# 'Ph. D., John', the one corpus name whose two initials views still +# disagree -- the facade merges the credential into one list element +# and renders 'P D' with no inner delimiter, which is +# fix(initials-per-word) the Ph. D. merge below and not this. +# +# Two of the four -- 'john e smith' and 'john e jones' -- were +# EXPLAINED BY THE CONNECTIVE RULE BELOW before this rule existed +# (measured 2026-09-13 at this baseline: both sat in its classified +# list). Its regex reaches them through their lowercase ' e ', its +# `fields` are these, and equal `fields` is not a `precedes_narrower` +# pair -- so file order is the whole instrument and this rule takes +# them by sitting first. That is the right answer on the merits: what +# moved on those two names is the READING of the letter, not the +# per-word GROUPING of an element that rule describes. +# _CROSS_RULE_WINNERS pins it and _RECORDED_DIFFS carries the shapes. +name_regex = "^(?:john e smith|john e jones|jones, john e|JUAN Y GARCIA)$" +fields = ["_initials"] +orders = ["DEFAULT"] + [[change]] issue = "fix(initials-per-word) a connective run initials each word (facade, since 2.0.0)" # 'Juan y Eva Garcia', 'Dean of Chemistry Robert Johns', 'John & Jane': diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index 29e49c40..dae04fc1 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -1795,23 +1795,29 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # every word it holds whatever the vocabulary says, so there was # nothing for the fix to restore. # -# 2026-09-13, #383/#479: 'JUAN Y GARCIA' now diffs here for TWO -# reasons and this rule explains both, because they are one -# `_initials` move. The FACADE's half is this rule's own -- a bare -# capital 'Y' is initial-shaped, so the 2.3 fix keeps it where this -# baseline dropped it ('J. G.' -> 'J. Y. G.', measured from the -# wheel). The CORE's half is new: in a name written wholly in one -# case nothing marks the letter as an initial, 'y' is not vocabulary -# that reads both ways, so it is the connective and contributes no -# initial ('J. Y. G.' -> 'J. G.', rules.md#P3 for which letters are -# connectives and rules.md#R3 for what a connective contributes). -# The two halves move the same pseudo-field in OPPOSITE directions on -# the two surfaces, and `_initials` is one field, so one rule takes -# it. That is why #383/#479's section at the end of this file writes +# 2026-09-13, #383/#479 then #528: 'JUAN Y GARCIA' diffs here on +# `_initials`, and it has had two different causes in one day. +# #383/#479 gave it two at once. The FACADE's half was this rule's +# own -- a bare capital 'Y' is initial-shaped, so the 2.3 fix kept it +# where this baseline dropped it, 'J. G.' -> 'J. Y. G.', measured +# from the wheel. The CORE's half was new: in a name written wholly +# in one case nothing marks the letter as an initial, 'y' is not +# vocabulary that reads both ways, so it is the connective and +# contributes no initial, 'J. Y. G.' -> 'J. G.' (rules.md#P3 for +# which letters are connectives and rules.md#R3 for what a connective +# contributes). #528 then made the facade read the parse's tags like +# the core, so the facade's half CLOSED: the wheel gives 'J. G.' and +# the tree gives 'J. G.' again, agreeing with this baseline, and the +# core's half is the whole diff. Measured 2026-09-13 on the 2.0.0, +# 2.1.0 and 2.2.0 wheels and on the tree. +# One rule still takes it, for the reason two halves needed one: +# `_initials` is ONE field, and this rule's regex reaches the name. +# That is also why #383/#479's section at the end of this file writes # no competing `_initials` rule -- an equal-`fields` contest decided -# by nothing but file order. At 2.3.0, where this rule does not -# exist because the facade fix has shipped, the core's half is -# classified on its own. +# by nothing but file order. At 2.3.0, where this rule does not exist +# because the facade fix has shipped, the core's half is classified +# on its own and BOTH surfaces move together since #528; that +# ledger's comment says so. # This is the ONE full copy: the 2.1.0 and 2.2.0 ledgers and the # three _CORPUS_CLAIMS entries point here rather than restating it. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" @@ -2188,13 +2194,15 @@ fields = ["_ambiguities", "family", "given", "middle", "suffix", "title"] # this baseline those names classify on the report alone. 'JUAN Y # GARCIA' is the one name of the population whose view moves with no # report beside it, and at this baseline it is already claimed: -# fix(#462) reaches it and admits `_initials`, because the FACADE's -# view moves there too (the 2.3 facade fix has not shipped at 2.0.0). -# That rule's comment records the double reason; a competing rule -# here would be an equal-`fields` contest decided by file order, +# fix(#462) reaches it and admits `_initials`. That rule's comment +# records what moved and when: the FACADE's view moved there too on +# the morning #383/#479 landed, the 2.3 facade fix not having shipped +# at 2.0.0, and #528 closed the facade's half the same day +# (2026-09-13), leaving the core's as the whole diff. A competing +# rule here would be an equal-`fields` contest decided by file order, # which is what _CROSS_RULE_WINNERS exists to stop rather than to -# create. It is classified on its own at 2.3.0, where the facade -# agrees and only the core moves. +# create. It is classified on its own at 2.3.0, where BOTH surfaces +# now move, and the same way. # # LAST in the file: every diff these two claim reported UNEXPLAINED # on the run that preceded them (measured 2026-09-13), so no rule diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 7c5ee367..ab73ca26 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -1716,11 +1716,14 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # every word it holds whatever the vocabulary says, so there was # nothing for the fix to restore. # -# 2026-09-13, #383/#479: 'JUAN Y GARCIA' now diffs here for TWO -# reasons at once -- the facade's, which is this rule's own, and the -# core's, which is new -- and this rule explains both as one -# `_initials` move. expected_since_2.0.0.toml's copy of this rule -# carries the account; it is the same at all three baselines. +# 2026-09-13, #383/#479 then #528: 'JUAN Y GARCIA' diffed here for +# TWO reasons at once that morning -- the facade's, which is this +# rule's own, and the core's, which was new -- and #528 closed the +# facade's half the same day, the facade now reading the parse's tags +# and agreeing with this baseline's 'J. G.' again. The core's half +# remains and this rule still explains it, one `_initials` move. +# expected_since_2.0.0.toml's copy of this rule carries the account; +# it is the same at all three baselines. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" fields = ["_initials"] orders = ["DEFAULT"] @@ -2079,13 +2082,15 @@ fields = ["_ambiguities", "family", "given", "middle", "suffix", "title"] # this baseline those names classify on the report alone. 'JUAN Y # GARCIA' is the one name of the population whose view moves with no # report beside it, and at this baseline it is already claimed: -# fix(#462) reaches it and admits `_initials`, because the FACADE's -# view moves there too (the 2.3 facade fix has not shipped at 2.1.0). -# That rule's comment records the double reason; a competing rule -# here would be an equal-`fields` contest decided by file order, +# fix(#462) reaches it and admits `_initials`. That rule's comment +# records what moved and when: the FACADE's view moved there too on +# the morning #383/#479 landed, the 2.3 facade fix not having shipped +# at 2.1.0, and #528 closed the facade's half the same day +# (2026-09-13), leaving the core's as the whole diff. A competing +# rule here would be an equal-`fields` contest decided by file order, # which is what _CROSS_RULE_WINNERS exists to stop rather than to -# create. It is classified on its own at 2.3.0, where the facade -# agrees and only the core moves. +# create. It is classified on its own at 2.3.0, where BOTH surfaces +# now move, and the same way. # # LAST in the file: every diff these two claim reported UNEXPLAINED # on the run that preceded them (measured 2026-09-13), so no rule diff --git a/tools/differential/expected_since_2.2.0.toml b/tools/differential/expected_since_2.2.0.toml index 5815102b..a27a0635 100644 --- a/tools/differential/expected_since_2.2.0.toml +++ b/tools/differential/expected_since_2.2.0.toml @@ -357,11 +357,14 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # every word it holds whatever the vocabulary says, so there was # nothing for the fix to restore. # -# 2026-09-13, #383/#479: 'JUAN Y GARCIA' now diffs here for TWO -# reasons at once -- the facade's, which is this rule's own, and the -# core's, which is new -- and this rule explains both as one -# `_initials` move. expected_since_2.0.0.toml's copy of this rule -# carries the account; it is the same at all three baselines. +# 2026-09-13, #383/#479 then #528: 'JUAN Y GARCIA' diffed here for +# TWO reasons at once that morning -- the facade's, which is this +# rule's own, and the core's, which was new -- and #528 closed the +# facade's half the same day, the facade now reading the parse's tags +# and agreeing with this baseline's 'J. G.' again. The core's half +# remains and this rule still explains it, one `_initials` move. +# expected_since_2.0.0.toml's copy of this rule carries the account; +# it is the same at all three baselines. name_regex = "(?:^|[\\s,])(?:[EY]|[EeYy]\\.)(?=[\\s,]|$)" fields = ["_initials"] orders = ["DEFAULT"] @@ -730,13 +733,15 @@ fields = ["_ambiguities", "family", "given", "middle", "suffix", "title"] # this baseline those names classify on the report alone. 'JUAN Y # GARCIA' is the one name of the population whose view moves with no # report beside it, and at this baseline it is already claimed: -# fix(#462) reaches it and admits `_initials`, because the FACADE's -# view moves there too (the 2.3 facade fix has not shipped at 2.2.0). -# That rule's comment records the double reason; a competing rule -# here would be an equal-`fields` contest decided by file order, +# fix(#462) reaches it and admits `_initials`. That rule's comment +# records what moved and when: the FACADE's view moved there too on +# the morning #383/#479 landed, the 2.3 facade fix not having shipped +# at 2.2.0, and #528 closed the facade's half the same day +# (2026-09-13), leaving the core's as the whole diff. A competing +# rule here would be an equal-`fields` contest decided by file order, # which is what _CROSS_RULE_WINNERS exists to stop rather than to -# create. It is classified on its own at 2.3.0, where the facade -# agrees and only the core moves. +# create. It is classified on its own at 2.3.0, where BOTH surfaces +# now move, and the same way. # # LAST in the file: every diff these two claim reported UNEXPLAINED # on the run that preceded them (measured 2026-09-13), so no rule diff --git a/tools/differential/expected_since_2.3.0.toml b/tools/differential/expected_since_2.3.0.toml index 411591c7..e4205a8a 100644 --- a/tools/differential/expected_since_2.3.0.toml +++ b/tools/differential/expected_since_2.3.0.toml @@ -117,12 +117,18 @@ issue = "fix(#383/#479) a marked connective letter in a one-case name is reporte # derived view enters a diff only where every role AND every # ambiguity kind agrees (#484, compare.py main()), and the report # moved on each of them, so no run can produce a shape here carrying -# it. That movement is visible nowhere else either -- the v1 facade -# re-derives the connective decision from vocabulary and shape -# instead of reading the parse's tags, so `HumanName.initials()` does -# not move at all and the 1.4.0 -# ledger, which compares the facade alone, has nothing to classify. -# decisions.md#P3 records the split. +# it. That movement was visible nowhere else either, on the morning +# this rule was written: the v1 facade re-derived the connective +# decision from vocabulary and shape instead of reading the parse's +# tags, so `HumanName.initials()` did not move at all and the 1.4.0 +# ledger, which compares the facade alone, had nothing to classify. +# #528 closed that split the same day (2026-09-13) -- the facade +# reads the tags now and moves with the core -- so the 1.4.0 ledger +# carries `fix(#528) the facade's initials follow the parse's +# connective tags` over these three names and 'JUAN Y GARCIA'. +# Nothing here changes: the report still moves on each of them, so +# `_initials` still cannot enter a shape at THIS baseline. +# decisions.md#P3 records the split and decisions.md#R3 its closing. name_regex = "^(?:JOSE E MARIA SANTOS|JOHN E SMITH|john e smith|john e jones|jones, john e|e j smith|e and e)$" fields = ["_ambiguities"] orders = ["DEFAULT"] @@ -151,8 +157,14 @@ issue = "fix(#383/#479) a bare capital connective in an all-upper name stops ini # the initial-shaped capital the older facades dropped), and # `_initials` is one field -- so `fix(#462)` explains both halves # there as one move, and its comment in those three ledgers says so. -# That fix has shipped at this baseline, so the facade agrees and -# only the core moves. +# That fix has shipped at this baseline, so the facade agreed with +# the core here and, when this rule was written, only the core moved. +# Since #528 BOTH surfaces move, and the same way: this baseline's +# wheel gives 'J. Y. G.' from HumanName.initials() and 'J. Y. G.' +# from parse(...).initials(), the tree gives 'J. G.' from each +# (measured 2026-09-13). One `_initials` field and one move, so the +# rule is unchanged and only this sentence is; decisions.md#R3 +# records why the facade started following the tags. # # Literal, one name: the shape would be "an all-upper three-word name # whose middle word is an unmarked single-letter connective", which From 1972b7377550b84316ffd3ec531468c7b012e613 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 13 Sep 2026 21:41:03 -0700 Subject: [PATCH 02/11] chore(differential): the ledger comments say what was measured Quality-review fixes on 919644b's ledger and guard prose, comments only. The #528 rule's comment said "'j. s.' -> 'j. e. s.' on the three e-names", which is true of one of them: measured against the 1.4.0 wheel, 'john e jones' and 'jones, john e' go 'j. j.' -> 'j. e. j.'. Each of the four is now stated on its own. "Six corpus names move `HumanName.initials()`" was an undated standing count and false read literally -- 'e and e' and 'juan garcia y lopez' move here too, on the 2.0.0 per-word grouping. It becomes a dated, scoped statement in both places it is written: six one-case names move for the TAG reason, measured 2026-09-13 by driving the facade before and after the change, two of them ('jose e maria santos', 'JUAN GARCIA Y LOPEZ') moving a role as well so that `_initials` never enters their diff. The equal-`fields`/file-order argument and the role-mover exclusion were each written twice inside expected_since_1.4.0.toml. The block header keeps both copies and the rule's comment points at it; the copies in compare.py and tests/v2/test_ledger_guards.py stay, being the cross-file convention. Also: _CROSS_RULE_WINNERS now gives one property per uncontested name ('jones, john e' by the trailing-whitespace requirement, 'JUAN Y GARCIA' by the case) rather than both at once; _CORPUS_CLAIMS' pronoun pointed at three rules where the two shared names are the connective rule's alone; the 2.0.0 ledger says "the wheel's FACADE gives 'J. G.'", the wheel's core giving 'J. Y. G.'; the "a connective run initials" roster's third stacked paragraph folds into a dated correction of the second; _MUST_NOT_MATCH said "the first four are MIXED-CASE" where a ninth probe had made it five; and cases.py's run-on splits. _RECORDED_DIFFS' two new rows move below #498's block so the three cohorts read down the dict in the order the PROVENANCE note tells them, and that note goes 45 -> 47 with #528's two named and dated. All five gates re-run at 0 unexplained and no moved shape. Co-Authored-By: Claude Fable 5.1 --- tests/v2/cases.py | 13 ++-- tests/v2/test_ledger_guards.py | 62 ++++++++++++-------- tools/differential/compare.py | 35 ++++++----- tools/differential/expected_since_1.4.0.toml | 55 +++++++++-------- tools/differential/expected_since_2.0.0.toml | 9 +-- 5 files changed, 99 insertions(+), 75 deletions(-) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index b2b9b782..5b6d131b 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -1143,12 +1143,15 @@ def _check_cjk_shape_purity(self) -> None: "'J. Y. G.', a conjunction contributing none " "(rules.md#R3) -- and the v1 facade's " "HumanName.initials() gave 'J. Y. G.' until #528 made " - "it read the same tags on 2026-09-13, the same split " - "the 'john e smith' row records, running the other way " - "and closed the same day " + "it read the same tags on 2026-09-13. That is the " + "split the 'john e smith' row records, closed on the " + "same day and by the same change " "(test_facade_initials_follow_the_one_case_fork in " - "tests/v2/test_render.py) -- which is why this row " - "needs the lowercase twin below to be readable", + "tests/v2/test_render.py). It runs the other way here: " + "there the facade GAINED a letter it had been " + "dropping, and here it loses one it had been keeping. " + "Which is why this row needs the lowercase twin below " + "to be readable", shape=1), Case("one_case_lower_y_keeps_the_three_word_carveout", "juan y garcia", {"given": "juan", "middle": "y", "family": "garcia"}, diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index f904ee52..5eb2fe3e 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -882,23 +882,21 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # case-sensitivity buys, and it is why this roster keeps probing # for it after the bug is gone. # - # Re-read 2026-09-13 against #383/#479 and left exactly as it is. - # The case-sensitivity argument is about the #462 shapes, which - # are MIXED case, and this change touched only one-case names. The - # sentence above still holds literally for both uppercase probes: - # measured at 1.4.0 that day, neither 'JOSE E MARIA SANTOS' nor - # 'Jose E Maria Santos' diffs at all, the facade's initials being - # what this baseline compares and #383/#479 moving only the core's. - # - # Re-read 2026-09-13 after #528, which made the facade follow the - # tags: #383/#479 no longer moves the core alone, and the four - # names whose facade view it moved are now claimed by + # Re-read 2026-09-13 against #383/#479 and then again after #528, + # and left exactly as it is both times. The case-sensitivity + # argument is about the #462 shapes, which are MIXED case, and + # #383/#479 touched only one-case names. The sentence above still + # holds literally for both uppercase probes: measured at 1.4.0 + # that day, neither 'JOSE E MARIA SANTOS' nor 'Jose E Maria + # Santos' diffs at all, the facade's initials being what this + # baseline compares and #383/#479 having moved only the core's -- + # a clause #528 retired the same day by making the facade follow + # the tags too, without moving either probe (re-measured after it; + # an all-upper and a capital-E mixed-case spelling, and neither + # surface moves on either). What #528 added is a claimant: the + # four names whose facade view it moved now go to # `fix(#528) the facade's initials follow the parse's connective - # tags`, written ahead of this rule. Both probes above still do - # not diff -- measured the same day, they are an all-upper and a - # capital-E mixed-case spelling and neither surface moves on - # either -- so the case-sensitivity argument is untouched and so - # is the roster. + # tags`, written ahead of this rule. "a connective run initials": ("Jose E Maria Santos", "JOSE E MARIA SANTOS", "Scott E. Werner", "Amy E Maid"), @@ -1113,7 +1111,7 @@ def test_case_shape_ids_exist_in_the_inventory() -> None: # corpora lack -- these probes are the wall, and the key is the # FULL issue string for the same reason the #383/#479 keys are. # - # The first four are MIXED-CASE spellings: mixed case is where the + # The first five are MIXED-CASE spellings: mixed case is where the # writing decides the letter (rules.md#P3), neither surface moved # there, and a rule reaching one would be absorbing a regression # in the half of P3 that change did not touch -- 'Scott E. Werner' @@ -2104,9 +2102,16 @@ class _LatinCopy(NamedTuple): # which READING the parse gave a letter, and the same four strings # would be claimed by a connective-shaped member together with # every other corpus name carrying a connective ('Juan y Eva - # Garcia', 'juan garcia y lopez', 'Rob And Beth Edmunds'). Four of - # the six names whose HumanName.initials() move, the other two - # moving roles as well so that `_initials` never enters their diff. + # Garcia', 'juan garcia y lopez', 'Rob And Beth Edmunds'). + # Measured 2026-09-13 over the corpus glob, driving the facade + # before and after #528: six one-case names' HumanName.initials() + # move for the TAG reason, and these are the four of them whose + # roles hold still. The other two, 'jose e maria santos' and + # 'JUAN GARCIA Y LOPEZ', move a role against 1.4.0 as well, so + # `_initials` never enters their diff and no `_initials` rule can + # list them. Scoped to that reason: facade initials that move + # against 1.4.0 for OTHER reasons -- 'e and e' and 'juan garcia y + # lopez', on the 2.0.0 per-word grouping -- are no part of the six. frozenset({"john e smith", "john e jones", "jones, john e", "JUAN Y GARCIA"}), }) @@ -2802,8 +2807,9 @@ def _claim(rule: dict) -> _Claim: _Claim(1, ('_initials',), "770ce7374f32", ('DEFAULT',)), # #528's literal name list, added 2026-09-13 and sitting ahead # of the three vocabulary rules below because it shares their - # `_initials` field and two of their corpus names. Four corpus - # names, `_initials` alone: the roles move on none of them, + # `_initials` field, and because the first of them -- the + # connective rule -- reaches two of its corpus names. + # Four corpus names, `_initials` alone: the roles move on none, # which is what leaves the derived view as the whole diff. A # fifth name here means the alternation grew. "fix(#528) the facade's initials follow the parse's connective tags": @@ -3842,10 +3848,14 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: # reaches both names through their lowercase ' e ' (measured # 2026-09-13) and describes the 2.0.0 per-word GROUPING change, # which is not what moved: what moved is the reading of the - # letter, and #528's rule says so. 'jones, john e' and - # 'JUAN Y GARCIA' are not contested -- the connective rule's - # regex wants whitespace after the letter and is case-sensitive - # on it -- so neither has a row. + # letter, and #528's rule says so. The other two of #528's four + # are not contested, each kept out by a different half of that + # regex (measured 2026-09-13): 'jones, john e' ends on its 'e', + # and the alternation wants whitespace AFTER the letter, so a + # letter at end of string never matches; 'JUAN Y GARCIA' has + # the whitespace but a capital 'Y', and the alternation is + # CASE-SENSITIVE, carrying only the lowercase letter. Neither + # has a row. "john e smith": "fix(#528) the facade's initials follow the parse's " "connective tags", diff --git a/tools/differential/compare.py b/tools/differential/compare.py index c711f874..5f2bce63 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -1704,12 +1704,13 @@ class _ShapeMismatch(NamedTuple): #: Default-order shapes only, because the roster classifies with no #: order. recorded_diff_mismatches below says what that leaves out. #: -#: PROVENANCE. The 45 rows at 1.4.0 are measured against the 1.4.0 -#: wheel, as the roster always claimed, and all 45 still agree -- the +#: PROVENANCE. The 47 rows at 1.4.0 are measured against the 1.4.0 +#: wheel, as the roster always claimed, and all 47 still agree -- the #: original 31 re-measured 2026-09-03, the fourteen #498 added measured -#: 2026-09-05 by the sweep that found them, and every one of the 45 -#: checked by the same recompute: drive main() at all four baselines and -#: feed its `diffing` and its post-skip corpus to +#: 2026-09-05 by the sweep that found them, #528's two measured +#: 2026-09-13 by the run that reported their contest, and every one of +#: the 47 checked by the same recompute: drive main() at all four +#: baselines and feed its `diffing` and its post-skip corpus to #: recorded_diff_mismatches, wrapping _run_worker to #: capture the post-skip entries and dormant_rules to capture `diffing`, #: since both receive exactly what main() built. @@ -1784,17 +1785,6 @@ class _ShapeMismatch(NamedTuple): "Abu Bakr Al Baghdadi, MD": ("_initials",), "abu bakr al baghdadi": ("_initials",), "Berg, abdul van": ("_initials",), - # #528's two, adjudicated 2026-09-13. Both are contested by - # `fix(#528) the facade's initials follow the parse's - # connective tags` against `fix(initials-per-word) a - # connective run initials each word`: equal `fields`, so - # neither is narrower, `precedes_narrower` has no narrower - # rule to name, and file order is the whole decision. The - # shapes are this run's, not guessed -- the gate reported the - # pair as an unpinned contest and these are the diffs it - # measured. The winners are pinned in _CROSS_RULE_WINNERS. - "john e smith": ("_initials",), - "john e jones": ("_initials",), # #498's fourteen, adjudicated 2026-09-05: every 1.4.0 diff # two or more rules admitted, where the winner beats a loser # by neither narrow-first nesting nor a `precedes_narrower` @@ -1817,6 +1807,19 @@ class _ShapeMismatch(NamedTuple): "Smith, Ph. D. MD": ("suffix", "title"), "Smith, Ph.D. Jr.": ("given", "suffix"), "Smith, PhD Jr.": ("given", "suffix", "title"), + # #528's two, adjudicated 2026-09-13, and kept BELOW #498's + # block so the three cohorts read down the dict in the order + # the PROVENANCE note above tells them. Both are contested by + # `fix(#528) the facade's initials follow the parse's + # connective tags` against `fix(initials-per-word) a + # connective run initials each word`: equal `fields`, so + # neither is narrower, `precedes_narrower` has no narrower + # rule to name, and file order is the whole decision. The + # shapes are this run's, not guessed -- the gate reported the + # pair as an unpinned contest and these are the diffs it + # measured. The winners are pinned in _CROSS_RULE_WINNERS. + "john e smith": ("_initials",), + "john e jones": ("_initials",), }, # #501's six, moved here from _WATCHED_DIFFS with their shapes # unchanged. The four CJK rows sit at 2.0.0 alone: the honorific diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 584dc5e7..a0310db4 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -2805,12 +2805,24 @@ fields = ["family", "suffix"] # not one of the four diffed here. #528 closed the split; the facade # reads the tags now, the same four names move here, and # `fix(#528) the facade's initials follow the parse's connective tags` -# below classifies them. The two role-moving names of the same change -# ('jose e maria santos', 'JUAN GARCIA Y LOPEZ') are NOT among them: -# their roles move against this baseline, so `_initials` never enters -# their diff (compare.py's roles-identical guard) and the role rule at -# the end of this file explains what happened to them. decisions.md#P3 -# records the split and decisions.md#R3 its closing. +# below classifies them. +# +# THE FOUR ARE NOT THE CHANGE'S WHOLE POPULATION, and this is the +# one place in this file that says which names it leaves out. +# Measured 2026-09-13 over the corpus glob, driving the facade before +# and after #528: SIX one-case names' `HumanName.initials()` move for +# the tag reason, and the two that are missing above +# ('jose e maria santos', 'JUAN GARCIA Y LOPEZ') move their ROLES +# against this baseline as well, so `_initials` never enters their +# diff (compare.py's roles-identical guard) and the role rule at the +# end of this file explains what happened to them. Scoped to that +# reason deliberately: other corpus names' facade initials move +# against 1.4.0 for other reasons and are no part of the six -- +# 'e and e' ('e a e.' -> 'e. a. e.') and 'juan garcia y lopez' +# ('j. g l.' -> 'j. g. l.'), both measured from the wheel the same +# day, move on the 2.0.0 per-word GROUPING the `fix(initials-per-word)` +# rules below describe. decisions.md#P3 records the split and +# decisions.md#R3 its closing. # # THE #528 RULE SITS AHEAD OF THE VOCABULARY RULES, with the literal # name lists, and the position is load-bearing rather than tidy: the @@ -2891,18 +2903,18 @@ issue = "fix(#528) the facade's initials follow the parse's connective tags" # wholly in one case a marked 'e' is an initial and an unmarked 'y' # joins -- so from 2.4's #527 until #528 the two views of one parse # disagreed about the same letter. They agree now, and the facade's -# answer is the core's: 'j. s.' -> 'j. e. s.' on the three e-names -# (1.4.0's values measured from the wheel, 2026-09-13) and 'J. Y. G.' -# -> 'J. G.' on the capital one. No role moves on any of the four -- -# P3's three-word carve-out is untouched -- which is exactly why the -# derived view is the whole diff here. -# -# LITERAL, four names, and the four are not the change's whole -# population: six corpus names move `HumanName.initials()`, and -# 'jose e maria santos' and 'JUAN GARCIA Y LOPEZ' move their ROLES -# against this baseline as well, so `_initials` never enters their -# diff and the #383/#479 role rule at the end of this file explains -# them. A regex for the class -- "a one-case name whose single-letter +# answer is the core's. Name by name, with 1.4.0's values measured +# from the wheel on 2026-09-13: 'john e smith' goes 'j. s.' -> +# 'j. e. s.'; 'john e jones' and 'jones, john e' both go 'j. j.' -> +# 'j. e. j.', the two e-names whose family initial is a `j` and not +# an `s`; and 'JUAN Y GARCIA' goes 'J. Y. G.' -> 'J. G.', the +# capital one losing a letter where the other three gain one. No +# role moves on any of the four -- P3's three-word carve-out is untouched -- which +# is exactly why the derived view is the whole diff here. +# +# LITERAL, four names; which two of the change's movers they leave +# out, and why, is the block header above. +# A regex for the class -- "a one-case name whose single-letter # connective the parse read against 1.4.0's reading" -- is not # writable over the raw string at all, the reading being the parse's. # _MUST_NOT_MATCH in tests/v2/test_ledger_guards.py carries the @@ -2917,12 +2929,7 @@ issue = "fix(#528) the facade's initials follow the parse's connective tags" # Two of the four -- 'john e smith' and 'john e jones' -- were # EXPLAINED BY THE CONNECTIVE RULE BELOW before this rule existed # (measured 2026-09-13 at this baseline: both sat in its classified -# list). Its regex reaches them through their lowercase ' e ', its -# `fields` are these, and equal `fields` is not a `precedes_narrower` -# pair -- so file order is the whole instrument and this rule takes -# them by sitting first. That is the right answer on the merits: what -# moved on those two names is the READING of the letter, not the -# per-word GROUPING of an element that rule describes. +# list). Why this rule takes them instead: the block header above. # _CROSS_RULE_WINNERS pins it and _RECORDED_DIFFS carries the shapes. name_regex = "^(?:john e smith|john e jones|jones, john e|JUAN Y GARCIA)$" fields = ["_initials"] diff --git a/tools/differential/expected_since_2.0.0.toml b/tools/differential/expected_since_2.0.0.toml index dae04fc1..84c64ae7 100644 --- a/tools/differential/expected_since_2.0.0.toml +++ b/tools/differential/expected_since_2.0.0.toml @@ -1806,10 +1806,11 @@ issue = "fix(#462) the facade keeps an initial-shaped conjunction letter" # contributes no initial, 'J. Y. G.' -> 'J. G.' (rules.md#P3 for # which letters are connectives and rules.md#R3 for what a connective # contributes). #528 then made the facade read the parse's tags like -# the core, so the facade's half CLOSED: the wheel gives 'J. G.' and -# the tree gives 'J. G.' again, agreeing with this baseline, and the -# core's half is the whole diff. Measured 2026-09-13 on the 2.0.0, -# 2.1.0 and 2.2.0 wheels and on the tree. +# the core, so the facade's half CLOSED: the WHEEL'S FACADE gives +# 'J. G.' and the tree's facade gives 'J. G.' again, agreeing with +# this baseline, and the core's half is the whole diff -- the wheel's +# CORE gives 'J. Y. G.', which is the half that still moves. Measured +# 2026-09-13 on the 2.0.0, 2.1.0 and 2.2.0 wheels and on the tree. # One rule still takes it, for the reason two halves needed one: # `_initials` is ONE field, and this rule's regex reaches the name. # That is also why #383/#479's section at the end of this file writes From 01ca5b5eaeda7542644771c91aec27685994c421 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 13 Sep 2026 22:11:12 -0700 Subject: [PATCH 03/11] docs(design+release): the record for #528, and the two initials views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decisions.md gains the R3 entry for #528: the facade's initials view reads the parse's tags, in R4's shape -- a word backed by a token asks its tags, a token carrying UNCLASSIFIED_TAG was read by no parse and the vocabulary answers through the same helper _cap_word calls. Its two accepted costs are measured and pinned: the connective answer is frozen at parse time, so a C.conjunctions edit waits for the next full_name assignment, and an unpickled name is spliced text throughout and takes the fallback, both as capitalize() has behaved all along. The stated private-method break is recorded with the remedy that actually works -- an override of _process_initial must ACCEPT `tokens` AND FORWARD it, since widening the signature alone leaves the override reading the token path's empty name_part and initials() comes back "" without raising. Measured, and now pinned by test_an_override_that_forwards_tokens_keeps_working with the widen-only spelling as its negative control. The entry also carries the one corpus name whose two views still differ ('Ph. D., John', the 2.0.0 per-word merge and not this change) and the ledger picture at all five baselines. P3's facade paragraph, which deferred this to a follow-up, is closed out in place and points here. rules.md#R3 and #R4 now distinguish the TWO initials views. The two agree on which words initial in a parsed name; what still differs is GROUPING, the facade initialing a joined run as one element. And on a spliced field the facade falls back on BOTH questions where the parsed name's own view falls back on neither -- the connective question through case repair's helper, the particle question through a live lookup, so a family spliced to "de la vega" initials "j. v." on one view and "j. d. l. v." on the other. mechanisms.md: FOLDED_TAG credits the walk that does the folded-first ordering rather than the string view built on it; VOCAB-TAGS adds the facade as a tag consumer, a module a sweep stopping at _render.py misses; RENDER-HONORS-THE-PARSE gains the second reader of the vocabulary fallback and RETRACTS, for the facade view only, its old "nothing observable" conclusion about the _INITIAL sync divergence -- with `conjunctions.add("太")`, 'Wang Chen 太. Li' initials 'W. C. L.' parsed and 'W. C. 太. L.' with the identical family text spliced in, while capitalized(force=True) shows nothing either way. Accepted rather than repaired, the remedy differing by view: revise() for the parsed name, a full_name reassignment for the facade, which has no revise. usage.rst's twin sentence is scoped to the parsed name's own views. docs/release_log.rst gains the 2.4.0 Fix bullet for #528 -- with the forwarding remedy, and with the 2.0.0-2.2.0 values stated rather than folded into "1.4.0 through 2.3.0", which is false for the two capital names -- and the #527 bullet's now-wrong facade sentence is corrected in place. tools/differential/README.md's "32 of the 45 today" becomes the dated form; the rows are 47 since #528. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 9 ++++++++- docs/design/mechanisms.md | 6 +++--- docs/design/rules.md | 29 ++++++++++++++++++++++++++--- docs/release_log.rst | 4 +++- docs/usage.rst | 12 ++++++++---- nameparser/_facade.py | 7 +++++++ tests/v2/test_facade.py | 34 +++++++++++++++++++++++++++++++++- tools/differential/README.md | 2 +- 8 files changed, 89 insertions(+), 14 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 73120af2..34a79f29 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -327,7 +327,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf CASELESS LETTERS NEVER ENTER THE FORK. Arabic و has no case, so `token.upper() != token.lower()` is false and today's reading stands. A caseless-script input counts as "one case" under the helper, harmlessly, because the fork also requires a cased token. NO SWITCH. The subset is the knob: remove "e" to restore the joining reading, add "y" for a Dutch-style "every single letter is an initial". That is also this shape's answer to #516's switch question. ACCEPTED, AND RECORDED RATHER THAN FIXED: a locale pack cannot express "remove e". `Locale.lexicon` is unioned onto the base and never removes — its own field comment says "a pack never removes base vocabulary" — so an `nl` pack CAN add "y" while a `pt` caller has to write `Parser(lexicon=Lexicon.default().remove(conjunctions_ambiguous={"e"}))` by hand. The first (A) bullet added to #3-0-reevaluations on this date asks whether packs should be able to remove vocabulary at all. - ACCEPTED, AND DEFERRED TO A FOLLOW-UP ISSUE (Derek, 2026-09-13): the facade and the core now disagree about the same letter. `HumanName.initials()` does not follow the parse's tags — `_facade._process_initial` re-derives "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stays "j. s." while `parse("john e smith").initials()` gives "j. e. s.", and `JUAN Y GARCIA` splits the other way, facade "J. Y. G." against core "J. G.". Both measured here 2026-09-13 and pinned by a contrastive test rather than left to prose. `_facade.py` is untouched in this PR; the fix has R4's shape — consult the PARSED token wherever the part maps to one and fall back to the vocabulary only where it does not — and it is a follow-up issue, not yet filed, so no number is cited here. `capitalize()` is not split: it follows the parse on both surfaces, so `HumanName("john e smith").capitalize()` gives "John E Smith" too. + ACCEPTED AND THEN CLOSED, SAME DAY (Derek, 2026-09-13): the facade and the core disagreed about the same letter for the length of one PR. `HumanName.initials()` did not follow the parse's tags — `_facade._process_initial` re-derived "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stayed "j. s." while `parse("john e smith").initials()` gave "j. e. s.", and `JUAN Y GARCIA` split the other way, facade "J. Y. G." against core "J. G.". `_facade.py` was untouched in the #383/#479 PR by decision, the split pinned by a contrastive test rather than left to prose, and #528 filed for it. #528 shipped the fix this paragraph predicted, with the R4 shape it predicted: the facade consults the PARSED token wherever the part maps to one and falls back to the vocabulary only where it does not. Both views give "j. e. s." and "J. G." now (measured 2026-09-13 after the fix); the contrastive test became the agreement test `tests/v2/test_render.py::test_facade_initials_follow_the_one_case_fork`. `decisions.md#R3`'s 2026-09-13 entry carries the fix, its accepted costs and the one corpus name where the two views still differ. `capitalize()` was never split: it followed the parse on both surfaces throughout, so `HumanName("john e smith").capitalize()` gave "John E Smith" all along. TWO OTHER CAUSES SHARE THE `_initials` FIELD IN THE LEDGERS AND NEITHER IS THIS PR'S, recorded because a reader meeting them under these names will reach for this entry. The Arabic `محمد و علي` diffs on `_initials` at 1.4.0 only (`م. و. ع.` → `م. ع.`), measured byte-identical either side of the fork: it is a pre-existing #269 consequence — a recognized non-Latin connective contributes no initial — surfaced by the row entering the contract corpus, and `expected_since_1.4.0.toml` ledgers it as `feat(#269)`. `JUAN Y GARCIA` diffs on `_initials` at 2.0.0 through 2.2.0 for a DIFFERENT reason than it does at 2.3.0: at those three baselines `fix(#462)` already admits the name (the facade moved `J. G.` → `J. Y. G.` there), and this PR's core move `J. Y. G.` → `J. G.` gets its own rule at 2.3.0 alone. Two causes, one field, documented in a dated paragraph on `fix(#462)` in those three ledgers rather than as a competing rule. Out of scope, each its own issue: #492 (whether a cased suffix token counts as case evidence — `is_one_case` is written so R5 and #492 can share it later, but render does not import it here); #478 (hyphenated connective repair — its spaced-form claim now depends on "y" staying OUT of the subset); #461 (R3's clause); #289 and #516 (the same case-class fact read at the suffix and post-comma slots); the render-side conjunction fallback for spliced raw text, which stays vocabulary-keyed as rules.md#R4 states. @@ -1180,6 +1180,13 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea WHY R3 GAINED A SENTENCE rather than leaning on R1. R1 already says folded family words "render before the rest of the family wherever they stood in the string" — but its subject is "Every FIELD", and `initials()` is not a field: R3 calls it "this view", and it takes a format spec and two separators rather than being read as an attribute. So R1 does not reach it by its own words, and #408 is precisely what leaving that to inference costs. R3's new clause states the order in R3's own voice, carries the O3 example line (`"Hassan, Mohamad Ahmad Ali"` under `middle_as_family` → `initials="M. A. A. H."`, reusing R1's own input so `corpus_rules.jsonl` is unchanged at 241 names), and R1 and R3 now name each other in `interacts:` — the field says which order, the view says it follows the field. P6's half gets no example line, for the same reason R1 carries none: the only input that witnesses it is `der, y van`, whose PARSE is itself contested, so pinning its initials normatively would pin a value another open question can move. It is pinned in the unit test instead, which is where today's-behavior pins belong. THE PARTITION IS PER ROLE, not scoped to FAMILY, and this is a deliberate choice rather than a copied line. Both producers re-role to FAMILY, so FAMILY is the only role a parse can put the tag on today and the GIVEN and MIDDLE arms are unreachable. They are there because `_text_for` partitions for every role it renders and the two views must not diverge again — the same uniformity `_post_rules`' own UNJOINED_TAG loop takes for its three roles, "for uniformity with the rule, not because all three are observable". A producer that ever folded into another part would otherwise reopen #408 there with nothing to fail. `test_initials_folds_in_every_role_it_renders` pins it from a hand-built name, since no input string can. Mutation-checked four ways — scoping the partition to FAMILY, and skipping it for GIVEN, for MIDDLE, for FAMILY — and each fails that test. It took THREE drafts of the fixture to make that true, and the two misses are the same miss one role apart. The first carried two GIVEN tokens and ONE MIDDLE: a one-element partition is the identity, so the MIDDLE arm was asserted and unpinned, and skipping the partition for MIDDLE passed the entire suite. The second fixed MIDDLE and carried NO FAMILY token at all, in a test named for every role it renders — so skipping the partition for FAMILY passed this test, and was caught only by its siblings and by R3's example line, neither of which is about per-role application. A zero-element group is the identity too, and reads even less like a gap than a one-element one, which is why the second miss survived a review that had just named the first. Two tokens per role is what closes it, and the general form is carried as mechanisms.md#TWO-ELEMENT-GROUPS rather than left here: a test written to pin a partition, a sort or a dedup needs at least two elements in every group it claims to cover, or the claim rides on a no-op. - 2026-09-01 #462 — DONE, the facade's twin: `HumanName.initials()` dropped a dotted or bare-capital `E`/`Y` from the middle and family groups because `_process_initial` tested conjunction membership alone, where 1.4.0's `is_conjunction` was "in the set AND NOT `is_an_initial`". `Scott E. Werner` gave `S. W.` from 2.0.0 through 2.2.0 and gives `S. E. W.` again; `John E Smith` (bare capital) and `Juan Y. Garcia` likewise; `parse().initials()` never had the bug, since `_classify` tags `E.` an initial and the render reads the tag (mechanisms.md#RENDER-HONORS-THE-PARSE). The issue said the bare form was "still correctly dropped", which is true of lowercase `e` and false of bare capital `E` — 7 of the 14 corpus names that move are bare capitals. Restored with `_render._INITIAL`, v1's own `initial` shape, because the facade may import `_render` and not `_pipeline`; scoped to `_process_initial`, its only caller, so a future reader of `_is_conjunction` does not inherit a decision made for initials. Found by #484's pseudo-field at the 1.4.0 baseline, where it had sat as 14 unreported diffs; the gate could not see it before because the seven fields never moved. +- 2026-09-13 #528 — DONE: `HumanName.initials()` reads the parse's tags. The facade's initials view was the last derived view still re-deciding the CONNECTIVE question after the parse had answered it — scoped to that question deliberately, since the PARTICLE question is still re-decided from the lexicon in two places by decision, `_cap_word`'s particle conjunct (`decisions.md#R4`'s NOT DONE bullet) and `_is_particle` here, R4 drawing that boundary per question rather than per view. #462's fix above is why the connective half was still being re-decided: restoring v1's `is_conjunction` ("in the set AND NOT `is_an_initial`") restored a SHAPE test standing in for a tag, which agreed with the parse only while the parser agreed with the shape. #383/#479 ended that the same day it landed — rules.md#P3 now reads a one-case single-letter connective from vocabulary rather than from case, and no shape over the raw word can see it — so the two views of one parse disagreed: `HumanName("john e smith").initials()` gave "j. s." against the core's "j. e. s.", and `HumanName("JUAN Y GARCIA").initials()` gave "J. Y. G." against the core's "J. G.". Both give the core's answer now. This is mechanisms.md#RENDER-HONORS-THE-PARSE applied to the last view that had not taken it, and #458's principle reaching the facade layer. + THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. + A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; nothing on the public v1 surface moves except the two initials values above. + ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. + ACCEPTED COST, THE SECOND ONE, and it is the pickle path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. A round-tripped `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the pickle is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_name_ask_the_vocabulary_too`. + ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. + DIFFERENTIAL. The facade's `_initials` is compared at every baseline, 1.4.0 included, and the core's from 2.0.0; the pseudo-field enters a name's diff only where every role and every ambiguity kind agrees. So at 1.4.0 FOUR of the six take a new rule (`fix(#528) the facade's initials follow the parse's connective tags`) and two do not — `jose e maria santos` and `JUAN GARCIA Y LOPEZ` move roles against that baseline, and #383/#479's role rule explains them. At 2.0.0 through 2.2.0 nothing new: the e-names' `_ambiguities` diff keeps `_initials` out of their diff, and `JUAN Y GARCIA`'s single `_initials` row survives with the facade half of its two causes closed, which `fix(#462)`'s dated paragraph in `expected_since_2.0.0.toml` now says. At 2.3.0 the existing rule stands and both surfaces move together. All five gates re-run at 0 unexplained on 2026-09-13. ### R4 — case repair reads the unjoined mark diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 966e76a4..8ea2697c 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -15,13 +15,13 @@ Problem shape. A later stage needs to refer to "that word." Contract statement. ## FOLDED_TAG — reorder at render time, not parse time -Problem shape. A rule wants words to RENDER in a different order than they sit in the string. Contract statement. Tokens never move: a rule that needs different rendering order tags the token, and the rendering views consult the tag — EVERY view that renders the affected role, ordering folded tokens first, not the field views alone. How it works. Reordering the token tuple would break span math and reintroduce the #100 family. Parse state stays in string order; only the view reorders (rule R1, rule R3's order clause, rule O3's render clause). The consumer list is where this mechanism fails, in two opposite directions — and only ONE of them has ever shipped, which is worth keeping straight because a roster is walked differently when it is guarding against a defect than when it is guarding against a possibility. SHIPPED: a consumer that never read the tag (`initials()` walked written order through 2.0, 2.1 and 2.2 — #408, filed as an instance of RENDER-HONORS-THE-PARSE and fixed 2026-08-30). HAZARD, never released: a consumer that reads the tag where it should not (the revise strip below), which arrived with `Parser.revise` in the same commit as the test that pins it, so no version has gone out without it and no issue, decision entry or test names a defect of that shape. Adding a producer is cheap; adding a view is where the roster has to be walked. Lives in. nameparser/_types.py (FOLDED_TAG, `_text_for` — every role it renders, not FAMILY alone), nameparser/_render.py (`initials`, the same partition per role), nameparser/_facade.py (`_list_for`, the v1 `*_list` views, which prepend the carriers the same way), nameparser/_pipeline/_post_rules.py (its two producers, O3's fold and P6's attachment, both of which re-role to FAMILY — so FAMILY is the only role a parse can put the tag on today, and the views partitioning every role is uniformity with this contract rather than reachable behavior), and nameparser/_parser.py (`Parser.revise`, the one deliberate CONSUMER-side strip: it removes the tag from the tokens it harvests, a revised value not being allowed to inherit fold ordering) — losing that strip is this mechanism's measured hazard, a family rendering "García Gabriel Márquez". A module path and not a method name, because this roster is what a reader walks when adding a view, and a bare `Parser.revise` sends them looking in the wrong file. Reach for it when. A new rule needs "X renders before Y" and you are tempted to swap tokens. Don't swap. Tag. +Problem shape. A rule wants words to RENDER in a different order than they sit in the string. Contract statement. Tokens never move: a rule that needs different rendering order tags the token, and the rendering views consult the tag — EVERY view that renders the affected role, ordering folded tokens first, not the field views alone. How it works. Reordering the token tuple would break span math and reintroduce the #100 family. Parse state stays in string order; only the view reorders (rule R1, rule R3's order clause, rule O3's render clause). The consumer list is where this mechanism fails, in two opposite directions — and only ONE of them has ever shipped, which is worth keeping straight because a roster is walked differently when it is guarding against a defect than when it is guarding against a possibility. SHIPPED: a consumer that never read the tag (`initials()` walked written order through 2.0, 2.1 and 2.2 — #408, filed as an instance of RENDER-HONORS-THE-PARSE and fixed 2026-08-30). HAZARD, never released: a consumer that reads the tag where it should not (the revise strip below), which arrived with `Parser.revise` in the same commit as the test that pins it, so no version has gone out without it and no issue, decision entry or test names a defect of that shape. Adding a producer is cheap; adding a view is where the roster has to be walked. Lives in. nameparser/_types.py (FOLDED_TAG, `_text_for` — every role it renders, not FAMILY alone), nameparser/_render.py (`initials`, the same partition per role), nameparser/_facade.py (`_list_tokens_for`, which prepends the carriers the same way — it is the ONE walk behind both the v1 `*_list` views and the facade's initials since #528, and `_list_for` is the string view built on it and holds no ordering of its own, so a reader who follows this roster to the string builder lands one level below the partition), nameparser/_pipeline/_post_rules.py (its two producers, O3's fold and P6's attachment, both of which re-role to FAMILY — so FAMILY is the only role a parse can put the tag on today, and the views partitioning every role is uniformity with this contract rather than reachable behavior), and nameparser/_parser.py (`Parser.revise`, the one deliberate CONSUMER-side strip: it removes the tag from the tokens it harvests, a revised value not being allowed to inherit fold ordering) — losing that strip is this mechanism's measured hazard, a family rendering "García Gabriel Márquez". A module path and not a method name, because this roster is what a reader walks when adding a view, and a bare `Parser.revise` sends them looking in the wrong file. Reach for it when. A new rule needs "X renders before Y" and you are tempted to swap tokens. Don't swap. Tag. ## VOCAB-TAGS — the vocabulary layer speaks once Problem shape. A later stage needs to know what the vocabulary knew about a word. Contract statement. classify tags every token with what the vocabulary knows about it, and later stages test tags — they never re-look a word up. How it works. One lookup site means one answer: a stage that re-derived vocabulary facts could disagree with the stage before it. Stable tags ("particle", "conjunction", "initial") are API; "vocab:"-namespaced ones are not. -Lives in. nameparser/_pipeline/_classify.py (producer); consumers throughout _group/_assign/_post_rules, and nameparser/_render.py, which is not a stage but reads the same tags to decide what a view shows (#458 moved case repair's conjunction test onto the tag; initials() had read them since 2.0). Reach for it when. A stage is about to import Lexicon to ask about a word classify already saw — or a view is, which is the harder one to notice, since a view legitimately holds a Lexicon for the questions classify never answered; the view side is RENDER-HONORS-THE-PARSE, which owns it and covers the decisions this entry does not produce. +Lives in. nameparser/_pipeline/_classify.py (producer); consumers throughout _group/_assign/_post_rules, and nameparser/_render.py, which is not a stage but reads the same tags to decide what a view shows (#458 moved case repair's conjunction test onto the tag; initials() had read them since 2.0), and nameparser/_facade.py, whose `_token_is_conjunction` joined them in #528 — the v1 layer is a tag consumer too now, so a sweep that stops at `_render.py` is one module short. Reach for it when. A stage is about to import Lexicon to ask about a word classify already saw — or a view is, which is the harder one to notice, since a view legitimately holds a Lexicon for the questions classify never answered; the view side is RENDER-HONORS-THE-PARSE, which owns it and covers the decisions this entry does not produce. ## PIECES — joining structure survives assignment @@ -59,7 +59,7 @@ Problem shape. Two stages need the same answer about the same input, and the one ## RENDER-HONORS-THE-PARSE — the parse decides it, the views honor it -Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which is filed and open here. How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is OPEN rather than closed: `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings give `A. V.` and `J.`, and the second is a rules.md#R3 example line the doc runner asserts. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and exactly one is: `capitalized(lexicon=...)`. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). `initials()` is the near miss and the instructive one: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). `capitalized()` guesses nothing: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within the one view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach is now nothing observable: it needs a caller-added conjunction written initial-SHAPED in a script that has no initials (`씨.`), and case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. `initials()` used to be the reader that could witness it, and no longer falls back at all. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py (the v1 pickle load, the SECOND producer of that mark — it is named in this list because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect), reading what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. +Problem shape. A render view needs a fact the parse already settled — whether a word is the conjunction or an initial, whether a particle is acting as one, which word renders first. Contract statement. The parse decides it; the render views honor those decisions and never re-evaluate them. Two directions break that, and each has been found here as a defect: a view RE-DERIVES the answer from the text, keeping its own copy of a pipeline predicate — that one shipped through 2.0 and 2.1 — or a view honors the record and then OVERRIDES it, readmitting what the decision excluded, which is filed and open here. How it works. Re-deriving fails because the two copies stop being the same question long before anyone notices they are two: `_cap_word` re-ran the conjunction-versus-initial decision from the word's spelling against a hand-maintained copy of the pipeline's `_INITIAL` pattern while classify had already answered it and recorded it on the token, and `_classify.py` asks `is_initial()`, the shape test ANDed with a script-repertoire test since #320, where `_render.py` asked the bare pattern, and asked it per WORD of a token's text rather than per token, so `juan e-f smith` repaired to `Juan e-F Smith` (#458). Overriding fails more directly, and this entry's instance of it is OPEN rather than closed: `initials()` honors the `conjunction` tag through `_SKIP_TAGS` and then readmits the token whenever it also carries UNJOINED_TAG, the mark of an all-particle part, where rules.md#R3 excludes a conjunction "even then" — so under a caller's vocabulary that puts a word in both sets the view readmits what the rule excluded, and under `Lexicon.default().add(particles={'y'})` — the lexicon those two readings need, and they hold under no other — `parse("Anh y Van")` initialed `A. y. V.` and `parse("Juan de y")` `J. d. y.` on 2026-08-29. Read those two values scoped to that lexicon or they are simply false: under the DEFAULT vocabulary the same strings give `A. V.` and `J.`, and the second is a rules.md#R3 example line the doc runner asserts. #461 is where that stands, and it is worth reading before re-fixing it: the narrowing was written, measured and BACKED OUT in the same PR, because honoring R3 there cost `initials()` its agreement with `family_base`, which reads that same `de y` as the base — this entry's other shape, arrived at from the render side, and the reason the question moved from the code to R3's clause (decisions.md carries the argument). What to carry away is the DIRECTION and not its verdict here: a view that honors a record and then readmits what the record excluded is overriding a decision it never took, whichever way this one settles. Case repair reads that same token and does NOT readmit it — `capitalized(force=True)` on `Anh y Van` gives `Anh y Van` under that same lexicon, R4 carrying the carve-out in its own words but ON R3's authority — its text reads "being no name word in any part — the carve-out R3 states for initials", so a change to R3's clause reaches R4's TEXT. What does NOT follow, though an earlier wording of this sentence asserted it, is that the two stand or fall together in BEHAVIOR: they have already come apart, over the 25 corpus names carrying a conjunction in the GIVEN group — `parse("john and jane smith").capitalized()` keeps `and` lowercase, so R4's carve-out holds there, while `.initials()` gives `j. a. j. s.`, so R3's does not (decisions.md#R2 carries that population, and rules.md#R3 now says so in its own words). The dependency is textual, and only textual — so the two views disagree today about that token exactly as they did before #461 and the backout restores that disagreement knowingly; only the FORCED call witnesses the repair half, R5's gate refusing a mixed-case name before any of this is consulted. This is the CONSUMER-side rule over the producer-side entries — VOCAB-TAGS records what the vocabulary knew, MARK-DONT-STRIP what a stage decided about it, FOLDED_TAG what order to render in — and a view reads what they recorded, whichever kind it is. It sits where ONE-PREDICATE-PER-QUESTION's stated limit leaves off: where two live sites need one answer they share a predicate, but a render view always comes AFTER the decider, so the answer is recorded rather than shared — on the TOKEN, which is the views' equivalent of that entry's `ParseState.order`, no view being able to see a ParseState at all. Known limit, and the half most easily got wrong next: a token the parse never saw carries no decision to honor, so a view falls back to the vocabulary — the tell is `UNCLASSIFIED_TAG`, which `ParsedName.replace()` stamps when it splices raw text into a field and the facade's v1 pickle load stamps when it rebuilds a name from `*_list` strings. It is NOT untaggedness, since an ordinary parsed name word carries no tags either; and it is NOT `span is None`, which was tried and is wrong in the other direction — span-less means SYNTHETIC, and `Parser.revise()` builds span-less tokens from a full sub-parse whose tags it keeps on purpose, so the span reading overrode exactly the tags `revise()` exists to preserve (`revise(middle='e-f')` repaired to `e-F` where the parse gave `E-F`). A hand-built span-less token is unmarked and therefore classified, which is the same tag-driven default every other view applies. A view can only fall back if it is HANDED a vocabulary, and TWO are: `capitalized(lexicon=...)`, and — since #528 — the v1 facade's `HumanName.initials()`, which holds the bound `Lexicon` its `Parser` was built from and so never has to guess one. `family_base` and `family_particles` are properties on ParsedName, whose fields are original/tokens/ambiguities and nothing else, so a spliced field empties the particles view and leaves the base the whole field, with `Parser.revise()` the crossing there too (docs/usage.rst says so where it documents the degradation). The CORE `ParsedName.initials()` is the near miss and the instructive one, and it is still the near miss: it is a METHOD, so it looks like it could ask, but its signature is `(spec, delimiter, separator)` and carries no lexicon — a fallback there was written and dropped because it had to GUESS `Lexicon.default()`, and the guess erased a whole field under a caller's own vocabulary (decisions.md#R4). Read the two initials views apart wherever this entry says "initials", because #528 made them differ exactly here: the core cannot fall back and the facade does. `capitalized()` guesses nothing either: it reads the lexicon it was handed, and only defaults to `Lexicon.default()` when the caller passes none, which is the documented meaning of omitting the argument rather than a fallback. Within a view that can fall back, the fallback is drawn per QUESTION (rules.md#R4's Accepted clause). Whether a word is the conjunction or an initial is a property of the word, which a vocabulary answers alone, so case repair asks it. Whether a part is wholly particles is a property of the whole PART, which the pipeline answers once and records as UNJOINED_TAG. What `_cap_word` cannot do is RE-DERIVE that answer where no word of the part carries a tag — it is handed the whole token's tags and gates the particle conjunct on UNJOINED_TAG, so it is not blind to the part, it simply has no evidence to reconstruct one from — so repair leaves that half to plain particle treatment and rules.md#R4's Accepted boundary records the consequence — a spliced field is not repaired as a parsed one is, with `Parser.revise()` as the supported crossing. A fallback is right only while it answers as the pipeline would, and THAT is held by hand rather than mechanically: test_regex_sync pins the two `_INITIAL` copies to each other and to config, while the repertoire half of the pipeline's predicate (#320) is deliberately not carried across, layering forbidding the import. What that divergence can reach was argued to be nothing observable, and #528 RETRACTED that for the facade view while leaving it standing for the core. The shape it needs is a caller-added conjunction written initial-SHAPED in a script that has no initials (`太.`, `씨.`), and the old argument was that case repair is the fallback's only reader, so the two paths differ by `lower()` versus `capitalize()` over a caseless script — the same string either way. The facade's `HumanName.initials()` is a second reader now, and it CAN show the difference, because a word the fallback calls a connective contributes no initial while one it does not contributes a letter. Witness, measured 2026-09-13 under `Constants()` with `conjunctions.add("太")`: `HumanName("Wang Chen 太. Li").initials()` is "W. C. L." — classify tags the `太.` `conjunction` in the family and the view honors it — while splicing the identical family text in, `h.last = "Chen 太. Li"`, gives "W. C. 太. L.", the fallback reading `太.` as initial-shaped and admitting it. Same string in the field, two answers, and `capitalized(force=True)` shows nothing at all on either. ACCEPTED rather than repaired, on this entry's own terms: the fallback path is only ever reached for text no parse read, a spliced field is the caller's own text, and it is answered by the SAME helper R4 hands case repair — so the remedy is the documented crossing, WHICH DIFFERS BY VIEW and must not be copied across: for the parsed name's views it is `Parser.revise()` (rules.md#R3's Accepted clause, decisions.md#R3), while the facade has no `revise` at all — its setters splice through `replace()` by decision — so the v1 remedy is the v1 one, parsing the whole string again by assigning `full_name`, which restores "W. C. L." on the name above (measured 2026-09-13 with the rest of this witness). What stays true for the core is the sentence that used to close this: `ParsedName.initials()` was once the reader that could witness it and no longer falls back at all, being handed no vocabulary. A second limit, recorded rather than closed: `_cap_word`'s PARTICLE conjunct still keys on the lexicon handed to the view rather than on the `particle` tag, so a repair run with a lexicon other than the parse's re-decides a word the parse already read — a name parsed under the default vocabulary, where `parse('juan smith vega')` reads `vega` as the family, repairs to `Juan Smith vega` when `capitalized()` is handed `Lexicon.default().add(particles={'vega'})` instead — the divergence needs the two lexicons to differ, and repairing under the parse's own lexicon gives `Juan Smith Vega`. Making it read the tag moves a boundary rules.md#R4 states in prose, so it is a separate decision and not a cleanup (decisions.md#R4, "NOT DONE"). Lives in. nameparser/_render.py (`capitalized`/`_cap_word`, `_reads_as_conjunction`, and `initials`, the CORE view, which honors tags and never falls back) and nameparser/_types.py (`_text_for` and `UNCLASSIFIED_TAG`, with the `ParsedName.replace()` producer beside it) and nameparser/_facade.py, which is on this list TWICE and for opposite reasons: the v1 pickle load is the SECOND producer of that mark — named here because a change that follows the list into `_types.py` alone leaves it behind, which is the site test_a_restored_pickle_keeps_v1_conjunction_repair exists to protect — and since #528 `_token_is_conjunction`/`_process_initial` are a CONSUMER, the facade's initials view reading the tag and calling `_render._reads_as_conjunction` for the mark's own tokens. All of them read what nameparser/_pipeline/ recorded — the mark those views read is recomputed producer-side in `_remarked`, which is deliberately silent about text nobody classified and is right to be. Reach for it when. A view is about to consult a Lexicon, a regex or an exception list about a word the parse already saw — or a view and a field disagree about the same parse. #408 was that second shape, and is CLOSED (2026-08-30): `initials()` walked tokens in written order where the family field applies FOLDED_TAG's ordering, so `parse("der, y van")` gave family `van der` and initials `y. d. v.` on 2026-08-29 and gives `y. v. d.` now, the view reading the tag as `_text_for` does. Worth keeping as the worked instance of the shape rather than deleting with the fix, and worth two notes on how it read once measured. The disagreement was not a judgment call anyone had taken: the FACADE already ordered folded-first through its own `*_list` views, so the core view was out of step with the field, with v1, and with the facade at once, and nothing in 6125 tests touched it. And where the change has a v1 reference at all it RESTORES rather than deviates, which is not what this entry's other instances have been -- but read that SCOPED to the population it was measured over, because an unscoped version of this sentence stood here until 2026-08-30 and overstated in both directions. Only the two DEFAULT-ORDER policies have a v1 reference: v1 had `middle_name_as_last` and no general `name_order`, so 588 of the 660 moving parses -- the two family-first orders -- restore nothing and break nothing, there being no v1 answer to come into or leave (decisions.md#R3 says the same and carries the rest of the measurement). Where the reference does exist the claim is exact and worth keeping: over the 1094-name corpus at the default order, 71 names move under `middle_as_family`, of which 54 return to 1.4.0's answer and none leaves it. And it is a claim about THOSE 71 rather than about every name the fix touches -- this entry's own lead example is the counterexample, `parse("der, y van")` giving `y. d. v.` before and `y. v. d.` after where 1.4.0 gives `y.`, v1 contributing nothing at all for a family that is all particles (rules.md#R2's territory, and a divergence decisions.md#R2 has already decided in favor of). A view that stopped honoring a record had been quietly reproducing a v1 bug that v1 did not have. ## CLAUSE-CONTENT-OVERRULES-DELIMITER — content wins diff --git a/docs/design/rules.md b/docs/design/rules.md index 7fcfea7a..ed958540 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1736,6 +1736,23 @@ R3. Rationale: initials abbreviate the person's name words; titles, and matches the parse in both views. Stated without an example line because every line here names an input string, and this shape needs a field edited after the parse. + The v1 facade's HumanName.initials() is a second view of this + question and IS handed a vocabulary: it reads the parse's reading + wherever a word is backed by a parsed token, and falls back + wherever a word is not — a field set as raw text, or a name + restored from a v1 pickle. It falls back on BOTH questions there, + not one: the connective question through the same helper case + repair uses, and the particle question through a live vocabulary + lookup, so a family spliced to "de la vega" initials "j. v." on + this view against "j. d. l. v." on the parsed name's own. So the + two views agree on WHICH WORDS initial in a parsed name; what + still differs there is GROUPING, the facade initialing a joined + run as one element, which is where the name "Ph. D., John" gives + a run-together "J. P D." on this view against "J. P. D." on the + other. Stated in prose and not as example lines because both + shapes need a field edited after the parse, or a rendering the + other view does not have. decisions.md#R3 carries what all of it + costs and where it is pinned. Accepted: the unsettled given-group answer above is neither rare nor hypothetical — 26 of the corpus names carry a conjunction among the given names (measured 2026-09-13; recompute by parsing @@ -1802,9 +1819,15 @@ R4. Rationale: case repair is a display concern, applied only on needs a reading on every word of the part, and a spliced field has none on any, so that half falls through to particle treatment and the "de la" boundary above stands. Initials are the contrast - worth knowing, and R3 states it: that view is handed no - vocabulary at all, so it falls back on neither question and a - spliced field's every word initials. revise() classifies the + worth knowing, and R3 states it — but ask which initials view, + because the two answer oppositely. The parsed name's own view is + handed no vocabulary at all, so it falls back on neither question + and a spliced field's every word initials. The v1 facade's view IS + handed one and falls back on BOTH: the connective question through + the helper this rule uses, and the particle question through a + live vocabulary lookup, so a family spliced to "de la vega" + initials "j. v." there against the parsed view's "j. d. l. v.". + revise() classifies the value and crosses both questions, in both views: a middle revised to "e-f" repairs to "E-F" as the parsed name does, where splicing the same text in gives "e-F". diff --git a/docs/release_log.rst b/docs/release_log.rst index 8a2050f9..87ca3cd7 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -6,7 +6,9 @@ Release Log **Behavior Changes** - - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. On those two short names ``HumanName.initials()`` is unchanged -- the facade reads the letter by vocabulary and written shape rather than by the parse, and a follow-up issue carries the split -- but where a ROLE moves the facade's initials follow the fields like any other view: ``HumanName("jose e maria santos").initials()`` is ``j. m. s.`` where 2.3.0 gave ``j. e. m. s.``. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) + - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) + + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)``. Widening the signature without forwarding is not enough and fails quietly: the token call supplies an empty ``name_part``, so the override returns nothing and ``initials()`` comes back empty instead of raising. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) **Additions** diff --git a/docs/usage.rst b/docs/usage.rst index c7c11c18..47f3d16e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -910,10 +910,14 @@ no longer knows ``de la`` are particles, so ``family_particles`` empties and ``family_base`` takes the whole field. A token the parse never saw carries no decision to honor, so a view -that is *handed* a vocabulary can fall back to it — -:meth:`~nameparser.ParsedName.capitalized` is the one that is, and it -falls back for one question only: whether a word is a conjunction or -an initial, which a word answers on its own. Whether a particle is +that is *handed* a vocabulary can fall back to it — of the parsed +name's own views, :meth:`~nameparser.ParsedName.capitalized` is the +one that is, and it falls back for one question only: whether a word +is a conjunction or an initial, which a word answers on its own. +(The v1 :class:`~nameparser.parser.HumanName` facade's ``initials()`` +is the other view that is handed one, and takes the same fallback for +spliced text; it is not a method of the parsed name and is not what +this section describes.) Whether a particle is acting as a particle is a fact about the whole part, and there is no reading on any word of a spliced field to derive it from, so a family set to ``de la`` stays lowercase where the same words parsed are diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 4c2a6b09..cfa202b0 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -574,6 +574,13 @@ def _process_initial(self, name_part: str, # alternative -- a string wrapper kept over a token core -- # would make such an override silently ineffective instead, # which hides the override rather than breaking it loudly. + # Such an override must ACCEPT `tokens` AND FORWARD it: + # super()._process_initial(name_part, firstname, tokens=tokens). + # Widening the signature alone is the silent failure the break + # exists to avoid -- `**kwargs`, or a `tokens=None` the super() + # call drops, leaves the override reading `name_part`, which on + # this path is "", so every group initials to "" and initials() + # returns "" without raising (measured 2026-09-13). # split() rather than split(" ") because split(" ") yields '' # between repeated spaces and `word[0]` below would raise # IndexError on it (#232). v1 stated the reason as `*_list` diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index c9316815..9d77a94a 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -7,7 +7,7 @@ from nameparser._config_shim import CONSTANTS, Constants from nameparser._facade import HumanName -from nameparser._types import UNCLASSIFIED_TAG, Role +from nameparser._types import UNCLASSIFIED_TAG, Role, Token _DATA_DIR = Path(__file__).parent / "data" @@ -761,6 +761,38 @@ def _process_initial(self, name_part: str, # type: ignore[override] hn.initials() +def test_an_override_that_forwards_tokens_keeps_working() -> None: + # The remedy for the break above, pinned because the OBVIOUS + # remedy is wrong in the quiet direction: widening the signature + # alone -- `**kwargs`, or a `tokens=None` the super() call drops -- + # leaves the override reading `name_part`, which the token path + # passes as "", so every group initials to "" and initials() + # returns "" without raising. Accepting AND FORWARDING is what + # restores the answer. decisions.md#R3's 2026-09-13 entry and the + # 2.4.0 release note both state the two-part remedy; this pins + # both halves so neither can be written as one again. + class Forwards(HumanName): + def _process_initial(self, name_part: str, + firstname: bool = False, + tokens: tuple[Token, ...] | None = None) -> str: + return super()._process_initial(name_part, firstname, + tokens=tokens) + + class WidensOnly(HumanName): + # The recorded negative control. `**kwargs` is the spelling a + # reader reaches for first, and mypy rejects it here -- worth + # noting, since a typed caller is warned and an untyped one is + # not, which is who this control is written for. + def _process_initial(self, name_part: str, # type: ignore[override] + firstname: bool = False, + **kwargs: object) -> str: + return super()._process_initial(name_part, firstname) + + assert HumanName("John Quincy Smith").initials() == "J. Q. S." + assert Forwards("John Quincy Smith").initials() == "J. Q. S." + assert WidensOnly("John Quincy Smith").initials() == "" + + def test_initials_of_a_spliced_field_ask_the_vocabulary() -> None: # A field assigned after the parse is raw text: ParsedName.replace() # stamps UNCLASSIFIED_TAG on it, which says the words were read by diff --git a/tools/differential/README.md b/tools/differential/README.md index 4d03dde1..fc76cbed 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -252,7 +252,7 @@ the default-order entry it was measured on, fatal on a contract name and printed under `MOVED SHAPE (radar)` on a radar one. Measured 2026-09-03, before the rule was written, MOST of the contest rows in `_RECORDED_DIFFS['expected_since_1.4.0.toml']` sat on radar-tier names --- 21 of 31, and 32 of the 45 today, the fourteen #498 pinned on +-- 21 of 31, and 32 of the 45 there were on 2026-09-05, the fourteen #498 pinned on 2026-09-05 adding ten radar rows and four contract ones on top of the 2026-09-05 period-class demotion, which moved `'김민준 씨.'` from the contract CJK corpus to the tolerated one (measured 2026-09-05; From 0261146f8b740c0fe0ef4fe7a40e901a11a92af4 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 02:49:17 -0700 Subject: [PATCH 04/11] test(#528): pins the review found missing - initials_separator was unguarded on the live token path: the two existing "Van Berg" tests exercise _process_initial's direct-string branch only. Pin it on HumanName.initials()/initials_list() over "Ph. D., John" instead, and note on the two direct-call tests that they cover the string path alone. - Add "john e jones" and "jones, john e" to the one-case-fork test: they back the 1.4.0 ledger rule, and the comma form is the shape where the connective-run regex never reaches the trailing "e". - test_list_tokens_for_carries_the_list_view_s_own_elements asserted the string view against _list_for(member) and that every group is truthy -- both true by construction (_list_for IS that join; an empty groups list is vacuously "all true"). Assert the string view against its own per-case literal instead, and the broader sweep against a second call for determinism. - Rename test_initials_of_an_unpickled_name_ask_the_vocabulary_too: copy.copy/copy.deepcopy go through the same __getstate__/__setstate__ hooks as pickle, so a copied name takes the identical vocabulary fallback. Measured and pinned alongside the pickle case. - Fix two stale comments: the mixed-case-controls comment covered a one-case control ("maria y lopez") as if it were mixed-case, and "joined" is healed by _list_tokens_for now, not _list_for (which is only the string view built from that walk). - Correct the corpus count: 1173 non-empty names, out of 1174 distinct names in the glob (one is empty), not "1173 names". Co-Authored-By: Claude Fable 5.1 --- tests/test_capitalization.py | 2 +- tests/test_initials.py | 16 ++++++++++++ tests/v2/test_facade.py | 48 ++++++++++++++++++++++++++---------- tests/v2/test_render.py | 23 +++++++++++++++++ 4 files changed, 75 insertions(+), 14 deletions(-) diff --git a/tests/test_capitalization.py b/tests/test_capitalization.py index 4d8c49a4..3fc91b8e 100644 --- a/tests/test_capitalization.py +++ b/tests/test_capitalization.py @@ -413,7 +413,7 @@ def test_a_restored_pickle_keeps_v1_conjunction_repair(self) -> None: # smith' also diverge on a round trip now, for the same reason and # through the same fallback, whether the reader is capitalize() or # (since #528) initials() -- pinned at - # tests/v2/test_facade.py::test_initials_of_an_unpickled_name_ask_the_vocabulary_too. + # tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too. def test_a_pickle_round_trip_loses_the_e_f_reading(self) -> None: direct = HumanName('juan e-f smith') direct.capitalize(force=True) diff --git a/tests/test_initials.py b/tests/test_initials.py index ab4095b6..8c4c041e 100644 --- a/tests/test_initials.py +++ b/tests/test_initials.py @@ -141,6 +141,10 @@ def test_initials_separator_custom_value(self) -> None: # Non-empty custom separator exercising _process_initial on a multi-word # token. "Van Berg" is a single name part whose two words produce two initials # joined by initials_separator. + # This calls _process_initial directly with a bare string (tokens=None), + # which is the v1 string path -- HumanName.initials() itself no longer + # takes it (#528). See test_initials_separator_is_honored_on_the_live_ + # token_path in tests/v2/test_render.py for the live token path's pin. hn = HumanName("", initials_separator="-", initials_delimiter=".") result = hn._process_initial("Van Berg", firstname=True) self.assertEqual(result, "V-B") @@ -196,6 +200,8 @@ def test_constructor_multiple(self) -> None: def test_initials_separator_kwarg_multiword_part(self) -> None: # Regression: initials_separator kwarg must flow into _process_initial # for multi-word name parts, not just into the initials() join calls. + # Direct string call (tokens=None), the v1 path -- see the comment on + # test_initials_separator_custom_value above. hn = HumanName("", initials_separator="") result = hn._process_initial("Van Berg", firstname=True) self.assertEqual(result, "VB") @@ -279,3 +285,13 @@ def test_initials_follow_the_one_case_fork_on_both_letters(self) -> None: self.m(hn.initials(), "j. e. s.", hn) hn = HumanName("maria y lopez") self.m(hn.initials(), "m. l.", hn) + # These two back the 1.4.0 ledger rule and its _CROSS_RULE_WINNERS / + # _RECORDED_DIFFS rows (tools/differential/expected_since_1.4.0.toml). + # "jones, john e" is the comma form, where 'e' ends the string -- + # the connective-run regex that reads a trailing letter's neighbors + # never reaches it, so it is a distinct shape from the space-written + # "john e jones" even though both give the same answer. + hn = HumanName("john e jones") + self.m(hn.initials(), "j. e. j.", hn) + hn = HumanName("jones, john e") + self.m(hn.initials(), "j. e. j.", hn) diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 9d77a94a..5d72a63d 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -1,4 +1,5 @@ """The 2.0 HumanName facade (mechanisms.md#FACADE-CONTRACT).""" +import copy import pickle import warnings from pathlib import Path @@ -178,9 +179,9 @@ def test_suffix_list_heals_joined_continuations() -> None: # v1 fix_phd def test_the_joined_tag_never_reaches_a_title( # #429 regression guard ) -> None: - """The "joined" tag is role-BLIND and _list_for heals it for every - role, so a tag placed for the suffix view is read by the title view - too. + """The "joined" tag is role-BLIND and _list_tokens_for heals it for + every role (_list_for is only the string view built from that walk), + so a tag placed for the suffix view is read by the title view too. So the pass that writes it (post_rules' R1 entry pass, #436) walks the SUFFIX tokens alone and reads every other role as transparent @@ -643,7 +644,8 @@ def test_list_tokens_for_carries_the_list_view_s_own_elements() -> None: # # Verified at review (2026-09-13): 0 mismatches between # _list_tokens_for's join and _list_for's own output over the - # full corpus, 1173 names x 7 members = 8211 pairs. That sweep + # full corpus, 1173 non-empty names (the glob holds 1174 distinct + # names, one of them empty) x 7 members = 8211 pairs. That sweep # can't fail by construction -- _list_for IS DEFINED as that join # -- so it does not stand as a test by itself. What a per-token # walk COULD get wrong is the element BOUNDARIES, so this pins the @@ -655,19 +657,24 @@ def test_list_tokens_for_carries_the_list_view_s_own_elements() -> None: folded_c.middle_name_as_last = True spliced = HumanName("john smith") spliced.middle = "e f" - cases: list[tuple[HumanName, str, list[list[str]]]] = [ + cases: list[tuple[HumanName, str, list[list[str]], list[str]]] = [ (HumanName("Hassan, Mohamad Ahmad Ali", constants=folded_c), "last", - [["Ahmad"], ["Ali"], ["Hassan"]]), # folded middle, first + [["Ahmad"], ["Ali"], ["Hassan"]], # folded middle, first + ["Ahmad", "Ali", "Hassan"]), (HumanName("Ph. D., John"), "last", - [["Ph.", "D."]]), # "joined" continuation - (spliced, "middle", [["e"], ["f"]]), # spliced field + [["Ph.", "D."]], # "joined" continuation + ["Ph. D."]), + (spliced, "middle", [["e"], ["f"]], # spliced field + ["e", "f"]), ] - for n, member, expected_shape in cases: + for n, member, expected_shape, expected_str in cases: groups = n._list_tokens_for(member) assert [[t.text for t in g] for g in groups] == expected_shape, \ (n.original, member) - assert [" ".join(t.text for t in g) for g in groups] \ - == n._list_for(member), (n.original, member) + # Asserted against its own literal, not against _list_for(member) -- + # _list_for IS DEFINED as this same join, so comparing the two + # can't fail by construction (measured; see the comment above). + assert n._list_for(member) == expected_str, (n.original, member) for name in ("Ph. D., John", "Dr. Juan Q. Xavier de la Vega III", "der, y van", "JUAN GARCIA Y LOPEZ", "Doe, John A."): @@ -675,7 +682,13 @@ def test_list_tokens_for_carries_the_list_view_s_own_elements() -> None: for member in ("title", "first", "middle", "last", "suffix", "nickname", "maiden"): groups = n._list_tokens_for(member) - assert all(g for g in groups), (name, member) + # `all(g for g in groups)` cannot fail by construction either -- + # _list_tokens_for never emits an empty group, vacuously true + # (including over the empty list every unused member here + # returns). What IS worth pinning is that the walk is + # deterministic: calling it again over the same parse gives + # the identical grouping, not a fresh (if equal-looking) one. + assert n._list_tokens_for(member) == groups, (name, member) def test_token_is_conjunction_reads_the_tag_then_the_vocabulary() -> None: @@ -831,7 +844,7 @@ def test_initials_freeze_the_connective_answer_at_parse_time() -> None: assert name.initials() == "j. y. g." -def test_initials_of_an_unpickled_name_ask_the_vocabulary_too() -> None: +def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> None: # __setstate__ is the second producer of UNCLASSIFIED_TAG tokens: a # v1 pickle carries the *_list STRINGS and no tags, so a restored # name is spliced text throughout and takes the fallback above. @@ -840,12 +853,21 @@ def test_initials_of_an_unpickled_name_ask_the_vocabulary_too() -> None: # capitalize() has done since the tag was introduced, for the same # reason and through the same helper. Pinned rather than left to # prose; decisions.md#R3 records it. + # + # copy.copy and copy.deepcopy go through the same __getstate__/ + # __setstate__ hooks as pickle (nameparser/_types.py's guarded pair), + # so a copied name takes the identical vocabulary fallback -- measured, + # not assumed. for name, live_initials, restored_initials, live_cap, restored_cap in ( ("JUAN Y GARCIA", "J. G.", "J. Y. G.", "Juan y Garcia", "Juan Y Garcia"), ("john e smith", "j. e. s.", "j. s.", "John E Smith", "John e Smith")): assert HumanName(name).initials() == live_initials + deep = copy.deepcopy(HumanName(name)) + assert deep.initials() == restored_initials + shallow = copy.copy(HumanName(name)) + assert shallow.initials() == restored_initials restored = pickle.loads(pickle.dumps(HumanName(name))) assert restored.initials() == restored_initials live = HumanName(name) diff --git a/tests/v2/test_render.py b/tests/v2/test_render.py index 3021af31..c83ddbd6 100644 --- a/tests/v2/test_render.py +++ b/tests/v2/test_render.py @@ -701,6 +701,12 @@ def test_facade_initials_follow_the_one_case_fork() -> None: # nothing moved on either surface assert HumanName("John E Smith").initials() == "J. E. S." assert HumanName("Juan Y. Garcia").initials() == "J. Y. G." + # 'maria y lopez' is a ONE-CASE control, not a mixed-case one: written + # wholly in lowercase, its 'y' is outside the marked set (rules.md#P3), + # so it stays the connective and drops on both surfaces -- unmoved, + # like the mixed-case names above, but for the vocabulary's reason + # rather than the writing's (tests/v2/test_ledger_guards.py's + # "one-case controls" wording, around line 1118). assert HumanName("maria y lopez").initials() == "m. l." # The one corpus name where the two views still differ, and it is # not this rule's: the facade merges 'Ph.' + 'D.' into one list @@ -708,3 +714,20 @@ def test_facade_initials_follow_the_one_case_fork() -> None: # (fix(initials-per-word) the Ph. D. merge, decisions.md#phd-merge) assert HumanName("Ph. D., John").initials() == "J. P D." assert parse("Ph. D., John").initials() == "J. P. D." + + +def test_initials_separator_is_honored_on_the_live_token_path() -> None: + # tests/test_initials.py's two "Van Berg" separator tests call + # _process_initial("Van Berg", firstname=True) directly -- v1's + # string path, with tokens=None -- which #528 kept working but no + # longer the path HumanName.initials() itself takes. This pins + # initials_separator on the live TOKEN path (tokens= passed by + # _initials_lists), so a regression that reads initials_separator + # only on the string branch would pass those two tests and fail + # here. Measured. + assert HumanName("Ph. D., John", initials_separator="-").initials() \ + == "J. P-D." + assert HumanName("Ph. D., John", initials_separator="").initials() \ + == "J. PD." + assert HumanName("Ph. D., John", initials_separator="") \ + .initials_list() == ["J", "PD"] From c4447660a06e48d8bb64331dc2ccb16f763ad9c1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 02:55:22 -0700 Subject: [PATCH 05/11] docs(review): the copy cost, the six names, and a restored dated entry - _facade.py/_render.py: two stale "_list_for's heal" / "text the parse never classified" comments corrected to name _list_tokens_for and the actual reader (a spliced field or a direct string call); the #232 doubled-space paragraph moved down into the tokens-is-None branch it now describes, and reworded to say so. - The accepted pickle cost is really a __getstate__/__setstate__ cost: copy.copy and copy.deepcopy take it too, measured. Widened the ACCEPTED COST paragraph in decisions.md#R3, the release_log.rst 2.4.0 Fix bullet, and the one _facade.py comment this PR added that named the pickle path alone (usage.rst and mechanisms.md's existing pickle prose predates this PR and is untouched). - decisions.md#R3's STATED PRIVATE-METHOD BREAK paragraph undersold its own scope: "the two initials values above" -> "the six names above", matching what the rest of the entry already says. - rules.md#R3/#R4: the particle question is not a fallback for the facade's initials view -- _is_particle is a live vocabulary lookup for every word, backed or spliced alike, and the facade never reads a particle tag. Reworded both paragraphs to say so, and disambiguated "in both views" (which now reads as {parsed, facade} next to the new facade paragraphs) to "in both of the parsed name's views" -- there is no HumanName.revise(). - decisions.md#P3: this PR had rewritten a dated entry that already landed in #527, instead of appending a new one. Restored the original "ACCEPTED, AND DEFERRED TO A FOLLOW-UP ISSUE" bullet verbatim and appended a new 2026-09-13 #528 paragraph carrying what the rewrite said, scoping "all along" to the window it actually covers (2.3.0 gave "John e Smith"). Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 7 ++++--- docs/design/rules.md | 34 ++++++++++++++++++---------------- docs/release_log.rst | 2 +- nameparser/_facade.py | 30 +++++++++++++++++------------- nameparser/_render.py | 3 ++- 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 34a79f29..57ea4ff5 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -327,7 +327,8 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf CASELESS LETTERS NEVER ENTER THE FORK. Arabic و has no case, so `token.upper() != token.lower()` is false and today's reading stands. A caseless-script input counts as "one case" under the helper, harmlessly, because the fork also requires a cased token. NO SWITCH. The subset is the knob: remove "e" to restore the joining reading, add "y" for a Dutch-style "every single letter is an initial". That is also this shape's answer to #516's switch question. ACCEPTED, AND RECORDED RATHER THAN FIXED: a locale pack cannot express "remove e". `Locale.lexicon` is unioned onto the base and never removes — its own field comment says "a pack never removes base vocabulary" — so an `nl` pack CAN add "y" while a `pt` caller has to write `Parser(lexicon=Lexicon.default().remove(conjunctions_ambiguous={"e"}))` by hand. The first (A) bullet added to #3-0-reevaluations on this date asks whether packs should be able to remove vocabulary at all. - ACCEPTED AND THEN CLOSED, SAME DAY (Derek, 2026-09-13): the facade and the core disagreed about the same letter for the length of one PR. `HumanName.initials()` did not follow the parse's tags — `_facade._process_initial` re-derived "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stayed "j. s." while `parse("john e smith").initials()` gave "j. e. s.", and `JUAN Y GARCIA` split the other way, facade "J. Y. G." against core "J. G.". `_facade.py` was untouched in the #383/#479 PR by decision, the split pinned by a contrastive test rather than left to prose, and #528 filed for it. #528 shipped the fix this paragraph predicted, with the R4 shape it predicted: the facade consults the PARSED token wherever the part maps to one and falls back to the vocabulary only where it does not. Both views give "j. e. s." and "J. G." now (measured 2026-09-13 after the fix); the contrastive test became the agreement test `tests/v2/test_render.py::test_facade_initials_follow_the_one_case_fork`. `decisions.md#R3`'s 2026-09-13 entry carries the fix, its accepted costs and the one corpus name where the two views still differ. `capitalize()` was never split: it followed the parse on both surfaces throughout, so `HumanName("john e smith").capitalize()` gave "John E Smith" all along. + ACCEPTED, AND DEFERRED TO A FOLLOW-UP ISSUE (Derek, 2026-09-13): the facade and the core now disagree about the same letter. `HumanName.initials()` does not follow the parse's tags — `_facade._process_initial` re-derives "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stays "j. s." while `parse("john e smith").initials()` gives "j. e. s.", and `JUAN Y GARCIA` splits the other way, facade "J. Y. G." against core "J. G.". `_facade.py` is untouched in this PR; the fix has R4's shape — consult the PARSED token wherever the part maps to one and fall back to the vocabulary only where it does not — and it is a follow-up issue, not yet filed, so no number is cited here. `capitalize()` is not split: it follows the parse on both surfaces, so `HumanName("john e smith").capitalize()` gives "John E Smith" too. + 2026-09-13 #528 — AMENDS the paragraph above: CLOSED THE SAME DAY. `_facade.py` was touched after all — #528 shipped the fix the paragraph predicted, with the R4 shape it predicted: the facade consults the PARSED token wherever the part maps to one and falls back to the vocabulary only where it does not. Both views give "j. e. s." and "J. G." now (measured 2026-09-13 after the fix); the contrastive test the paragraph above describes became the agreement test `tests/v2/test_render.py::test_facade_initials_follow_the_one_case_fork`. `decisions.md#R3`'s 2026-09-13 entry carries the fix, its accepted costs and the one corpus name where the two views still differ. `capitalize()` was never split: it followed the parse on both surfaces throughout the window the paragraph above describes, giving "John E Smith" for that whole window — not "all along", since 2.3.0, before #383/#479 landed, gave "John e Smith" for the same name. TWO OTHER CAUSES SHARE THE `_initials` FIELD IN THE LEDGERS AND NEITHER IS THIS PR'S, recorded because a reader meeting them under these names will reach for this entry. The Arabic `محمد و علي` diffs on `_initials` at 1.4.0 only (`م. و. ع.` → `م. ع.`), measured byte-identical either side of the fork: it is a pre-existing #269 consequence — a recognized non-Latin connective contributes no initial — surfaced by the row entering the contract corpus, and `expected_since_1.4.0.toml` ledgers it as `feat(#269)`. `JUAN Y GARCIA` diffs on `_initials` at 2.0.0 through 2.2.0 for a DIFFERENT reason than it does at 2.3.0: at those three baselines `fix(#462)` already admits the name (the facade moved `J. G.` → `J. Y. G.` there), and this PR's core move `J. Y. G.` → `J. G.` gets its own rule at 2.3.0 alone. Two causes, one field, documented in a dated paragraph on `fix(#462)` in those three ledgers rather than as a competing rule. Out of scope, each its own issue: #492 (whether a cased suffix token counts as case evidence — `is_one_case` is written so R5 and #492 can share it later, but render does not import it here); #478 (hyphenated connective repair — its spaced-form claim now depends on "y" staying OUT of the subset); #461 (R3's clause); #289 and #516 (the same case-class fact read at the suffix and post-comma slots); the render-side conjunction fallback for spliced raw text, which stays vocabulary-keyed as rules.md#R4 states. @@ -1182,9 +1183,9 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-01 #462 — DONE, the facade's twin: `HumanName.initials()` dropped a dotted or bare-capital `E`/`Y` from the middle and family groups because `_process_initial` tested conjunction membership alone, where 1.4.0's `is_conjunction` was "in the set AND NOT `is_an_initial`". `Scott E. Werner` gave `S. W.` from 2.0.0 through 2.2.0 and gives `S. E. W.` again; `John E Smith` (bare capital) and `Juan Y. Garcia` likewise; `parse().initials()` never had the bug, since `_classify` tags `E.` an initial and the render reads the tag (mechanisms.md#RENDER-HONORS-THE-PARSE). The issue said the bare form was "still correctly dropped", which is true of lowercase `e` and false of bare capital `E` — 7 of the 14 corpus names that move are bare capitals. Restored with `_render._INITIAL`, v1's own `initial` shape, because the facade may import `_render` and not `_pipeline`; scoped to `_process_initial`, its only caller, so a future reader of `_is_conjunction` does not inherit a decision made for initials. Found by #484's pseudo-field at the 1.4.0 baseline, where it had sat as 14 unreported diffs; the gate could not see it before because the seven fields never moved. - 2026-09-13 #528 — DONE: `HumanName.initials()` reads the parse's tags. The facade's initials view was the last derived view still re-deciding the CONNECTIVE question after the parse had answered it — scoped to that question deliberately, since the PARTICLE question is still re-decided from the lexicon in two places by decision, `_cap_word`'s particle conjunct (`decisions.md#R4`'s NOT DONE bullet) and `_is_particle` here, R4 drawing that boundary per question rather than per view. #462's fix above is why the connective half was still being re-decided: restoring v1's `is_conjunction` ("in the set AND NOT `is_an_initial`") restored a SHAPE test standing in for a tag, which agreed with the parse only while the parser agreed with the shape. #383/#479 ended that the same day it landed — rules.md#P3 now reads a one-case single-letter connective from vocabulary rather than from case, and no shape over the raw word can see it — so the two views of one parse disagreed: `HumanName("john e smith").initials()` gave "j. s." against the core's "j. e. s.", and `HumanName("JUAN Y GARCIA").initials()` gave "J. Y. G." against the core's "J. G.". Both give the core's answer now. This is mechanisms.md#RENDER-HONORS-THE-PARSE applied to the last view that had not taken it, and #458's principle reaching the facade layer. THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. - A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; nothing on the public v1 surface moves except the two initials values above. + A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; on the public v1 surface nothing moves but the initials of the six names above. ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. - ACCEPTED COST, THE SECOND ONE, and it is the pickle path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. A round-tripped `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the pickle is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_name_ask_the_vocabulary_too`. + ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-13). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. DIFFERENTIAL. The facade's `_initials` is compared at every baseline, 1.4.0 included, and the core's from 2.0.0; the pseudo-field enters a name's diff only where every role and every ambiguity kind agrees. So at 1.4.0 FOUR of the six take a new rule (`fix(#528) the facade's initials follow the parse's connective tags`) and two do not — `jose e maria santos` and `JUAN GARCIA Y LOPEZ` move roles against that baseline, and #383/#479's role rule explains them. At 2.0.0 through 2.2.0 nothing new: the e-names' `_ambiguities` diff keeps `_initials` out of their diff, and `JUAN Y GARCIA`'s single `_initials` row survives with the facade half of its two causes closed, which `fix(#462)`'s dated paragraph in `expected_since_2.0.0.toml` now says. At 2.3.0 the existing rule stands and both surfaces move together. All five gates re-run at 0 unexplained on 2026-09-13. diff --git a/docs/design/rules.md b/docs/design/rules.md index ed958540..2d880a5b 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1733,18 +1733,19 @@ R3. Rationale: initials abbreviate the person's name words; titles, name gives "j. v.". Case repair IS handed a vocabulary, so it falls back for the one question a word can answer on its own, and R4 says which. Revising the field through the parser classifies it - and matches the parse in both views. Stated without an example - line because every line here names an input string, and this - shape needs a field edited after the parse. + and matches the parse in both of the parsed name's views. Stated + without an example line because every line here names an input + string, and this shape needs a field edited after the parse. The v1 facade's HumanName.initials() is a second view of this question and IS handed a vocabulary: it reads the parse's reading wherever a word is backed by a parsed token, and falls back wherever a word is not — a field set as raw text, or a name - restored from a v1 pickle. It falls back on BOTH questions there, - not one: the connective question through the same helper case - repair uses, and the particle question through a live vocabulary - lookup, so a family spliced to "de la vega" initials "j. v." on - this view against "j. d. l. v." on the parsed name's own. So the + restored from a v1 pickle. The connective question falls back to + the same helper case repair uses; the particle question was never + asked of the parse in this view at all — `_is_particle` is a live + vocabulary lookup for every word, backed or spliced alike — so a + family spliced to "de la vega" initials "j. v." on this view + against "j. d. l. v." on the parsed name's own. So the two views agree on WHICH WORDS initial in a parsed name; what still differs there is GROUPING, the facade initialing a joined run as one element, which is where the name "Ph. D., John" gives @@ -1823,14 +1824,15 @@ R4. Rationale: case repair is a display concern, applied only on because the two answer oppositely. The parsed name's own view is handed no vocabulary at all, so it falls back on neither question and a spliced field's every word initials. The v1 facade's view IS - handed one and falls back on BOTH: the connective question through - the helper this rule uses, and the particle question through a - live vocabulary lookup, so a family spliced to "de la vega" - initials "j. v." there against the parsed view's "j. d. l. v.". - revise() classifies the - value and crosses both questions, in both views: a middle revised - to "e-f" repairs to "E-F" as the parsed name does, where splicing - the same text in gives "e-F". + handed one: the connective question falls back to the helper this + rule uses, while the particle question is never asked of the + parse in this view at all — a live vocabulary lookup for every + word, backed or spliced alike — so a family spliced to "de la + vega" initials "j. v." there against the parsed view's "j. d. l. + v.". revise() classifies the value and crosses both questions, in + both of the parsed name's views: a middle revised to "e-f" + repairs to "E-F" as the parsed name does, where splicing the same + text in gives "e-F". history: decisions.md#R4 · interacts: R2, R3, R5 · implemented: nameparser/_render.py R5. Rationale: mixed case is evidence that the writer cased the name diff --git a/docs/release_log.rst b/docs/release_log.rst index 87ca3cd7..d962e15c 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) - - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)``. Widening the signature without forwarding is not enough and fails quietly: the token call supplies an empty ``name_part``, so the override returns nothing and ``initials()`` comes back empty instead of raising. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)``. Widening the signature without forwarding is not enough and fails quietly: the token call supplies an empty ``name_part``, so the override returns nothing and ``initials()`` comes back empty instead of raising. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) **Additions** diff --git a/nameparser/_facade.py b/nameparser/_facade.py index cfa202b0..4ee4ae51 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -501,8 +501,9 @@ def _token_is_conjunction(self, tok: Token) -> bool: # # UNCLASSIFIED_TAG is the one case with no parse to honor: the # words were spliced into a field as raw text, by `hn.middle = - # ...` (ParsedName.replace) or by the v1 pickle load in - # __setstate__. They carry no reading, so the vocabulary + # ...` (ParsedName.replace) or restored via __setstate__ -- a + # pickle load, or a copy.copy/copy.deepcopy, which go through + # the same state hooks. They carry no reading, so the vocabulary # answers -- the same fallback _render._cap_word takes for the # same tokens and through the same helper, which is why the # helper is imported rather than the predicate rewritten @@ -510,8 +511,8 @@ def _token_is_conjunction(self, tok: Token) -> bool: # word is a connective is a fact the word can answer alone, # whether a particle is acting as one is a fact about the part). # - # _resolve() first, as _is_particle above does: an unpickled - # instance has no _lexicon until resolved. + # _resolve() first, as _is_particle above does: an unpickled or + # copied instance has no _lexicon until resolved. self._resolve() if UNCLASSIFIED_TAG in tok.tags: return _render._reads_as_conjunction(tok.text, self._lexicon) @@ -581,19 +582,22 @@ def _process_initial(self, name_part: str, # call drops, leaves the override reading `name_part`, which on # this path is "", so every group initials to "" and initials() # returns "" without raising (measured 2026-09-13). - # split() rather than split(" ") because split(" ") yields '' - # between repeated spaces and `word[0]` below would raise - # IndexError on it (#232). v1 stated the reason as `*_list` - # attributes bypassing whitespace normalization, which no - # longer holds -- the `*_list` properties are read-only in 2.x, - # and assignment through `hn.middle = ...` normalizes -- but a - # doubled space anywhere in a part still reaches here. # # Particles are NOT decided per token: _is_particle stays a # live vocabulary lookup, as _render._cap_word keeps it -- # rules.md#R4 draws this boundary per question, not per field. self._resolve() if tokens is None: + # STRING PATH. split() rather than split(" ") because + # split(" ") yields '' between repeated spaces and + # `word[0]` below would raise IndexError on it (#232). v1 + # stated the reason as `*_list` attributes bypassing + # whitespace normalization, which no longer holds -- the + # `*_list` properties are read-only in 2.x, and assignment + # through `hn.middle = ...` normalizes -- but a doubled + # space in a bare string handed directly to this method + # (`_process_initial(name_part, ...)` with no `tokens=`) + # still reaches here. words: tuple[str, ...] = tuple(name_part.split()) # No parse read this text, so every word takes the same # fallback _token_is_conjunction takes for a spliced one. @@ -815,8 +819,8 @@ def __setstate__(self, state: dict[str, Any]) -> None: # ("Ph. D.", "Q.C. M.P."), and re-splitting the joined string on # whitespace would promote each word to its own entry, which the # suffix view then renders comma-separated ("Ph., D."). Marking - # continuation words "joined" is the inverse of _list_for's heal, - # so list -> pickle -> list is the identity v1 gave us. + # continuation words "joined" is the inverse of _list_tokens_for's + # heal, so list -> pickle -> list is the identity v1 gave us. tokens: list[Token] = [] for member in _MEMBERS: role = Role(_V2_FIELD.get(member, member)) diff --git a/nameparser/_render.py b/nameparser/_render.py index c418c65e..2f816457 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -46,7 +46,8 @@ # the empty alternative) -- layering forbids importing the pipeline here; # keep in sync with _pipeline/_vocab.py by hand. # Its one reader is _reads_as_conjunction below, and that reader only -# ever sees text the parse never classified: for anything the +# ever sees text handed over as a bare string -- a spliced field, or a +# direct string call -- never a classified token: for anything the # parser DID see, the tag is the answer and this pattern is not asked. # So the two copies no longer decide the same question about the same # token -- _vocab's says what the parse decided, this one says what it From 85741fe2cdc3b76cbb0d2b9c5cde74992b493b1e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 03:05:42 -0700 Subject: [PATCH 06/11] docs(design): the restored bullet keeps its measurement sentence The design-docs review of the previous commit found three prose slips. The restored 2026-09-13 P3 bullet had dropped the sentence its own amendment points back to; it is byte-exact against master again. The #528 entry's "six names above" pointed at the wrong names, so the six are now listed. R3's enumeration of where the facade's initials view falls back said pickle only, while the decision entry and the release note already said pickle or copy. Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 4 ++-- docs/design/rules.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 57ea4ff5..427fe756 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -327,7 +327,7 @@ The reconciled v1-style banks (`tests/test_*.py`) carried eight `@pytest.mark.xf CASELESS LETTERS NEVER ENTER THE FORK. Arabic و has no case, so `token.upper() != token.lower()` is false and today's reading stands. A caseless-script input counts as "one case" under the helper, harmlessly, because the fork also requires a cased token. NO SWITCH. The subset is the knob: remove "e" to restore the joining reading, add "y" for a Dutch-style "every single letter is an initial". That is also this shape's answer to #516's switch question. ACCEPTED, AND RECORDED RATHER THAN FIXED: a locale pack cannot express "remove e". `Locale.lexicon` is unioned onto the base and never removes — its own field comment says "a pack never removes base vocabulary" — so an `nl` pack CAN add "y" while a `pt` caller has to write `Parser(lexicon=Lexicon.default().remove(conjunctions_ambiguous={"e"}))` by hand. The first (A) bullet added to #3-0-reevaluations on this date asks whether packs should be able to remove vocabulary at all. - ACCEPTED, AND DEFERRED TO A FOLLOW-UP ISSUE (Derek, 2026-09-13): the facade and the core now disagree about the same letter. `HumanName.initials()` does not follow the parse's tags — `_facade._process_initial` re-derives "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stays "j. s." while `parse("john e smith").initials()` gives "j. e. s.", and `JUAN Y GARCIA` splits the other way, facade "J. Y. G." against core "J. G.". `_facade.py` is untouched in this PR; the fix has R4's shape — consult the PARSED token wherever the part maps to one and fall back to the vocabulary only where it does not — and it is a follow-up issue, not yet filed, so no number is cited here. `capitalize()` is not split: it follows the parse on both surfaces, so `HumanName("john e smith").capitalize()` gives "John E Smith" too. + ACCEPTED, AND DEFERRED TO A FOLLOW-UP ISSUE (Derek, 2026-09-13): the facade and the core now disagree about the same letter. `HumanName.initials()` does not follow the parse's tags — `_facade._process_initial` re-derives "is this the connective" from vocabulary plus initial shape on the part's raw text — so `HumanName("john e smith").initials()` stays "j. s." while `parse("john e smith").initials()` gives "j. e. s.", and `JUAN Y GARCIA` splits the other way, facade "J. Y. G." against core "J. G.". Both measured here 2026-09-13 and pinned by a contrastive test rather than left to prose. `_facade.py` is untouched in this PR; the fix has R4's shape — consult the PARSED token wherever the part maps to one and fall back to the vocabulary only where it does not — and it is a follow-up issue, not yet filed, so no number is cited here. `capitalize()` is not split: it follows the parse on both surfaces, so `HumanName("john e smith").capitalize()` gives "John E Smith" too. 2026-09-13 #528 — AMENDS the paragraph above: CLOSED THE SAME DAY. `_facade.py` was touched after all — #528 shipped the fix the paragraph predicted, with the R4 shape it predicted: the facade consults the PARSED token wherever the part maps to one and falls back to the vocabulary only where it does not. Both views give "j. e. s." and "J. G." now (measured 2026-09-13 after the fix); the contrastive test the paragraph above describes became the agreement test `tests/v2/test_render.py::test_facade_initials_follow_the_one_case_fork`. `decisions.md#R3`'s 2026-09-13 entry carries the fix, its accepted costs and the one corpus name where the two views still differ. `capitalize()` was never split: it followed the parse on both surfaces throughout the window the paragraph above describes, giving "John E Smith" for that whole window — not "all along", since 2.3.0, before #383/#479 landed, gave "John e Smith" for the same name. TWO OTHER CAUSES SHARE THE `_initials` FIELD IN THE LEDGERS AND NEITHER IS THIS PR'S, recorded because a reader meeting them under these names will reach for this entry. The Arabic `محمد و علي` diffs on `_initials` at 1.4.0 only (`م. و. ع.` → `م. ع.`), measured byte-identical either side of the fork: it is a pre-existing #269 consequence — a recognized non-Latin connective contributes no initial — surfaced by the row entering the contract corpus, and `expected_since_1.4.0.toml` ledgers it as `feat(#269)`. `JUAN Y GARCIA` diffs on `_initials` at 2.0.0 through 2.2.0 for a DIFFERENT reason than it does at 2.3.0: at those three baselines `fix(#462)` already admits the name (the facade moved `J. G.` → `J. Y. G.` there), and this PR's core move `J. Y. G.` → `J. G.` gets its own rule at 2.3.0 alone. Two causes, one field, documented in a dated paragraph on `fix(#462)` in those three ledgers rather than as a competing rule. Out of scope, each its own issue: #492 (whether a cased suffix token counts as case evidence — `is_one_case` is written so R5 and #492 can share it later, but render does not import it here); #478 (hyphenated connective repair — its spaced-form claim now depends on "y" staying OUT of the subset); #461 (R3's clause); #289 and #516 (the same case-class fact read at the suffix and post-comma slots); the render-side conjunction fallback for spliced raw text, which stays vocabulary-keyed as rules.md#R4 states. @@ -1183,7 +1183,7 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-01 #462 — DONE, the facade's twin: `HumanName.initials()` dropped a dotted or bare-capital `E`/`Y` from the middle and family groups because `_process_initial` tested conjunction membership alone, where 1.4.0's `is_conjunction` was "in the set AND NOT `is_an_initial`". `Scott E. Werner` gave `S. W.` from 2.0.0 through 2.2.0 and gives `S. E. W.` again; `John E Smith` (bare capital) and `Juan Y. Garcia` likewise; `parse().initials()` never had the bug, since `_classify` tags `E.` an initial and the render reads the tag (mechanisms.md#RENDER-HONORS-THE-PARSE). The issue said the bare form was "still correctly dropped", which is true of lowercase `e` and false of bare capital `E` — 7 of the 14 corpus names that move are bare capitals. Restored with `_render._INITIAL`, v1's own `initial` shape, because the facade may import `_render` and not `_pipeline`; scoped to `_process_initial`, its only caller, so a future reader of `_is_conjunction` does not inherit a decision made for initials. Found by #484's pseudo-field at the 1.4.0 baseline, where it had sat as 14 unreported diffs; the gate could not see it before because the seven fields never moved. - 2026-09-13 #528 — DONE: `HumanName.initials()` reads the parse's tags. The facade's initials view was the last derived view still re-deciding the CONNECTIVE question after the parse had answered it — scoped to that question deliberately, since the PARTICLE question is still re-decided from the lexicon in two places by decision, `_cap_word`'s particle conjunct (`decisions.md#R4`'s NOT DONE bullet) and `_is_particle` here, R4 drawing that boundary per question rather than per view. #462's fix above is why the connective half was still being re-decided: restoring v1's `is_conjunction` ("in the set AND NOT `is_an_initial`") restored a SHAPE test standing in for a tag, which agreed with the parse only while the parser agreed with the shape. #383/#479 ended that the same day it landed — rules.md#P3 now reads a one-case single-letter connective from vocabulary rather than from case, and no shape over the raw word can see it — so the two views of one parse disagreed: `HumanName("john e smith").initials()` gave "j. s." against the core's "j. e. s.", and `HumanName("JUAN Y GARCIA").initials()` gave "J. Y. G." against the core's "J. G.". Both give the core's answer now. This is mechanisms.md#RENDER-HONORS-THE-PARSE applied to the last view that had not taken it, and #458's principle reaching the facade layer. THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. - A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; on the public v1 surface nothing moves but the initials of the six names above. + A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-13). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. diff --git a/docs/design/rules.md b/docs/design/rules.md index 2d880a5b..60bdf378 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -1740,9 +1740,10 @@ R3. Rationale: initials abbreviate the person's name words; titles, question and IS handed a vocabulary: it reads the parse's reading wherever a word is backed by a parsed token, and falls back wherever a word is not — a field set as raw text, or a name - restored from a v1 pickle. The connective question falls back to - the same helper case repair uses; the particle question was never - asked of the parse in this view at all — `_is_particle` is a live + restored from a v1 pickle or copied through the same state hooks. + The connective question falls back to the same helper case repair + uses; the particle question was never asked of the parse in this + view at all — `_is_particle` is a live vocabulary lookup for every word, backed or spliced alike — so a family spliced to "de la vega" initials "j. v." on this view against "j. d. l. v." on the parsed name's own. So the From 84d900035144f7f367b7ebe67bf62f32da149c6b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 18:52:04 -0700 Subject: [PATCH 07/11] fix(#528): the token path hands the override real text _initials_lists called self._process_initial("", firstname=firstname, tokens=group) -- the empty placeholder never mattered on the token path, since _process_initial ignores name_part whenever tokens is given. But a v1-style subclass that widens _process_initial's signature to swallow `tokens` (`**kwargs`, or an unforwarded `tokens=None`) without forwarding it takes the STRING path in the super() call, reading that same "" for name_part -- so initials() came back "" for every group, silently, for such an override. Pass the group's real text instead, " ".join(tok.text for tok in group), which _list_tokens_for already builds the tuple form of (no extra parse-path walk: _initials_lists walks tokens directly and never calls the cheap _list_for string view). A correct override that forwards tokens is unaffected, since the token path still wins whenever tokens is passed. Before: WidensOnly("john e smith").initials() == "" After: WidensOnly("john e smith").initials() == "j. s." (the PRE-#528 answer -- the STRING path's vocabulary fallback reads "e" as the connective, where the token path's parse-backed reading, restored by #528, does not) Forwarding still receives the fix either way: Forwards("john e smith").initials() == "j. e. s." HumanName("john e smith").initials() == "j. e. s." Updates the STATED BREAK comment in _process_initial, the R3 entry in docs/design/decisions.md (written in this PR, not yet on master, amended in place), the 2.4.0 Fix bullet in docs/release_log.rst, and tests/v2/test_facade.py's WidensOnly negative control to assert the new pre-#528 answer instead of "". Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 2 +- docs/release_log.rst | 2 +- nameparser/_facade.py | 22 ++++++++++++++++------ tests/v2/test_facade.py | 24 ++++++++++++++++-------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 427fe756..69208b4a 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1183,7 +1183,7 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-01 #462 — DONE, the facade's twin: `HumanName.initials()` dropped a dotted or bare-capital `E`/`Y` from the middle and family groups because `_process_initial` tested conjunction membership alone, where 1.4.0's `is_conjunction` was "in the set AND NOT `is_an_initial`". `Scott E. Werner` gave `S. W.` from 2.0.0 through 2.2.0 and gives `S. E. W.` again; `John E Smith` (bare capital) and `Juan Y. Garcia` likewise; `parse().initials()` never had the bug, since `_classify` tags `E.` an initial and the render reads the tag (mechanisms.md#RENDER-HONORS-THE-PARSE). The issue said the bare form was "still correctly dropped", which is true of lowercase `e` and false of bare capital `E` — 7 of the 14 corpus names that move are bare capitals. Restored with `_render._INITIAL`, v1's own `initial` shape, because the facade may import `_render` and not `_pipeline`; scoped to `_process_initial`, its only caller, so a future reader of `_is_conjunction` does not inherit a decision made for initials. Found by #484's pseudo-field at the 1.4.0 baseline, where it had sat as 14 unreported diffs; the gate could not see it before because the seven fields never moved. - 2026-09-13 #528 — DONE: `HumanName.initials()` reads the parse's tags. The facade's initials view was the last derived view still re-deciding the CONNECTIVE question after the parse had answered it — scoped to that question deliberately, since the PARTICLE question is still re-decided from the lexicon in two places by decision, `_cap_word`'s particle conjunct (`decisions.md#R4`'s NOT DONE bullet) and `_is_particle` here, R4 drawing that boundary per question rather than per view. #462's fix above is why the connective half was still being re-decided: restoring v1's `is_conjunction` ("in the set AND NOT `is_an_initial`") restored a SHAPE test standing in for a tag, which agreed with the parse only while the parser agreed with the shape. #383/#479 ended that the same day it landed — rules.md#P3 now reads a one-case single-letter connective from vocabulary rather than from case, and no shape over the raw word can see it — so the two views of one parse disagreed: `HumanName("john e smith").initials()` gave "j. s." against the core's "j. e. s.", and `HumanName("JUAN Y GARCIA").initials()` gave "J. Y. G." against the core's "J. G.". Both give the core's answer now. This is mechanisms.md#RENDER-HONORS-THE-PARSE applied to the last view that had not taken it, and #458's principle reaching the facade layer. THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. - A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. THE REMEDY IS TWO EDITS AND NOT ONE, and the one-edit version reproduces exactly the silent failure this paragraph rejects, measured 2026-09-13: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not — `**kwargs`, or an explicit `tokens=None` that the super() call drops, both send the token path's `name_part`, which is the empty string, and the subclass returns "" for every group: initials of "John Quincy Smith" come back as "" rather than raising. So "accept a `tokens` keyword" is the wrong instruction and was written here first; the forwarding is the whole of it. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. + A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. FORWARDING IS STILL THE ONLY WAY TO RECEIVE THE FIX: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not receive the fix either — `**kwargs`, or an explicit `tokens=None` that the super() call drops. Revised 2026-09-14, before this paragraph ever reached master: the token path now passes the group's own text as `name_part` (`" ".join(tok.text for tok in group)`) rather than an empty placeholder, precisely so an override that ignores the `tokens` keyword falls onto the STRING path and behaves as it did before this upgrade, rather than going quietly blank. Measured 2026-09-14: `WidensOnly("john e smith").initials()` is "j. s.", the pre-#528 answer, where a forwarding override and the library itself both give "j. e. s." — the STRING path's vocabulary fallback reads the middle "e" as the connective and drops it, where the token path's parse-backed reading (restored by #528) does not. `John Quincy Smith` does not witness the difference (no word in it is contested), which is why the divergence has to be measured on a name the fix actually moves. So "accept a `tokens` keyword" is still the wrong instruction on its own; the forwarding is what receives the fix, not merely what avoids a `TypeError`. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-13). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. diff --git a/docs/release_log.rst b/docs/release_log.rst index d962e15c..edd6fac5 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) - - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)``. Widening the signature without forwarding is not enough and fails quietly: the token call supplies an empty ``name_part``, so the override returns nothing and ``initials()`` comes back empty instead of raising. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) **Additions** diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 4ee4ae51..5c601938 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -577,11 +577,19 @@ def _process_initial(self, name_part: str, # which hides the override rather than breaking it loudly. # Such an override must ACCEPT `tokens` AND FORWARD it: # super()._process_initial(name_part, firstname, tokens=tokens). - # Widening the signature alone is the silent failure the break - # exists to avoid -- `**kwargs`, or a `tokens=None` the super() - # call drops, leaves the override reading `name_part`, which on - # this path is "", so every group initials to "" and initials() - # returns "" without raising (measured 2026-09-13). + # That is still the only way to RECEIVE #528's fix. Widening + # the signature alone -- `**kwargs`, or a `tokens=None` the + # super() call drops -- does not raise and does not go silent + # either: `name_part` on this path is the group's own text + # (Derek, 2026-09-14), so the override takes the STRING path + # below and keeps working, just without the fix -- the + # PRE-#528 answer, computed from the vocabulary fallback + # instead of the parse (measured 2026-09-14: + # `WidensOnly("john e smith").initials()` is "j. s.", where a + # forwarding override and the library itself give "j. e. s."). + # Real text was chosen over the empty placeholder precisely so + # an override that ignores the keyword behaves as it did + # before the upgrade, rather than going quietly blank. # # Particles are NOT decided per token: _is_particle stays a # live vocabulary lookup, as _render._cap_word keeps it -- @@ -638,7 +646,9 @@ def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: def group_initials(groups: list[tuple[Token, ...]], firstname: bool = False) -> list[str]: got = [i for i in ( - self._process_initial("", firstname=firstname, tokens=group) + self._process_initial( + " ".join(tok.text for tok in group), + firstname=firstname, tokens=group) for group in groups) if i] words = [tok.text for group in groups for tok in group] if got or not words or not all(self._is_particle(w) diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 5d72a63d..c0e43ab5 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -779,11 +779,19 @@ def test_an_override_that_forwards_tokens_keeps_working() -> None: # remedy is wrong in the quiet direction: widening the signature # alone -- `**kwargs`, or a `tokens=None` the super() call drops -- # leaves the override reading `name_part`, which the token path - # passes as "", so every group initials to "" and initials() - # returns "" without raising. Accepting AND FORWARDING is what - # restores the answer. decisions.md#R3's 2026-09-13 entry and the - # 2.4.0 release note both state the two-part remedy; this pins - # both halves so neither can be written as one again. + # now passes as the group's own text (Derek, 2026-09-14 -- #528 + # passed "" here originally), so a widen-only override takes the + # STRING path instead of raising or going silent: it gets the + # PRE-#528 answer, computed from the vocabulary fallback rather + # than the parse. "john e smith" is where the two views disagree + # -- the parse reads the middle "e" as initial-shaped + # (test_process_initial_with_tokens_reads_the_parse), but the + # bare-word vocabulary fallback reads lowercase "e" as the + # connective and drops it -- so WidensOnly keeps working but + # without #528's fix. Accepting AND FORWARDING `tokens` is still + # the only way to receive the fix. decisions.md#R3's 2026-09-13 + # entry (amended 2026-09-14) and the 2.4.0 release note both + # state this. class Forwards(HumanName): def _process_initial(self, name_part: str, firstname: bool = False, @@ -801,9 +809,9 @@ def _process_initial(self, name_part: str, # type: ignore[override] **kwargs: object) -> str: return super()._process_initial(name_part, firstname) - assert HumanName("John Quincy Smith").initials() == "J. Q. S." - assert Forwards("John Quincy Smith").initials() == "J. Q. S." - assert WidensOnly("John Quincy Smith").initials() == "" + assert HumanName("john e smith").initials() == "j. e. s." + assert Forwards("john e smith").initials() == "j. e. s." + assert WidensOnly("john e smith").initials() == "j. s." def test_initials_of_a_spliced_field_ask_the_vocabulary() -> None: From 8ed4affa6eca64b8498c45eafc588d9a2a2602f9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 19:33:25 -0700 Subject: [PATCH 08/11] fix(#528): an overridden *_list property still feeds initials() Second review round on #528 found that _initials_lists moved from reading self.first_list/middle_list/last_list -- public properties a v1 subclass may override -- to calling self._list_tokens_for(member) directly. A subclass overriding one of those properties is now silently ignored by initials(), while last_base/surnames/given_names (via _split_last, which still reads self.last_list) keep honoring it. Measured before this fix (three-property Sub, "john Xavier smith"): pre-#528 (338daf7): "J. Z." 84d9000 (the bug): "j. X. s." this commit: "J. Z." And a one-property override (SubLastOnly.last_list -> ["Zorro"], on "john e smith"), pinning that an un-overridden member keeps the (post-#528) token path: this commit: "j. e. Z." Fix: _initials_lists now checks, per member, whether the class overrides that property (getattr(type(self), f"{member}_list") is not getattr(HumanName, f"{member}_list") -- three cheap identity checks, nothing on the parse path). An overridden member takes the pre-#528 STRING path over the override's own strings (reproduced from `git show 338daf7:nameparser/_facade.py`, all-particle readmission and the `if n` filter included); an un-overridden member is unchanged. Also: zip(words, conjunctions, strict=True) in _process_initial, so a future filter on one branch raises instead of silently truncating. Verified 0 differences in HumanName(n).initials() across every name in tools/differential/corpus*.jsonl (527 distinct) between 84d9000 and this tree -- the plain, un-overridden facade path is unchanged. tests/v2/test_benchmark.py stays 17 passed. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Sonnet 5 --- nameparser/_facade.py | 77 ++++++++++++++++++++++++++++++++++------- tests/v2/test_facade.py | 75 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 5c601938..510d426e 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -512,7 +512,12 @@ def _token_is_conjunction(self, tok: Token) -> bool: # whether a particle is acting as one is a fact about the part). # # _resolve() first, as _is_particle above does: an unpickled or - # copied instance has no _lexicon until resolved. + # copied instance has no _lexicon until resolved -- and neither + # does a keyword-constructed one (`HumanName(first=..., middle= + # "y", last=...)`), which never runs the full-string parse path + # other callers rely on to have called _resolve() already. + # LOAD-BEARING for exactly those caller shapes: do not delete + # this call as dead just because most callers arrive resolved. self._resolve() if UNCLASSIFIED_TAG in tok.tags: return _render._reads_as_conjunction(tok.text, self._lexicon) @@ -591,6 +596,16 @@ def _process_initial(self, name_part: str, # an override that ignores the keyword behaves as it did # before the upgrade, rather than going quietly blank. # + # An overridden PUBLIC `first_list`/`middle_list`/`last_list` + # property lands here the same way: _initials_lists (above) + # detects the override and calls this method for that member + # with no `tokens=` at all, so it takes the STRING path below + # regardless of what `tokens=` a WidensOnly-style override + # might otherwise forward -- the override supplies strings, + # not tokens, so there is nothing to forward. Same degradation, + # same pre-#528 vocabulary answer, for the same reason: no + # tokens exist to read a parse's tag from. + # # Particles are NOT decided per token: _is_particle stays a # live vocabulary lookup, as _render._cap_word keeps it -- # rules.md#R4 draws this boundary per question, not per field. @@ -617,7 +632,7 @@ def _process_initial(self, name_part: str, conjunctions = tuple(self._token_is_conjunction(tok) for tok in tokens) initials = [] - for word, conjunction in zip(words, conjunctions): + for word, conjunction in zip(words, conjunctions, strict=True): if not (self._is_particle(word) or conjunction) or firstname: initials.append(word[0]) if len(initials) > 0: @@ -642,15 +657,17 @@ def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: the `*_list` view (#528), so every word carries the reading the parse gave it; the elements are the list view's own, folded first and continuations merged, because one walk builds both. + A member whose PUBLIC `first_list`/`middle_list`/`last_list` + property is overridden is the one exception: `first`/`middle`/ + `last` (the private token walk this method otherwise uses) do + not consult that override, so honoring it means reading the + override's own strings instead -- the pre-#528 walk, degraded + to the vocabulary fallback exactly as a widen-only + `_process_initial` override is (STATED BREAK, below): the + override supplies strings, not tokens, so there is nothing to + forward even if it accepted `tokens=`. """ - def group_initials(groups: list[tuple[Token, ...]], - firstname: bool = False) -> list[str]: - got = [i for i in ( - self._process_initial( - " ".join(tok.text for tok in group), - firstname=firstname, tokens=group) - for group in groups) if i] - words = [tok.text for group in groups for tok in group] + def all_particle_guard(got: list[str], words: list[str]) -> list[str]: if got or not words or not all(self._is_particle(w) for w in words): return got @@ -668,9 +685,43 @@ def group_initials(groups: list[tuple[Token, ...]], # already applies the same guard to the base, which is why # last_base was never empty here. return [w[0] for w in words] - return (group_initials(self._list_tokens_for("first"), True), - group_initials(self._list_tokens_for("middle")), - group_initials(self._list_tokens_for("last"))) + + def group_initials(groups: list[tuple[Token, ...]], + firstname: bool = False) -> list[str]: + got = [i for i in ( + self._process_initial( + " ".join(tok.text for tok in group), + firstname=firstname, tokens=group) + for group in groups) if i] + words = [tok.text for group in groups for tok in group] + return all_particle_guard(got, words) + + def group_initials_from_list(names: list[str], + firstname: bool = False) -> list[str]: + # PRE-#528 walk (`git show 338daf7:nameparser/_facade.py`), + # reproduced exactly: no tokens exist to walk here, only + # the override's own strings, so each element goes through + # `_process_initial` with no `tokens=` -- the STRING path, + # answered from the vocabulary rather than the parse. + got = [i for i in (self._process_initial(n, firstname=firstname) + for n in names if n) if i] + words = [w for n in names if n for w in n.split()] + return all_particle_guard(got, words) + + def initials_for(member: str, firstname: bool) -> list[str]: + # One class-attribute identity check per member (cheap: no + # parse involved) decides which walk honors that member's + # public property. + overridden = (getattr(type(self), f"{member}_list") + is not getattr(HumanName, f"{member}_list")) + if overridden: + return group_initials_from_list( + getattr(self, f"{member}_list"), firstname) + return group_initials(self._list_tokens_for(member), firstname) + + return (initials_for("first", True), + initials_for("middle", False), + initials_for("last", False)) def initials_list(self) -> list[str]: first, middle, last = self._initials_lists() diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index c0e43ab5..048330db 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -720,6 +720,19 @@ def test_token_is_conjunction_reads_the_tag_then_the_vocabulary() -> None: assert spliced._token_is_conjunction(upper) is False +def test_token_is_conjunction_resolves_an_unresolved_instance() -> None: + # _token_is_conjunction calls self._resolve() before reading + # self._lexicon (see the comment on that call). Pinning that it is + # load-bearing, not defensive dead code: a keyword-constructed + # HumanName never runs the full-string parse path other callers + # rely on to have resolved already, so _lexicon is absent here + # until this method's own _resolve() call builds it. + hn = HumanName(first="John", middle="y", last="Smith") + assert not hasattr(hn, "_lexicon") + tok = hn._list_tokens_for("middle")[0][0] + assert hn._token_is_conjunction(tok) is True + + def test_process_initial_direct_call_keeps_the_v1_string_path() -> None: # tests/test_initials.py calls this with a bare string and no # tokens, which is v1's shape and stays supported: with no tokens @@ -813,6 +826,68 @@ def _process_initial(self, name_part: str, # type: ignore[override] assert Forwards("john e smith").initials() == "j. e. s." assert WidensOnly("john e smith").initials() == "j. s." + # The multi-token-group pin: "Ph." + "D." is a "joined" continuation + # (_list_tokens_for), the only producer of a group with more than one + # token, so it is the one place the join's SHAPE -- space-separated, + # both words -- is observable at all. A join without the separator, + # or with only one token, gives "J. P." / "J. D." and passes every + # single-token name; this is what actually exercises `zip(words, + # conjunctions, strict=True)` over more than one element per group. + # Measured 2026-09-14. + assert HumanName("Ph. D., John").initials() == "J. P D." + assert Forwards("Ph. D., John").initials() == "J. P D." + assert WidensOnly("Ph. D., John").initials() == "J. P D." + assert (WidensOnly("Ph. D., John", initials_separator="-").initials() + == "J. P-D.") + + +def test_initials_honor_an_overridden_list_property() -> None: + # F1, second review round: before #528, _initials_lists read + # self.first_list/middle_list/last_list -- public properties a v1 + # subclass may override -- and #528 switched it to the private + # token walk (_list_tokens_for) directly, which does not consult + # such an override. last_base/surnames/given_names still honor it + # (they route through _split_last, which reads self.last_list), so + # initials() alone went silently stale. The fix detects an + # overridden property per member (a cheap class-attribute identity + # check) and, for that member only, takes the pre-#528 STRING path + # over the override's own strings -- same degradation a + # WidensOnly-style _process_initial override gets, and the same + # pre-#528 answer. Measured 2026-09-14 against `git show + # 338daf7:nameparser/_facade.py` (before #528): both values below + # are that package's answer for the same construction. + class Sub(HumanName): + @property + def first_list(self) -> list[str]: + return [p.upper() for p in super().first_list] + + @property + def middle_list(self) -> list[str]: + return [p for p in super().middle_list if not p.startswith("X")] + + @property + def last_list(self) -> list[str]: + return ["Zorro"] + + sub = Sub("john Xavier smith") + assert sub.initials() == "J. Z." + # last_base/surnames already honored the override before this fix; + # pinned here so a future change can't silently regress it back + # into agreement with initials() for the wrong reason. + assert sub.last_base == "Zorro" + assert sub.surnames == "Zorro" + + class SubLastOnly(HumanName): + @property + def last_list(self) -> list[str]: + return ["Zorro"] + + # Only ONE property overridden: first/middle are un-overridden and + # must keep the (post-#528) TOKEN path -- the middle "e" is "e." + # because the parse tagged it an initial, not the connective -- and + # only last takes the string path and the override's "Zorro". + assert SubLastOnly("john e smith").initials() == "j. e. Z." + def test_initials_of_a_spliced_field_ask_the_vocabulary() -> None: # A field assigned after the parse is raw text: ParsedName.replace() From 53c1cb32690536a6bc75c4f688f86075bbfd0f66 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 19:40:33 -0700 Subject: [PATCH 09/11] docs(review): the comments say what the second round measured Second review round on #528/the *_list-override fix found several comments that had drifted from what the code and tests actually do: - tests/v2/test_facade.py: the "deterministic" claim on _list_tokens_for's repeat-call assert was false (the method rebuilds fresh lists every call); reworded to what the assert actually pins -- an equal grouping, which a one-shot walk behind tokens_for() would not give a second time. - tests/v2/test_facade.py: fixed the __getstate__/__setstate__ attribution -- HumanName defines its own pair in _facade.py, not nameparser/_types.py's guarded pair (a different set of classes). - tests/v2/test_facade.py: "initial-shaped" misused the repo's term (matches _render._INITIAL's shape, which "e" does not); reworded to say the parse TAGS "e" an initial via rules.md#P3's one-case fork, which is exactly why the bare-word vocabulary fallback disagrees. - tests/v2/test_facade.py: added the multi-token-group pin the reviewer measured on "Ph. D., John" (HumanName/Forwards/WidensOnly all "J. P D.", WidensOnly with initials_separator="-" giving "J. P-D."), the one shape that exercises the join's actual form. - tests/v2/test_facade.py + nameparser/_facade.py: added tests and a comment pinning that _token_is_conjunction's _resolve() call is load-bearing for a keyword-constructed HumanName, not just for unpickled/copied instances. - tests/test_initials.py: corrected which of #528's four movers has rows in _CROSS_RULE_WINNERS/_RECORDED_DIFFS ("john e jones", not "jones, john e") and why -- the connective-run ledger rule's operative half is a `(?=\s)` lookahead that a string-final letter can't satisfy -- and named the rule instead of describing it loosely. - nameparser/_render.py: _reads_as_conjunction's docstring and its _INITIAL comment claimed it is asked "only of text the parse never saw", which 84d9000 falsified -- a widen-only _process_initial override now sends parse-classified text down this path too. Reworded around the caller's contract (no token passed, whether or not the parse read the text) and added __setstate__ (pickle, copy.copy, copy.deepcopy) as a producer alongside replace(). - docs/design/decisions.md, the unlanded 2026-09-13 #528 R3 entry: recast the remedy illustration onto "john e smith" (Forwards gives "j. e. s.", WidensOnly "j. s.") since every widened override gives "John Quincy Smith" its "J. Q. S." back regardless, making that example vestigial; added a paragraph recording this PR's F1 fix (an overridden public *_list property is honored, taking the pre-#528 string path for that member -- measured "J. Z." and "j. e. Z." on the two Sub subclasses); corrected the copy.copy/copy.deepcopy measurement date (2026-09-14, not 09-13 -- the pickle half stays 09-13); and added the population figure: over the deduped tools/differential/corpus*.jsonl glob (1174 distinct names) exactly 6 give a different initials() after a pickle/copy round trip than live, the same six the STATED BREAK paragraph names. - docs/release_log.rst: the 2.4.0 Fix bullet now also states that a subclass overriding one of the public first_list/middle_list/ last_list properties keeps working, that member taking the pre-2.4 vocabulary reading. The P3 bullet near decisions.md line 330 (landed on master) is untouched; verified byte-identical to the 338daf7 copy. Co-Authored-By: Claude Sonnet 5 --- docs/design/decisions.md | 7 +++-- docs/release_log.rst | 2 +- nameparser/_render.py | 66 ++++++++++++++++++++++++---------------- tests/test_initials.py | 22 ++++++++++---- tests/v2/test_facade.py | 26 ++++++++++------ 5 files changed, 79 insertions(+), 44 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 69208b4a..6a08469e 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1183,9 +1183,12 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea - 2026-09-01 #462 — DONE, the facade's twin: `HumanName.initials()` dropped a dotted or bare-capital `E`/`Y` from the middle and family groups because `_process_initial` tested conjunction membership alone, where 1.4.0's `is_conjunction` was "in the set AND NOT `is_an_initial`". `Scott E. Werner` gave `S. W.` from 2.0.0 through 2.2.0 and gives `S. E. W.` again; `John E Smith` (bare capital) and `Juan Y. Garcia` likewise; `parse().initials()` never had the bug, since `_classify` tags `E.` an initial and the render reads the tag (mechanisms.md#RENDER-HONORS-THE-PARSE). The issue said the bare form was "still correctly dropped", which is true of lowercase `e` and false of bare capital `E` — 7 of the 14 corpus names that move are bare capitals. Restored with `_render._INITIAL`, v1's own `initial` shape, because the facade may import `_render` and not `_pipeline`; scoped to `_process_initial`, its only caller, so a future reader of `_is_conjunction` does not inherit a decision made for initials. Found by #484's pseudo-field at the 1.4.0 baseline, where it had sat as 14 unreported diffs; the gate could not see it before because the seven fields never moved. - 2026-09-13 #528 — DONE: `HumanName.initials()` reads the parse's tags. The facade's initials view was the last derived view still re-deciding the CONNECTIVE question after the parse had answered it — scoped to that question deliberately, since the PARTICLE question is still re-decided from the lexicon in two places by decision, `_cap_word`'s particle conjunct (`decisions.md#R4`'s NOT DONE bullet) and `_is_particle` here, R4 drawing that boundary per question rather than per view. #462's fix above is why the connective half was still being re-decided: restoring v1's `is_conjunction` ("in the set AND NOT `is_an_initial`") restored a SHAPE test standing in for a tag, which agreed with the parse only while the parser agreed with the shape. #383/#479 ended that the same day it landed — rules.md#P3 now reads a one-case single-letter connective from vocabulary rather than from case, and no shape over the raw word can see it — so the two views of one parse disagreed: `HumanName("john e smith").initials()` gave "j. s." against the core's "j. e. s.", and `HumanName("JUAN Y GARCIA").initials()` gave "J. Y. G." against the core's "J. G.". Both give the core's answer now. This is mechanisms.md#RENDER-HONORS-THE-PARSE applied to the last view that had not taken it, and #458's principle reaching the facade layer. THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. - A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. FORWARDING IS STILL THE ONLY WAY TO RECEIVE THE FIX: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `HumanName("John Quincy Smith").initials()` its "J. Q. S." back. Widening the signature alone does not receive the fix either — `**kwargs`, or an explicit `tokens=None` that the super() call drops. Revised 2026-09-14, before this paragraph ever reached master: the token path now passes the group's own text as `name_part` (`" ".join(tok.text for tok in group)`) rather than an empty placeholder, precisely so an override that ignores the `tokens` keyword falls onto the STRING path and behaves as it did before this upgrade, rather than going quietly blank. Measured 2026-09-14: `WidensOnly("john e smith").initials()` is "j. s.", the pre-#528 answer, where a forwarding override and the library itself both give "j. e. s." — the STRING path's vocabulary fallback reads the middle "e" as the connective and drops it, where the token path's parse-backed reading (restored by #528) does not. `John Quincy Smith` does not witness the difference (no word in it is contested), which is why the divergence has to be measured on a name the fix actually moves. So "accept a `tokens` keyword" is still the wrong instruction on its own; the forwarding is what receives the fix, not merely what avoids a `TypeError`. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. + A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. FORWARDING IS STILL THE ONLY WAY TO RECEIVE THE FIX: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `Forwards("john e smith").initials()` the fix's own answer, "j. e. s.". Widening the signature alone does not receive the fix either — `**kwargs`, or an explicit `tokens=None` that the super() call drops. Revised 2026-09-14, before this paragraph ever reached master: the token path now passes the group's own text as `name_part` (`" ".join(tok.text for tok in group)`) rather than an empty placeholder, precisely so an override that ignores the `tokens` keyword falls onto the STRING path and behaves as it did before this upgrade, rather than going quietly blank. Measured 2026-09-14: `WidensOnly("john e smith").initials()` is "j. s.", the pre-#528 answer, where the forwarding override above and the library itself both give "j. e. s." — the STRING path's vocabulary fallback reads the middle "e" as the connective and drops it, where the token path's parse-backed reading (restored by #528) does not. So "accept a `tokens` keyword" is still the wrong instruction on its own; the forwarding is what receives the fix, not merely what avoids a `TypeError`. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. + + A SECOND STATED BREAK, FOUND AND FIXED IN REVIEW (2026-09-14): before #528, `_initials_lists` read `self.first_list`/`self.middle_list`/`self.last_list` — public properties a v1 subclass may override — and the move to `_list_tokens_for(member)` above dropped that: an overridden property was silently ignored by `initials()` alone, while `last_base`/`surnames`/`given_names` (via `_split_last`, which still reads `self.last_list`) kept honoring it. Fixed by checking, per member, whether the class overrides that property (`getattr(type(self), f"{member}_list") is not getattr(HumanName, f"{member}_list")` — three class-attribute identity checks, nothing on the parse path) and, where it is overridden, taking the pre-#528 STRING path over the override's own strings instead of the token walk. Measured 2026-09-14 against `git show 338daf7:nameparser/_facade.py` (before #528): a three-property `Sub` (`first_list`/`middle_list`/`last_list` all overridden, `last_list` returning `["Zorro"]`) on `"john Xavier smith"` gives "J. Z." both there and here, where the unfixed tree gave "j. X. s."; a one-property `Sub` (only `last_list` overridden, same `["Zorro"]`) on `"john e smith"` gives "j. e. Z." — the un-overridden first/middle keep the (post-#528) token path's own answer, and only the overridden last takes the pre-#528 fallback. Honoring the override, rather than leaving it silently ignored, is why "This is the only break in #528" above still holds: a second silent break on a PUBLIC surface would have gone unnoticed by every existing test, unlike the private-method break above, which is loud by design. ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. - ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-13). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (measured 2026-09-13). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. + ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-14). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (the pickle round trip measured 2026-09-13, copy and deepcopy measured 2026-09-14, all four giving the identical two values). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. + THE POPULATION, MEASURED 2026-09-14: over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names), exactly 6 give a different `initials()` after a pickle round trip than the live parse, up from 0 before #528 — the same six the STATED BREAK paragraph above names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`. `copy.copy` and `copy.deepcopy` move the identical six over the same glob, confirming the state hooks are one mechanism rather than a pickle-specific accident. ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. DIFFERENTIAL. The facade's `_initials` is compared at every baseline, 1.4.0 included, and the core's from 2.0.0; the pseudo-field enters a name's diff only where every role and every ambiguity kind agrees. So at 1.4.0 FOUR of the six take a new rule (`fix(#528) the facade's initials follow the parse's connective tags`) and two do not — `jose e maria santos` and `JUAN GARCIA Y LOPEZ` move roles against that baseline, and #383/#479's role rule explains them. At 2.0.0 through 2.2.0 nothing new: the e-names' `_ambiguities` diff keeps `_initials` out of their diff, and `JUAN Y GARCIA`'s single `_initials` row survives with the facade half of its two causes closed, which `fix(#462)`'s dated paragraph in `expected_since_2.0.0.toml` now says. At 2.3.0 the existing rule stands and both surfaces move together. All five gates re-run at 0 unexplained on 2026-09-13. diff --git a/docs/release_log.rst b/docs/release_log.rst index edd6fac5..a99a5258 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) - - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. A subclass overriding one of the public ``first_list``, ``middle_list`` or ``last_list`` properties keeps working too: that member takes the pre-2.4 vocabulary reading instead of the change above, while an un-overridden member still moves. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) **Additions** diff --git a/nameparser/_render.py b/nameparser/_render.py index 2f816457..b76ef461 100644 --- a/nameparser/_render.py +++ b/nameparser/_render.py @@ -46,13 +46,19 @@ # the empty alternative) -- layering forbids importing the pipeline here; # keep in sync with _pipeline/_vocab.py by hand. # Its one reader is _reads_as_conjunction below, and that reader only -# ever sees text handed over as a bare string -- a spliced field, or a -# direct string call -- never a classified token: for anything the -# parser DID see, the tag is the answer and this pattern is not asked. +# ever sees a bare string with no token attached to it -- a spliced +# field (replace()), restored state (__setstate__: a pickle load, +# copy.copy, or copy.deepcopy), a direct string call, or -- since +# 84d9000 -- a widen-only _process_initial override that drops the +# token it was handed on its way to the string path. The first two +# never had a token to begin with: for anything the parser classified +# AND the caller passed the token along, the tag is the answer and +# this pattern is not asked. The last two might have been classified +# and the reader cannot tell -- it only knows no token was passed. # So the two copies no longer decide the same question about the same # token -- _vocab's says what the parse decided, this one says what it -# WOULD have decided about text spliced in afterwards -- which is why -# they must keep answering alike, and why test_regex_sync pins the +# WOULD have decided about text handed over with no token -- which is +# why they must keep answering alike, and why test_regex_sync pins the # patterns against each other and against config. # Deliberately NOT composed with _vocab's repertoire test (#320): # layering forbids the import. The divergence is reachable only for a @@ -60,32 +66,40 @@ # vocabulary carries one, and it costs nothing there: CJK is caseless, # so the carve-out's lower() and the fall-through's capitalize() return # the same string. Since #528 this pattern is read by more than case -# repair: through _reads_as_conjunction below, asking it only about -# text no parse read -- case repair's spliced field, and the v1 -# facade's initials view, both where a token carries UNCLASSIFIED_TAG -# (_facade._token_is_conjunction) and where there is no token at all -# (_process_initial's bare-string path, a direct call with no parse -# behind it). For initials the CJK divergence would decide whether -# such a spliced connective contributes a letter rather than which -# case it renders in -- still unreachable from any shipped vocabulary, -# and still not worth the import layering forbids. +# repair: through _reads_as_conjunction below, whose callers are case +# repair's spliced field and the v1 facade's initials view, the latter +# in three shapes -- a token carrying UNCLASSIFIED_TAG +# (_facade._token_is_conjunction), a direct _process_initial call with +# no tokens at all, and, since 84d9000, a widen-only _process_initial +# override that drops a token the parse DID classify. For initials the +# CJK divergence would decide whether such a spliced, dropped or +# never-parsed connective contributes a letter rather than which case +# it renders in -- still unreachable from any shipped vocabulary, and +# still not worth the import layering forbids. _INITIAL = re.compile(r"^(\w\.|[A-Z])$") def _reads_as_conjunction(word: str, lex: Lexicon) -> bool: - """v1's is_conjunction, asked only of text the parse never saw. + """v1's is_conjunction, asked only where the CALLER supplies no + token. - A token the parse classified carries its reading in its tags and - this is not consulted. A token carrying UNCLASSIFIED_TAG was - spliced into a field as raw text -- by replace(), or by the - facade's v1 pickle load -- and carries no reading, so the two - views that hold a vocabulary fall back to this: case repair, and - since #528 the v1 facade's initials view. It gives the answer the - parser would have given, the initial carve-out included ('E.' - assigned to middle is an initial, not the Italian conjunction). - What it cannot give is an answer the parse reached by looking at - the whole NAME -- rules.md#P3's one-case fork is the live example - -- which is why it is the fallback and the tags are the rule. + A token the parse classified carries its reading in its tags, and + the library's own token path -- _facade._token_is_conjunction -- + consults that tag first and never reaches here for a token it + holds. This function is reached only where there is no token to + consult: a field spliced in as raw text (replace()) or restored + state (__setstate__: a pickle load, copy.copy, or copy.deepcopy), + both carrying UNCLASSIFIED_TAG; a direct call with no parse behind + it; or, since 84d9000, a widen-only _process_initial override that + drops the token it was handed on its way to the string path -- + text the parse DID classify, whose reading the override chose not + to forward. This function cannot tell any of those apart from one + another; it can only give the answer the parser would have given + from the word alone, the initial carve-out included ('E.' assigned + to middle is an initial, not the Italian conjunction). What it + cannot give is an answer the parse reached by looking at the whole + NAME -- rules.md#P3's one-case fork is the live example -- which + is why it is the fallback and the tags are the rule. """ return bool(_normalize(word) in lex.conjunctions and not _INITIAL.fullmatch(word)) diff --git a/tests/test_initials.py b/tests/test_initials.py index 8c4c041e..09ab620b 100644 --- a/tests/test_initials.py +++ b/tests/test_initials.py @@ -285,12 +285,22 @@ def test_initials_follow_the_one_case_fork_on_both_letters(self) -> None: self.m(hn.initials(), "j. e. s.", hn) hn = HumanName("maria y lopez") self.m(hn.initials(), "m. l.", hn) - # These two back the 1.4.0 ledger rule and its _CROSS_RULE_WINNERS / - # _RECORDED_DIFFS rows (tools/differential/expected_since_1.4.0.toml). - # "jones, john e" is the comma form, where 'e' ends the string -- - # the connective-run regex that reads a trailing letter's neighbors - # never reaches it, so it is a distinct shape from the space-written - # "john e jones" even though both give the same answer. + # These two are among #528's four movers, and the #528 ledger + # rule's own name_regex (a literal alternation, in + # tools/differential/expected_since_1.4.0.toml) covers all four + # names by itself -- but only "john e jones" also has a row in + # _CROSS_RULE_WINNERS (tests/v2/test_ledger_guards.py) and + # _RECORDED_DIFFS (tools/differential/compare.py): those two + # Python dicts, keyed by the LEDGER FILENAME rather than + # anything stored in the TOML, record that + # "fix(initials-per-word) a connective run initials each word" + # also reaches "john e jones" through its lowercase ' e ' and + # contests the name with #528's rule. "jones, john e" has no + # row in either: that connective-run rule's operative half is a + # `(?=\s)` lookahead requiring whitespace AFTER the letter, and + # a letter at the end of the string can never satisfy it -- so + # the comma form is a distinct shape from the space-written + # form even though both give the same initials() answer. hn = HumanName("john e jones") self.m(hn.initials(), "j. e. j.", hn) hn = HumanName("jones, john e") diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 048330db..d18cf9a1 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -685,9 +685,12 @@ def test_list_tokens_for_carries_the_list_view_s_own_elements() -> None: # `all(g for g in groups)` cannot fail by construction either -- # _list_tokens_for never emits an empty group, vacuously true # (including over the empty list every unused member here - # returns). What IS worth pinning is that the walk is - # deterministic: calling it again over the same parse gives - # the identical grouping, not a fresh (if equal-looking) one. + # returns). What IS worth pinning: a second call returns an + # EQUAL grouping. That is not a given for free -- a one-shot + # walk built over an exhausted iterator behind tokens_for() + # would return nothing at all the second time, not merely a + # fresh-but-equal list, so this assert would catch that shape + # of bug too. assert n._list_tokens_for(member) == groups, (name, member) @@ -797,10 +800,13 @@ def test_an_override_that_forwards_tokens_keeps_working() -> None: # STRING path instead of raising or going silent: it gets the # PRE-#528 answer, computed from the vocabulary fallback rather # than the parse. "john e smith" is where the two views disagree - # -- the parse reads the middle "e" as initial-shaped - # (test_process_initial_with_tokens_reads_the_parse), but the - # bare-word vocabulary fallback reads lowercase "e" as the - # connective and drops it -- so WidensOnly keeps working but + # -- the parse TAGS the middle "e" an initial + # (test_process_initial_with_tokens_reads_the_parse); rules.md#P3's + # one-case fork reads it from the vocabulary, not from its shape, + # which is exactly why the bare-word vocabulary fallback below + # reads it the other way -- "e" is not initial-SHAPED by any + # pattern over the bare word, so a fallback with no parse behind it + # reads lowercase "e" as the # without #528's fix. Accepting AND FORWARDING `tokens` is still # the only way to receive the fix. decisions.md#R3's 2026-09-13 # entry (amended 2026-09-14) and the 2.4.0 release note both @@ -938,8 +944,10 @@ def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> Non # prose; decisions.md#R3 records it. # # copy.copy and copy.deepcopy go through the same __getstate__/ - # __setstate__ hooks as pickle (nameparser/_types.py's guarded pair), - # so a copied name takes the identical vocabulary fallback -- measured, + # __setstate__ hooks as pickle -- HumanName's own pair, defined + # right here in nameparser/_facade.py (not nameparser/_types.py's + # guarded pair, which belongs to a different set of classes) -- so + # a copied name takes the identical vocabulary fallback -- measured, # not assumed. for name, live_initials, restored_initials, live_cap, restored_cap in ( ("JUAN Y GARCIA", "J. G.", "J. Y. G.", From 0a3b83755d2311660f5533cbfbcb87678ae4edc7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 20:00:45 -0700 Subject: [PATCH 10/11] docs(design): the no-parse paths, and recipes a reader can rerun Widens the unlanded #528 R3 entry's second accepted cost from "the state-restoration path" to all three no-parse paths: pickle/copy.copy/ copy.deepcopy AND the keyword constructor (HumanName(first=..., ...) never runs the tokenizer either), with measured keyword-constructor numbers and a corpus-wide recompute (1174 distinct names via the two line shapes in tools/differential/corpus*.jsonl, 1 mover before #528, 7 now). Widens the release_log.rst 2.4.0 Fix bullet's parenthetical to match. Corrects the overridden-*_list paragraph's fixture description (all three overrides, not just last_list) and adds its "Pinned at" pointer. Fixes a misattributed parenthetical (only last_base routes through _split_last; surnames and given_names never do). Fixes "all four" to "all three" for the pickle/copy.copy/copy.deepcopy count. Adds a one-sentence reading recipe for "(1174 distinct names)" at its first occurrence. Renames the wrong method in the _initials_lists docstring in nameparser/_facade.py (_list_tokens_for, not first/middle/last, is the private token walk). Co-Authored-By: Claude Fable 5.1 --- docs/design/decisions.md | 6 +++--- docs/release_log.rst | 2 +- nameparser/_facade.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 6a08469e..bd8393f9 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -1185,10 +1185,10 @@ R3's earlier history is under `decisions.md#R2`, which this entry does not repea THE SHAPE IS R4's, not a new one. A word backed by a token asks `"conjunction" in tok.tags`; a token carrying `UNCLASSIFIED_TAG` — text spliced in by `hn.middle = ...` through `ParsedName.replace()`, or restored by the v1 pickle path in `__setstate__` — was read by no parse, so the vocabulary answers, through `_render._reads_as_conjunction`, the same helper `_cap_word` calls for the same tokens. One question and not two: whether a word is a connective is a fact the word can answer alone, while whether a particle is acting as a particle is a fact about the whole part, which is R4's own boundary and is why `_is_particle` stays a live vocabulary lookup here. `_is_conjunction` lost its only caller and is gone, which retires the last sentence of the 2026-09-01 entry above — there is no future reader of it to inherit a decision made for initials. A STATED PRIVATE-METHOD BREAK, accepted rather than worked around. `_process_initial` keeps v1's `(name_part, firstname=False)` signature for direct callers and gains an optional `tokens=`, and `_initials_lists` always passes it — so a v1-shaped subclass that overrides `_process_initial` with the two-argument signature now raises `TypeError` the first time `initials()` runs. The alternative considered was a string wrapper kept over a token core, which would leave such an override silently INEFFECTIVE instead: it would be called, and its answer discarded by the token path beside it. A loud break on a private hook beats a quiet one, and it is stated in the method's own comment and pinned by a test. FORWARDING IS STILL THE ONLY WAY TO RECEIVE THE FIX: an override must ACCEPT `tokens` **and forward it**, `def _process_initial(self, name_part, firstname=False, tokens=None): return super()._process_initial(name_part, firstname, tokens=tokens)`, which gives `Forwards("john e smith").initials()` the fix's own answer, "j. e. s.". Widening the signature alone does not receive the fix either — `**kwargs`, or an explicit `tokens=None` that the super() call drops. Revised 2026-09-14, before this paragraph ever reached master: the token path now passes the group's own text as `name_part` (`" ".join(tok.text for tok in group)`) rather than an empty placeholder, precisely so an override that ignores the `tokens` keyword falls onto the STRING path and behaves as it did before this upgrade, rather than going quietly blank. Measured 2026-09-14: `WidensOnly("john e smith").initials()` is "j. s.", the pre-#528 answer, where the forwarding override above and the library itself both give "j. e. s." — the STRING path's vocabulary fallback reads the middle "e" as the connective and drops it, where the token path's parse-backed reading (restored by #528) does not. So "accept a `tokens` keyword" is still the wrong instruction on its own; the forwarding is what receives the fix, not merely what avoids a `TypeError`. This is the only break in #528; on the public v1 surface nothing moves but the initials of six corpus names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`, the six the ONE CORPUS NAME paragraph below counts. - A SECOND STATED BREAK, FOUND AND FIXED IN REVIEW (2026-09-14): before #528, `_initials_lists` read `self.first_list`/`self.middle_list`/`self.last_list` — public properties a v1 subclass may override — and the move to `_list_tokens_for(member)` above dropped that: an overridden property was silently ignored by `initials()` alone, while `last_base`/`surnames`/`given_names` (via `_split_last`, which still reads `self.last_list`) kept honoring it. Fixed by checking, per member, whether the class overrides that property (`getattr(type(self), f"{member}_list") is not getattr(HumanName, f"{member}_list")` — three class-attribute identity checks, nothing on the parse path) and, where it is overridden, taking the pre-#528 STRING path over the override's own strings instead of the token walk. Measured 2026-09-14 against `git show 338daf7:nameparser/_facade.py` (before #528): a three-property `Sub` (`first_list`/`middle_list`/`last_list` all overridden, `last_list` returning `["Zorro"]`) on `"john Xavier smith"` gives "J. Z." both there and here, where the unfixed tree gave "j. X. s."; a one-property `Sub` (only `last_list` overridden, same `["Zorro"]`) on `"john e smith"` gives "j. e. Z." — the un-overridden first/middle keep the (post-#528) token path's own answer, and only the overridden last takes the pre-#528 fallback. Honoring the override, rather than leaving it silently ignored, is why "This is the only break in #528" above still holds: a second silent break on a PUBLIC surface would have gone unnoticed by every existing test, unlike the private-method break above, which is loud by design. + A SECOND STATED BREAK, FOUND AND FIXED IN REVIEW (2026-09-14): before #528, `_initials_lists` read `self.first_list`/`self.middle_list`/`self.last_list` — public properties a v1 subclass may override — and the move to `_list_tokens_for(member)` above dropped that: an overridden property was silently ignored by `initials()` alone, while `last_base` (via `_split_last`, which reads `self.last_list`), `surnames` (`self.middle_list + self.last_list`) and `given_names` (`self.first_list + self.middle_list`, which never reaches `last_list` at all) kept honoring it. Fixed by checking, per member, whether the class overrides that property (`getattr(type(self), f"{member}_list") is not getattr(HumanName, f"{member}_list")` — three class-attribute identity checks, nothing on the parse path) and, where it is overridden, taking the pre-#528 STRING path over the override's own strings instead of the token walk. Measured 2026-09-14 against `git show 338daf7:nameparser/_facade.py` (before #528): a three-property `Sub` (`first_list` uppercasing every element of `super().first_list`; `middle_list` dropping every element of `super().middle_list` that starts with `"X"`; `last_list` always returning `["Zorro"]`) on `"john Xavier smith"` gives "J. Z." both there and here — reading only the `last_list` override into the recipe predicts "j. X. Z." instead (measured 2026-09-14 with `last_list` alone overridden on this tree), since the upper-cased first and the filtered-out `Xavier` both feed the result too — where the unfixed tree gave "j. X. s."; a one-property `Sub` (only `last_list` overridden, same `["Zorro"]`) on `"john e smith"` gives "j. e. Z." — the un-overridden first/middle keep the (post-#528) token path's own answer, and only the overridden last takes the pre-#528 fallback. Honoring the override, rather than leaving it silently ignored, is why "This is the only break in #528" above still holds: a second silent break on a PUBLIC surface would have gone unnoticed by every existing test, unlike the private-method break above, which is loud by design. Pinned at `tests/v2/test_facade.py::test_initials_honor_an_overridden_list_property`. ACCEPTED COST, MEASURED (2026-09-13, this branch): for a word backed by a token the connective answer is fixed at parse time, as `capitalize()`'s already was. `C = Constants(); h = HumanName("juan y garcia", constants=C)` gives "j. g."; `C.conjunctions.remove("y")` leaves it "j. g." with no re-parse, where before #528 it gave "j. y. g." immediately; `h.full_name = "juan y garcia"` re-parses and gives "j. y. g.". Unpinned before this change and pinned by it, at `tests/v2/test_facade.py::test_initials_freeze_the_connective_answer_at_parse_time`. - ACCEPTED COST, THE SECOND ONE, and it is the state-restoration path rather than the configuration: `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout and every word takes the vocabulary fallback. This reaches a name restored from a pickle, or copied: `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-14). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (the pickle round trip measured 2026-09-13, copy and deepcopy measured 2026-09-14, all four giving the identical two values). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — so this is the existing shape reaching one more view rather than a new one, and the remedy is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. - THE POPULATION, MEASURED 2026-09-14: over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names), exactly 6 give a different `initials()` after a pickle round trip than the live parse, up from 0 before #528 — the same six the STATED BREAK paragraph above names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`. `copy.copy` and `copy.deepcopy` move the identical six over the same glob, confirming the state hooks are one mechanism rather than a pickle-specific accident. + ACCEPTED COST, THE SECOND ONE, and it is the no-parse paths rather than the configuration, THREE of them, not one: state restoration (pickle, `copy.copy`, `copy.deepcopy`) and the keyword constructor all skip the tokenizer, so every word they carry takes the vocabulary fallback instead of a parse's tag. `__setstate__` stamps every restored token `UNCLASSIFIED_TAG`, because a v1 pickle carries the `*_list` STRINGS and no tags, so a restored name is spliced text throughout; `copy.copy` and `copy.deepcopy` use the same `__getstate__`/`__setstate__` state hooks, so a copy takes the identical fallback (measured 2026-09-14). The THIRD producer, unrecorded until this review round: `HumanName(first=..., middle=..., last=...)` routes straight through the field-setter properties in `__init__` and never calls `_apply_full_name`, so its tokens are `UNCLASSIFIED_TAG` for the identical reason a restored pickle's are — not a new bug, a third route to a cost already accepted. `HumanName(first="john", middle="e", last="smith").initials()` gives "j. s." where `HumanName("john e smith").initials()` gives "j. e. s."; `HumanName(first="JUAN", middle="Y", last="GARCIA").initials()` gives "J. Y. G." where `HumanName("JUAN Y GARCIA").initials()` gives "J. G." (measured 2026-09-14). A round-tripped, copied or deep-copied `JUAN Y GARCIA` therefore gives "J. Y. G." where the live parse gives "J. G.", and a round-tripped, copied or deep-copied `john e smith` gives "j. s." where the live parse gives "j. e. s." (the pickle round trip measured 2026-09-13, copy and deepcopy measured 2026-09-14, all three giving the identical two values). `capitalize()` has disagreed with the live parse on exactly those two names for exactly that reason since the tag was introduced — "Juan Y Garcia" against "Juan y Garcia", "John e Smith" against "John E Smith", all four measured the same day — and the keyword constructor gives the identical disagreement, `HumanName(first="john", middle="e", last="smith").capitalize()` leaving "John e Smith" against the live parse's "John E Smith" on both this tree and 338daf7 (measured 2026-09-14) — so this is the existing shape reaching one more view rather than a new one, and the remedy for the state-restoration pair is the one `__setstate__`'s own comment gives: the restored state is a v1 blob, not a parse; the keyword constructor has no state to restore an argument from — the caller supplied fields, not a string, so there was never a tokenizer run to have tagged them. RECOMPUTE ACROSS THE CORPUS, ALL THREE PATHS TOGETHER: rebuild every one of the deduped `tools/differential/corpus*.jsonl` glob's 1174 distinct names from its own parse's seven keyword-constructible fields (`first`, `middle`, `last`, `title`, `suffix`, `nickname`, `maiden`) and diff `initials()` against the live parse — 1 of 1174 differs before #528 (`Ph. D., John`, the pre-existing merge the ONE CORPUS NAME paragraph below names, unrelated to this mechanism) and 7 now, the added six being the same six `initials()` movers named throughout this entry. Pinned at `tests/v2/test_facade.py::test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too`. + THE POPULATION, MEASURED 2026-09-14: the deduped `tools/differential/corpus*.jsonl` glob holds two line shapes — `{"name": ...}` objects in `corpus.jsonl` and `corpus_shapes.jsonl`, bare JSON strings in the other four — so reading both and deduping is what gives its 1174 distinct names; a dict-only read stops at 528. Over that glob, exactly 6 give a different `initials()` after a pickle round trip than the live parse, up from 0 before #528 — the same six the STATED BREAK paragraph above names: `john e smith`, `john e jones`, `jones, john e`, `jose e maria santos`, `JUAN GARCIA Y LOPEZ` and `JUAN Y GARCIA`. `copy.copy` and `copy.deepcopy` move the identical six over the same glob, confirming the state hooks are one mechanism rather than a pickle-specific accident. ONE CORPUS NAME STILL DIVERGES and it is not this rule's. Measured over the deduped `tools/differential/corpus*.jsonl` glob (1174 distinct names) before and after: seven names had the two views disagreeing, six moved here, and `Ph. D., John` remains — the facade merges "Ph." and "D." into ONE list element (v1's `fix_phd`) and renders "P D" with the separator and no inner delimiter, giving "J. P D." against the core's "J. P. D.". That is `fix(initials-per-word) the Ph. D. merge`, a 2.0.0 rendering change ledgered at 1.4.0 since #484, and #528 preserves the element boundaries exactly so it neither moves nor is absorbed. RECOMPUTE by parsing every name of the glob on both surfaces and diffing `initials()`; the before half needs the pre-#528 `_facade.py` and `_render.py` on the path, which `git show` writes into a scratch copy of the package — never a checkout in a shared worktree. DIFFERENTIAL. The facade's `_initials` is compared at every baseline, 1.4.0 included, and the core's from 2.0.0; the pseudo-field enters a name's diff only where every role and every ambiguity kind agrees. So at 1.4.0 FOUR of the six take a new rule (`fix(#528) the facade's initials follow the parse's connective tags`) and two do not — `jose e maria santos` and `JUAN GARCIA Y LOPEZ` move roles against that baseline, and #383/#479's role rule explains them. At 2.0.0 through 2.2.0 nothing new: the e-names' `_ambiguities` diff keeps `_initials` out of their diff, and `JUAN Y GARCIA`'s single `_initials` row survives with the facade half of its two causes closed, which `fix(#462)`'s dated paragraph in `expected_since_2.0.0.toml` now says. At 2.3.0 the existing rule stands and both surfaces move together. All five gates re-run at 0 unexplained on 2026-09-13. diff --git a/docs/release_log.rst b/docs/release_log.rst index a99a5258..7b567dab 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Fix a one-letter connective joining a name that gives no sign it is a connective.** ``HumanName("jose e maria santos")`` gives first ``jose``, middle ``e maria``, last ``santos``, where 1.4.0 through 2.3.0 gave first ``jose e maria``; and ``JUAN GARCIA Y LOPEZ`` gives last ``GARCIA Y LOPEZ``, where every release since 1.4.0 read the bare capital as an initial and gave middle ``GARCIA Y``. A single letter is an initial where the writing says so -- a bare Latin capital in a name that is not written wholly in one case -- and a name written wholly in one case says nothing either way, so the reading comes from the vocabulary there: ``e`` reads as an initial and ``y`` joins. Mixed-case input is untouched in both directions: ``Jose e Maria Santos`` still gives first ``Jose e Maria`` and ``Jose E Maria Santos`` still gives middle ``E Maria``. Short names move in the derived views rather than the fields, P3's three-word carve-out being unchanged: ``parse("john e smith").initials()`` is ``j. e. s.`` where 2.3.0 gave ``j. s.``, and ``HumanName("john e smith").capitalize()`` gives ``John E Smith`` where 2.3.0 gave ``John e Smith``; ``JUAN Y GARCIA`` moves the same way in reverse, ``parse(...).initials()`` giving ``J. G.`` where 2.3.0 gave ``J. Y. G.`` and ``capitalize()`` giving ``Juan y Garcia``. ``HumanName.initials()`` moves with them -- see the #528 bullet below, which closed a split this change opened and the same release closes. Seventeen names in the differential corpora are written in one case and carry a cased single-letter connective, and ten of them move something against 2.3.0. The Cyrillic reading is unchanged (``Хосе И Мария Сантос`` still gives first ``Хосе И Мария``), and Arabic ``و`` never enters the rule, having no case to be written against. A ``Lexicon`` knob decides which letters are marked, so the reading is configurable rather than fixed. See the ``P3`` entry of ``docs/design/decisions.md`` (closes #383, closes #479) - - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, or copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. A subclass overriding one of the public ``first_list``, ``middle_list`` or ``last_list`` properties keeps working too: that member takes the pre-2.4 vocabulary reading instead of the change above, while an un-overridden member still moves. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) + - **Fix HumanName.initials() reading a one-letter connective by vocabulary and written shape instead of by the parse.** ``HumanName("john e smith").initials()`` gives ``j. e. s.``, where every release from 1.4.0 through 2.3.0 gave ``j. s.``; ``JUAN Y GARCIA`` gives ``J. G.`` where 2.3.0 gave ``J. Y. G.``, and ``JUAN GARCIA Y LOPEZ`` gives ``J. G. L.`` where 2.3.0 gave ``J. G. Y. L.``. Those last two read 1.4.0's way at 2.3.0 and only there: 2.0.0 through 2.2.0 already gave today's answer, by the unrelated bug the 2.3.0 note below records as fixed (the facade dropping a bare capital that is also a one-letter conjunction, #462), so against those three releases neither name moves at all. The v1 facade decided whether a word was the connective by looking the word up and checking its shape, while ``parse(...).initials()`` read the tag the parse recorded -- so the change above, which reads a single letter in a one-case name from the vocabulary rather than from its case, moved one view and not the other. Both views of a parse now give the same answer. Mixed-case names are untouched on both, the writing having decided the letter: ``John E Smith`` is still ``J. E. S.`` and ``Scott E. Werner`` still ``S. E. W.``. So is a one-case name whose letter is outside the marked set -- ``maria y lopez`` is still ``m. l.``, ``y`` having joined before this release and after it. Two costs, and both match what ``capitalize()`` has always done: editing ``C.conjunctions`` after a name is parsed no longer changes its initials until ``full_name`` is assigned again, and a name restored from a pickle, copied with ``copy.copy``/``copy.deepcopy`` (the same state hooks), or built from keyword fields (``HumanName(first=..., middle=..., last=...)``) carries no tags, so its initials come from the vocabulary and can differ from a fresh parse of the same string. One private break, stated because a v1 subclass can hit it: an override of ``_process_initial`` written to v1's ``(name_part, firstname=False)`` signature now raises ``TypeError`` the first time ``initials()`` runs, since ``initials()`` passes the part's tokens. Such an override has to accept a ``tokens`` keyword *and pass it on* -- ``return super()._process_initial(name_part, firstname, tokens=tokens)`` -- to receive this fix. Widening the signature without forwarding still works, but on the pre-#528 STRING path: the token call hands the override the group's own text as ``name_part`` rather than an empty placeholder, so ``john e smith`` initials ``j. s.`` under such an override, not the ``j. e. s.`` above. A subclass overriding one of the public ``first_list``, ``middle_list`` or ``last_list`` properties keeps working too: that member takes the pre-2.4 vocabulary reading instead of the change above, while an un-overridden member still moves. See the ``R3`` entry of ``docs/design/decisions.md`` (closes #528) **Additions** diff --git a/nameparser/_facade.py b/nameparser/_facade.py index 510d426e..c0642463 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -658,9 +658,9 @@ def _initials_lists(self) -> tuple[list[str], list[str], list[str]]: parse gave it; the elements are the list view's own, folded first and continuations merged, because one walk builds both. A member whose PUBLIC `first_list`/`middle_list`/`last_list` - property is overridden is the one exception: `first`/`middle`/ - `last` (the private token walk this method otherwise uses) do - not consult that override, so honoring it means reading the + property is overridden is the one exception: `_list_tokens_for` + (the private token walk this method otherwise uses) does not + consult that override, so honoring it means reading the override's own strings instead -- the pre-#528 walk, degraded to the vocabulary fallback exactly as a widen-only `_process_initial` override is (STATED BREAK, below): the From 5eef00d722362e8673c5a1da24457a85cc5c027e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 14 Sep 2026 20:06:49 -0700 Subject: [PATCH 11/11] test(#528): the keyword constructor is pinned beside pickle and copy The decisions entry's second cost now names three no-parse paths and cites one test; that test covered two of them. A name rebuilt from the live parse's own fields now asserts the same fallback values, so the citation covers what the paragraph claims. The override test's comment also repeats the corrected attribution: only last_base goes through _split_last; surnames and given_names read the properties directly. Co-Authored-By: Claude Fable 5.1 --- tests/v2/test_facade.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index d18cf9a1..a89b7b3a 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -848,13 +848,14 @@ def _process_initial(self, name_part: str, # type: ignore[override] def test_initials_honor_an_overridden_list_property() -> None: - # F1, second review round: before #528, _initials_lists read + # Found in review: before #528, _initials_lists read # self.first_list/middle_list/last_list -- public properties a v1 # subclass may override -- and #528 switched it to the private # token walk (_list_tokens_for) directly, which does not consult - # such an override. last_base/surnames/given_names still honor it - # (they route through _split_last, which reads self.last_list), so - # initials() alone went silently stale. The fix detects an + # such an override. last_base still honors it through _split_last, + # which reads self.last_list; surnames is middle_list + last_list + # and given_names is first_list + middle_list, so both read the + # properties directly. initials() alone went silently stale. The fix detects an # overridden property per member (a cheap class-attribute identity # check) and, for that member only, takes the pre-#528 STRING path # over the override's own strings -- same degradation a @@ -949,6 +950,12 @@ def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> Non # guarded pair, which belongs to a different set of classes) -- so # a copied name takes the identical vocabulary fallback -- measured, # not assumed. + # + # The keyword constructor is the third no-parse path: a name built + # from its fields never ran the full-string parse, so its tokens + # carry the same mark and take the same fallback. Rebuilt here from + # the live parse's own fields, so the strings are identical and + # only the missing parse explains the difference. for name, live_initials, restored_initials, live_cap, restored_cap in ( ("JUAN Y GARCIA", "J. G.", "J. Y. G.", "Juan y Garcia", "Juan Y Garcia"), @@ -962,7 +969,12 @@ def test_initials_of_an_unpickled_or_copied_name_ask_the_vocabulary_too() -> Non restored = pickle.loads(pickle.dumps(HumanName(name))) assert restored.initials() == restored_initials live = HumanName(name) + built = HumanName(first=live.first, middle=live.middle, + last=live.last) + assert built.initials() == restored_initials live.capitalize() assert str(live) == live_cap restored.capitalize() assert str(restored) == restored_cap + built.capitalize() + assert str(built) == restored_cap