diff --git a/.github/workflows/templates/tinytex-issue-body.md b/.github/workflows/templates/tinytex-issue-body.md index 5dde77d3b7d..e58f5b6efba 100644 --- a/.github/workflows/templates/tinytex-issue-body.md +++ b/.github/workflows/templates/tinytex-issue-body.md @@ -1,6 +1,6 @@ ## TinyTeX Pattern Update: {{DATE}} -The daily TinyTeX regex patterns have changed and need review. +The daily TinyTeX patterns (`regex.json`) or hyphenation map (`hyphen.json`) have changed and need review. ### Pattern Diff @@ -22,6 +22,7 @@ See [dev-docs/tinytex-pattern-maintenance.md](./dev-docs/tinytex-pattern-mainten - [ ] Review diff for significant changes - [ ] Determine if patterns need adaptation - [ ] Update `parse-error.ts` if needed +- [ ] If `hyphen.json` changed, update `kHyphenationPackageNames` in `parse-error.ts` and the copy in the unit test - [ ] Add/update filter functions - [ ] Run tests: `unit\latexmk\parse-error.test.ts` - [ ] Add test cases for new patterns if needed diff --git a/.github/workflows/verify-tinytex-patterns.yml b/.github/workflows/verify-tinytex-patterns.yml index c7654dadd32..9890da59d3f 100644 --- a/.github/workflows/verify-tinytex-patterns.yml +++ b/.github/workflows/verify-tinytex-patterns.yml @@ -19,7 +19,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Download and extract regex.json + - name: Download and extract regex.json and hyphen.json env: GH_TOKEN: ${{ github.token }} run: | @@ -28,33 +28,55 @@ jobs: exit 0 fi tar -xzf regex.tar.gz - echo "✓ Downloaded and extracted regex.json" + # Files compared by the later steps + PATTERN_FILES="regex.json hyphen.json" + if [ ! -f hyphen.json ]; then + echo "::warning::hyphen.json not found in regex.tar.gz" + PATTERN_FILES="regex.json" + fi + echo "PATTERN_FILES=$PATTERN_FILES" >> "$GITHUB_ENV" + echo "✓ Downloaded and extracted: $PATTERN_FILES" - - name: Restore cached regex.json + - name: Restore cached patterns id: cache-restore uses: actions/cache/restore@v5 with: - path: .cache/regex.json - key: tinytex-regex-latest + path: .cache + key: tinytex-patterns - name: Compare versions id: compare run: | - if [ -f .cache/regex.json ]; then - if git diff --no-index --quiet .cache/regex.json regex.json; then - echo "changed=false" >> $GITHUB_OUTPUT - echo "first_run=false" >> $GITHUB_OUTPUT - echo "✓ No changes detected" - else - echo "changed=true" >> $GITHUB_OUTPUT - echo "first_run=false" >> $GITHUB_OUTPUT - echo "✗ Changes detected" - git diff --no-index .cache/regex.json regex.json > pattern-diff.txt || true - fi - else + # The download step sets PATTERN_FILES only when it extracted the files + if [ -z "${PATTERN_FILES:-}" ]; then + echo "changed=false" >> $GITHUB_OUTPUT + echo "first_run=false" >> $GITHUB_OUTPUT + echo "skipped=true" >> $GITHUB_OUTPUT + echo "::warning::No pattern files downloaded - comparison skipped" + exit 0 + fi + + if [ ! -f .cache/regex.json ]; then echo "changed=false" >> $GITHUB_OUTPUT echo "first_run=true" >> $GITHUB_OUTPUT echo "⚠ No cached version (first run)" + exit 0 + fi + + changed=false + for f in $PATTERN_FILES; do + if [ ! -f ".cache/$f" ]; then + echo "✗ $f has no cached version" + changed=true + elif ! git diff --no-index --quiet ".cache/$f" "$f"; then + echo "✗ Changes detected in $f" + changed=true + fi + done + echo "changed=$changed" >> $GITHUB_OUTPUT + echo "first_run=false" >> $GITHUB_OUTPUT + if [ "$changed" = "false" ]; then + echo "✓ No changes detected" fi - name: Handle first run @@ -69,6 +91,10 @@ jobs: # Count patterns and categories PATTERN_COUNT=$(jq '[.[] | length] | add' regex.json) CATEGORY_COUNT=$(jq 'keys | length' regex.json) + # A checked assignment: invalid JSON must fail the step before it is cached + if [ -f hyphen.json ]; then + HYPHEN_COUNT=$(jq 'length' hyphen.json) + fi # Write GitHub Actions summary echo "## TinyTeX Pattern Baseline Established" >> "$GITHUB_STEP_SUMMARY" @@ -77,18 +103,23 @@ jobs: echo "- **TinyTeX commit:** [\`$TINYTEX_SHORT\`](https://github.com/rstudio/tinytex/commit/$TINYTEX_COMMIT)" >> "$GITHUB_STEP_SUMMARY" echo "- **Pattern source:** [R/latex.R](https://github.com/rstudio/tinytex/blob/$TINYTEX_COMMIT/R/latex.R)" >> "$GITHUB_STEP_SUMMARY" echo "- **Baseline:** $PATTERN_COUNT patterns across $CATEGORY_COUNT categories" >> "$GITHUB_STEP_SUMMARY" - echo "- **Cache key:** tinytex-regex-latest" >> "$GITHUB_STEP_SUMMARY" + if [ -f hyphen.json ]; then + echo "- **Hyphenation map:** $HYPHEN_COUNT names" >> "$GITHUB_STEP_SUMMARY" + fi + echo "- **Cache key:** tinytex-patterns" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "No issue created (first run - baseline established)." >> "$GITHUB_STEP_SUMMARY" # Prepare cache directory mkdir -p .cache - cp regex.json .cache/regex.json + for f in $PATTERN_FILES; do + cp "$f" ".cache/$f" + done echo "✓ Baseline established - cache will be saved" - name: Exit if unchanged - if: steps.compare.outputs.changed == 'false' && steps.compare.outputs.first_run == 'false' + if: steps.compare.outputs.changed == 'false' && steps.compare.outputs.first_run == 'false' && steps.compare.outputs.skipped != 'true' run: | echo "No pattern changes detected. Cache hit - exiting." exit 0 @@ -96,15 +127,18 @@ jobs: - name: Prepare readable diff if: steps.compare.outputs.changed == 'true' run: | - # Pretty-print both JSON files for readable diff - if [ -f .cache/regex.json ]; then - jq --sort-keys . .cache/regex.json > old-formatted.json - jq --sort-keys . regex.json > new-formatted.json - git diff --no-index old-formatted.json new-formatted.json > readable-diff.txt || true - else - jq --sort-keys . regex.json > new-formatted.json - echo "First run - no previous version to compare" > readable-diff.txt - fi + # Pretty-print both versions of each file for a readable diff + : > readable-diff.txt + for f in $PATTERN_FILES; do + if [ -f ".cache/$f" ]; then + jq --sort-keys . ".cache/$f" > "old-formatted-$f" + else + old="/dev/null" + fi + jq --sort-keys . "$f" > "new-formatted-$f" + git diff --no-index "${old:-old-formatted-$f}" "new-formatted-$f" >> readable-diff.txt || true + unset old + done - name: Create or update issue if: steps.compare.outputs.changed == 'true' @@ -147,7 +181,9 @@ jobs: if: steps.compare.outputs.changed == 'true' run: | mkdir -p .cache - cp regex.json .cache/regex.json + for f in $PATTERN_FILES; do + cp "$f" ".cache/$f" + done echo "✓ Cache updated with new patterns" - name: Delete old cache @@ -156,19 +192,21 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - gh cache delete tinytex-regex-latest || echo "No existing cache to delete" + gh cache delete tinytex-patterns || echo "No existing cache to delete" - name: Save new cache if: steps.compare.outputs.changed == 'true' || steps.compare.outputs.first_run == 'true' uses: actions/cache/save@v5 with: - path: .cache/regex.json - key: tinytex-regex-latest + path: .cache + key: tinytex-patterns - name: Summary if: always() run: | - if [ "${{ steps.compare.outputs.first_run }}" == "true" ]; then + if [ "${{ steps.compare.outputs.skipped }}" == "true" ]; then + echo "⚠ Download failed - comparison skipped" + elif [ "${{ steps.compare.outputs.first_run }}" == "true" ]; then echo "✓ Baseline established - cache created" elif [ "${{ steps.compare.outputs.changed }}" == "true" ]; then echo "✗ Pattern changes detected - issue created/updated" diff --git a/dev-docs/tinytex-pattern-maintenance.md b/dev-docs/tinytex-pattern-maintenance.md index adf20b1c159..037ceb6863e 100644 --- a/dev-docs/tinytex-pattern-maintenance.md +++ b/dev-docs/tinytex-pattern-maintenance.md @@ -11,6 +11,8 @@ The R package **tinytex** maintains a comprehensive database of LaTeX error patt - Every problem will be fixed in the R package first - Low update frequency (~4 changes/year) makes manual adaptation practical +`regex.tar.gz` also contains `hyphen.json`, tinytex's map from language names to the package with their hyphenation patterns (only the exceptions to `hyphen-`). It is generated from the `execute AddHyphen` lines of `texlive.tlpdb` (`R/hyphen.R`). + **Our process:** - Daily automated check detects when TinyTeX patterns change @@ -52,6 +54,16 @@ tinytex R package and Quarto LaTeX engine use patterns differently: } ``` +## Hyphenation packages + +`kHyphenationPackageNames` in [parse-error.ts](../src/command/render/latexmk/parse-error.ts) is a copy of `hyphen.json`, and the unit tests keep a second copy. Detection of missing hyphenation patterns follows tinytex's `hyphen_pkgs()`, `check_babel()`, `detect_hyphen()` and `check_polyglossia()` in `R/latex.R`. + +Quarto-only differences: + +- the `chinese` and `chinese-hans` skip list ([#10291](https://github.com/quarto-dev/quarto-cli/issues/10291)) +- the `Hyphen rules for '...' set to \l@nil` fallback +- unquoted language names in the babel "You haven't defined the language" error regex + ## Manual Adaptation Process When the automated workflow detects TinyTeX pattern changes, it creates/updates a GitHub issue with: @@ -90,7 +102,7 @@ When the automated workflow detects TinyTeX pattern changes, it creates/updates The automated workflow runs daily: 1. Downloads `regex.tar.gz` from [TinyTeX releases](https://github.com/rstudio/tinytex-releases) -2. Extracts and compares `regex.json` with cached version +2. Extracts and compares `regex.json` and `hyphen.json` with the cached versions 3. If changed: generates diff and creates/updates issue 4. If unchanged: exits early (no notification) @@ -115,4 +127,5 @@ The automated workflow runs daily: - [parse-error.ts](../src/command/render/latexmk/parse-error.ts) - Pattern implementation - [parse-error.test.ts](../tests/unit/latexmk/parse-error.test.ts) - Unit tests - [TinyTeX R source](https://github.com/rstudio/tinytex/blob/main/R/latex.R) - How patterns are used in R +- [TinyTeX hyphenation map](https://github.com/rstudio/tinytex/blob/main/R/hyphen.R) - [TinyTeX releases](https://github.com/rstudio/tinytex-releases) - Source of regex.json diff --git a/news/changelog-1.11.md b/news/changelog-1.11.md index ea88ca6ae90..1c907e87c8f 100644 --- a/news/changelog-1.11.md +++ b/news/changelog-1.11.md @@ -52,6 +52,7 @@ All changes included in 1.11: - ([#14982](https://github.com/quarto-dev/quarto-cli/issues/14982)): Fix a footnote in document metadata, such as `title` or `author`, making PDF and Typst output shift heading levels, which turned the first level-1 heading into the document title. - ([#15027](https://github.com/quarto-dev/quarto-cli/issues/15027)): Fix PDF rendering with regional German variants such as `lang: de-CH` failing when German hyphenation patterns are not installed, instead of installing them automatically. +- ([#15035](https://github.com/quarto-dev/quarto-cli/issues/15035)): Fix automatic installation of missing hyphenation patterns for the pinned German patterns (`ngerman-x-latest`), for several languages at once, and for languages loaded with `polyglossia`. ### `revealjs` diff --git a/src/command/render/latexmk/parse-error.ts b/src/command/render/latexmk/parse-error.ts index a9c1588b0c0..04c7cdbfbc7 100644 --- a/src/command/render/latexmk/parse-error.ts +++ b/src/command/render/latexmk/parse-error.ts @@ -157,78 +157,110 @@ export function findPdfAccessibilityWarnings( } // Hyphenation pattern names (in language.dat) whose package is not -// hyphen-, according to `execute AddHyphen` in texlive.tlpdb +// hyphen-, with the package name. Generated in tinytex from +// `execute AddHyphen` in texlive.tlpdb (R/hyphen.R), and exported as +// hyphen.json in the daily regex.tar.gz. const kHyphenationPackageNames: Record = { - ngerman: "german", - swissgerman: "german", - pinyin: "chinese", - ukenglish: "english", - british: "english", - usenglishmax: "english", - bokmal: "norwegian", - norsk: "norwegian", - nynorsk: "norwegian", - schoolfinnish: "finnish", - monogreek: "greek", - ibycus: "ancientgreek", - classiclatin: "latin", - liturgicallatin: "latin", - mongolianlmc: "mongolian", - serbianc: "serbian", + ngerman: "hyphen-german", + swissgerman: "hyphen-german", + pinyin: "hyphen-chinese", + ukenglish: "hyphen-english", + british: "hyphen-english", + usenglishmax: "hyphen-english", + bokmal: "hyphen-norwegian", + norsk: "hyphen-norwegian", + nynorsk: "hyphen-norwegian", + schoolfinnish: "hyphen-finnish", + monogreek: "hyphen-greek", + ibycus: "hyphen-ancientgreek", + classiclatin: "hyphen-latin", + liturgicallatin: "hyphen-latin", + mongolianlmc: "hyphen-mongolian", + serbianc: "hyphen-serbian", // older names (synonyms in texlive.tlpdb) - amharic: "ethiopic", - geez: "ethiopic", - persian: "farsi", - patois: "french", - francais: "french", - polygreek: "greek", - portuges: "portuguese", - slovene: "slovenian", - espanol: "spanish", + amharic: "hyphen-ethiopic", + geez: "hyphen-ethiopic", + persian: "hyphen-farsi", + patois: "hyphen-french", + francais: "hyphen-french", + polygreek: "hyphen-greek", + portuges: "hyphen-portuguese", + slovene: "hyphen-slovenian", + espanol: "hyphen-spanish", // all of these are in the hyphen-indic package - assamese: "indic", - bengali: "indic", - gujarati: "indic", - hindi: "indic", - kannada: "indic", - malayalam: "indic", - marathi: "indic", - oriya: "indic", - pali: "indic", - panjabi: "indic", - tamil: "indic", - telugu: "indic", + assamese: "hyphen-indic", + bengali: "hyphen-indic", + gujarati: "hyphen-indic", + hindi: "hyphen-indic", + kannada: "hyphen-indic", + malayalam: "hyphen-indic", + marathi: "hyphen-indic", + oriya: "hyphen-indic", + pali: "hyphen-indic", + panjabi: "hyphen-indic", + tamil: "hyphen-indic", + telugu: "hyphen-indic", + // pinned versions of the German patterns + "german-x-latest": "dehyph-exptl", + "ngerman-x-latest": "dehyph-exptl", + "german-x-2024-02-28": "dehyph-exptl", + "ngerman-x-2024-02-28": "dehyph-exptl", }; -// Maps a babel language name to the TeX Live package providing its -// hyphenation patterns (undefined when there is no such package) -function hyphenationPackageForLanguage(lang: string) { - // Names in language.dat may be in mixed case (e.g. UKenglish) - lang = lang.toLowerCase(); - - // It seems some languages have no hyphenation files, so we just filter them out - // e.g. `lang: zh` has no hyphenation files - // https://github.com/quarto-dev/quarto-cli/issues/10291 - const noHyphen = ["chinese-hans", "chinese"]; - if (noHyphen.includes(lang)) { - return; - } +// Maps language names found in a log to the TeX Live packages providing their +// hyphenation patterns. Follows hyphen_pkgs() in tinytex (R/latex.R). +function hyphenationPackagesForLanguages(names: string[]): string[] { + const packages: string[] = []; + for (const raw of names) { + // Names in language.dat may be in mixed case (e.g. UKenglish) + const lang = raw.replace(/[`']/g, "").toLowerCase() + // Some warnings give a description rather than a name + // (e.g. `German (current orthography)`): the first word is the language + .replace(/\s.*$/s, ""); + + // The name comes from the log and is passed to tlmgr (through a .bat file + // on Windows), so only keep what looks like a package name (all + // hyphenation names in texlive.tlpdb are lowercase letters, digits and hyphens) + if (!/^[a-z0-9-]+$/.test(lang)) { + continue; + } - // NOTE Although the names of the corresponding lfd files match those in this list, - // there are some exceptions, particularly in German and Serbian. So, ngerman is - // called here german, which is the name in the CLDR and, actually, the most logical. - // - // See https://ctan.math.utah.edu/ctan/tex-archive/macros/latex/required/babel/base/babel.pdf - const name = Object.hasOwn(kHyphenationPackageNames, lang) - ? kHyphenationPackageNames[lang] - : lang; - const pkg = `hyphen-${name}`; - // The name comes from the log and is passed to tlmgr (through a .bat file - // on Windows), so only keep what looks like a package name (all - // hyphenation names in texlive.tlpdb are lowercase letters, digits and hyphens) - if (/^hyphen-[a-z0-9-]+$/.test(pkg)) { - return pkg; + // It seems some languages have no hyphenation files, so we just filter them out + // e.g. `lang: zh` has no hyphenation files + // https://github.com/quarto-dev/quarto-cli/issues/10291 + if (lang === "chinese" || lang === "chinese-hans") { + continue; + } + + // NOTE Although the names of the corresponding lfd files match those in this list, + // there are some exceptions, particularly in German and Serbian. So, ngerman is + // called here german, which is the name in the CLDR and, actually, the most logical. + // + // See https://ctan.math.utah.edu/ctan/tex-archive/macros/latex/required/babel/base/babel.pdf + packages.push( + Object.hasOwn(kHyphenationPackageNames, lang) + ? kHyphenationPackageNames[lang] + : `hyphen-${lang}`, + ); } + return ld.uniq(packages); +} + +// Port of latex_warning() in tinytex (R/latex.R): the lines of every +// `LaTeX Warning:` / `Package Warning:` block, up to the next blank line +function latexWarningBlocks(logText: string): string { + const logLines = logText.split(/\r?\n/); + const kept = new Set(); + const startRegex = /^(LaTeX|Package [A-Za-z0-9]+) Warning:/; + logLines.forEach((line, i) => { + if (!startRegex.test(line)) { + return; + } + for (let j = i; j < logLines.length && !/^\s*$/.test(logLines[j]); j++) { + kept.add(j); + } + }); + return [...kept].sort((a, b) => a - b).map((i) => logLines[i]).join("\n"); } // babel raises a fatal error (rather than the warning handled by @@ -246,38 +278,34 @@ const kBabelUndefinedLanguageRegex = // (to be installed directly by name, as opposed to the file search terms // returned by findMissingFontsAndPackages) export function findMissingHyphenationPackages(logText: string): string[] { - const packages: string[] = []; - for (const match of logText.matchAll(kBabelUndefinedLanguageRegex)) { - const pkg = hyphenationPackageForLanguage(match[1]); - if (pkg) { - packages.push(pkg); - } - } - return ld.uniq(packages); + return hyphenationPackagesForLanguages( + [...logText.matchAll(kBabelUndefinedLanguageRegex)].map((m) => m[1]), + ); } // Finds missing hyphenation files (these appear as warnings in the log file) -export function findMissingHyphenationFiles(logText: string) { - const babelWarningRegex = /^Package babel Warning:/m; - const hasWarning = logText.match(babelWarningRegex); - if (hasWarning) { - const languageRegex = /^\(babel\).* language [`'](\S+)[`'].*$/m; - const languageMatch = logText.match(languageRegex); - if (languageMatch) { - return hyphenationPackageForLanguage(languageMatch[1]); - } +export function findMissingHyphenationFiles(logText: string): string[] { + // Same regexes as check_babel() and check_polyglossia() in tinytex, on + // warning blocks only + const babelRegex = + /^(?:\(babel\).* |Package babel Warning: No hyphenation patterns were preloaded for the )language [`']([^']+)'.*$/gm; + const polyglossiaRegex = + /^Package polyglossia Warning: No hyphenation patterns were loaded for ([`'][^']+'|British English).*$/gm; + const warnings = latexWarningBlocks(logText); + const packages = hyphenationPackagesForLanguages([ + ...warnings.matchAll(babelRegex), + ...warnings.matchAll(polyglossiaRegex), + ].map((m) => m[1])); + if (packages.length > 0) { + return packages; } // Try an alternative way of parsing const hyphenRulesRegex = - /Package babel Info: Hyphen rules for '(.*?)' set to \\l@nil/m; - const match = logText.match(hyphenRulesRegex); - if (match) { - const language = match[1]; - if (language) { - return hyphenationPackageForLanguage(language); - } - } + /Package babel Info: Hyphen rules for '(.*?)' set to \\l@nil/gm; + return hyphenationPackagesForLanguages( + [...logText.matchAll(hyphenRulesRegex)].map((m) => m[1]), + ); } // Parse a log file to find latex errors diff --git a/src/command/render/latexmk/pdf.ts b/src/command/render/latexmk/pdf.ts index 756906d21e3..80bf8c5469c 100644 --- a/src/command/render/latexmk/pdf.ts +++ b/src/command/render/latexmk/pdf.ts @@ -182,12 +182,12 @@ async function initialCompileLatex( // See whether there are warnings about hyphenation // See (https://github.com/rstudio/tinytex/commit/0f2007426f730a6ed9d45369233c1349a69ddd29) const logText = Deno.readTextFileSync(response.log); - const missingHyphenationFile = findMissingHyphenationFiles(logText); - if (missingHyphenationFile) { - // try to install it, unless auto install is opted out + const missingHyphenationPackages = findMissingHyphenationFiles(logText); + if (missingHyphenationPackages.length > 0) { + // try to install them, unless auto install is opted out if (pkgMgr.autoInstall) { logProgress("Installing missing hyphenation file..."); - if (await pkgMgr.installPackages([missingHyphenationFile])) { + if (await pkgMgr.installPackages(missingHyphenationPackages)) { // We installed hyphenation files, retry continue; } else { @@ -197,7 +197,7 @@ async function initialCompileLatex( // Let's just through a warning, but it may not be fatal for the compilation // and we can end normally warning( - `Possibly missing hyphenation file: '${missingHyphenationFile}'. See more in logfile (by setting 'latex-clean: false').\n`, + `Possibly missing hyphenation file: '${missingHyphenationPackages.join("', '")}'. See more in logfile (by setting 'latex-clean: false').\n`, ); } diff --git a/tests/unit/latexmk/parse-error.test.ts b/tests/unit/latexmk/parse-error.test.ts index 0c991a0b56c..cfaac7c0b97 100644 --- a/tests/unit/latexmk/parse-error.test.ts +++ b/tests/unit/latexmk/parse-error.test.ts @@ -115,8 +115,9 @@ unitTest("Detect missing hyphenation with babel warnings", async () => { (babel) Please, configure your TeX system to add them and (babel) rebuild the format. Now I will use the patterns (babel) preloaded for \\language=0 instead on input line 51.`; - assert( - findMissingHyphenationFiles(logWithBacktick) === "hyphen-spanish", + assertEquals( + findMissingHyphenationFiles(logWithBacktick), + ["hyphen-spanish"], "Should detect hyphen-spanish from backtick-quote format" ); @@ -126,40 +127,45 @@ unitTest("Detect missing hyphenation with babel warnings", async () => { (babel) Please, configure your TeX system to add them and (babel) rebuild the format. Now I will use the patterns (babel) preloaded for \\language=0 instead on input line 51.`; - assert( - findMissingHyphenationFiles(logWithStraightQuotes) === "hyphen-spanish", + assertEquals( + findMissingHyphenationFiles(logWithStraightQuotes), + ["hyphen-spanish"], "Should detect hyphen-spanish from straight-quote format" ); // Test ngerman special case (should return hyphen-german, not hyphen-ngerman) const logGerman = `Package babel Warning: No hyphenation patterns were preloaded for (babel) the language 'ngerman' into the format.`; - assert( - findMissingHyphenationFiles(logGerman) === "hyphen-german", + assertEquals( + findMissingHyphenationFiles(logGerman), + ["hyphen-german"], "Should map ngerman to hyphen-german" ); // Test Chinese - no hyphen package exists const logChinese = `Package babel Warning: No hyphenation patterns were preloaded for (babel) the language 'chinese' into the format.`; - assert( - findMissingHyphenationFiles(logChinese) === undefined, - "Should return undefined for Chinese (no hyphen package)" + assertEquals( + findMissingHyphenationFiles(logChinese), + [], + "Should return nothing for Chinese (no hyphen package)" ); // Test alternative Info pattern (issue #10291) const logInfoChinese = `Package babel Info: Hyphen rules for 'chinese-hans' set to \\l@nil (babel) (\\language10). Reported on input line 143.`; - assert( - findMissingHyphenationFiles(logInfoChinese) === undefined, - "Should return undefined for chinese-hans via Info pattern" + assertEquals( + findMissingHyphenationFiles(logInfoChinese), + [], + "Should return nothing for chinese-hans via Info pattern" ); // Test no warning present const logNoWarning = "Some other log text without babel warnings"; - assert( - findMissingHyphenationFiles(logNoWarning) === undefined, - "Should return undefined when no babel warning present" + assertEquals( + findMissingHyphenationFiles(logNoWarning), + [], + "Should return nothing when no babel warning present" ); }, { cwd: () => "unit/latexmk/" @@ -191,6 +197,9 @@ Type H for immediate help.`; ); // The file-based search finds nothing for this error assertEquals(findMissingFontsAndPackages(logSwissGerman, "."), []); + // The warning names a description, not a pattern name: the first word is + // used, not the nswissgerman from the \l@nil line + assertEquals(findMissingHyphenationFiles(logSwissGerman), ["hyphen-german"]); // Backtick-quote format (older babel) assertEquals( @@ -245,7 +254,7 @@ unitTest("Drop hyphenation package names that are not valid package names", asyn assertEquals( findMissingHyphenationFiles(`Package babel Warning: No hyphenation patterns were preloaded for (babel) the language '${lang}' into the format.`), - undefined, + [], `Should drop '${lang}' from the babel warning`, ); } @@ -302,6 +311,11 @@ unitTest("Map hyphenation language names to their TeX Live package", async () => panjabi: "hyphen-indic", tamil: "hyphen-indic", telugu: "hyphen-indic", + // pinned German patterns are not in a hyphen-* package + "german-x-latest": "dehyph-exptl", + "ngerman-x-latest": "dehyph-exptl", + "german-x-2024-02-28": "dehyph-exptl", + "ngerman-x-2024-02-28": "dehyph-exptl", }; for (const [lang, pkg] of Object.entries(expected)) { assertEquals( @@ -314,7 +328,7 @@ unitTest("Map hyphenation language names to their TeX Live package", async () => assertEquals( findMissingHyphenationFiles(`Package babel Warning: No hyphenation patterns were preloaded for (babel) the language '${lang}' into the format.`), - pkg, + [pkg], `Should map '${lang}' to ${pkg} from the babel warning`, ); } @@ -336,3 +350,86 @@ unitTest("Map hyphenation language names to their TeX Live package", async () => }, { cwd: () => "unit/latexmk/" }) + +unitTest("Detect missing pinned German hyphenation patterns", async () => { + // `\begin{hyphenrules}{ngerman-x-latest}` (or `\selectlanguage`) without + // dehyph-exptl installed + const log = `! Package babel Error: You haven't defined the language 'ngerman-x-latest' yet. +(babel) Perhaps you misspelled it or your installation +(babel) is not complete. + +See the babel package documentation for explanation. +Type H for immediate help. + ... + +l.185 \\begin{hyphenrules}{ngerman-x-latest}`; + assertEquals(findMissingHyphenationPackages(log), ["dehyph-exptl"]); +}, { + cwd: () => "unit/latexmk/" +}) + +unitTest("Ignore babel info messages naming a language", async () => { + // Only warning blocks are searched: the (babel) continuation of an Info + // message is not a missing-patterns warning + const info = `Package babel Info: No hyphenation patterns were set for +(babel) language 'nohyphenation'. Reported on input line 12.`; + assertEquals(findMissingHyphenationFiles(info), []); + + // same with an unrelated warning elsewhere in the log + const withWarning = `Package babel Warning: Something else +(babel) on input line 3. + +${info}`; + assertEquals(findMissingHyphenationFiles(withWarning), []); +}, { + cwd: () => "unit/latexmk/" +}) + +unitTest("Detect every language missing hyphenation patterns", async () => { + // babel warns once per language + const babel = `Package babel Warning: No hyphenation patterns were preloaded for +(babel) the language 'Spanish' into the format. + +Package babel Warning: No hyphenation patterns were preloaded for +(babel) the language 'Italian' into the format.`; + assertEquals(findMissingHyphenationFiles(babel), ["hyphen-spanish", "hyphen-italian"]); + + // polyglossia, real xelatex log with \setotherlanguages{spanish,italian} + const polyglossia = `Package polyglossia Warning: No hyphenation patterns were loaded for \`spanish' +(polyglossia) I will use \\language=\\l@nohyphenation instead. + +Package polyglossia Warning: No hyphenation patterns were loaded for \`italian' +(polyglossia) I will use \\language=\\l@nohyphenation instead.`; + assertEquals(findMissingHyphenationFiles(polyglossia), ["hyphen-spanish", "hyphen-italian"]); + + // polyglossia also reports some languages by their description + assertEquals( + findMissingHyphenationFiles("Package polyglossia Warning: No hyphenation patterns were loaded for British English"), + ["hyphen-english"], + ); + // the same package is installed once + assertEquals(findMissingHyphenationFiles(`${babel}\n\n${polyglossia}`), ["hyphen-spanish", "hyphen-italian"]); + // two languages, one package: deduplicated after mapping + const indic = `Package babel Warning: No hyphenation patterns were preloaded for +(babel) the language 'hindi' into the format. + +Package babel Warning: No hyphenation patterns were preloaded for +(babel) the language 'tamil' into the format.`; + assertEquals(findMissingHyphenationFiles(indic), ["hyphen-indic"]); + assertEquals(findMissingHyphenationFiles("asdf qwer"), []); +}, { + cwd: () => "unit/latexmk/" +}) + +unitTest("Detect missing hyphenation from a language description with spaces", async () => { + // babel-german describes the patterns it needs instead of naming them, + // e.g. `\@nopatterns{German (current orthography)}` in babel-german.def + const log = `Package babel Warning: No hyphenation patterns were preloaded for +(babel) the language 'German (current orthography)' into the format. +(babel) Please, configure your TeX system to add them and +(babel) rebuild the format. Now I will use the patterns +(babel) preloaded for \\language=0 instead on input line 1260.`; + assertEquals(findMissingHyphenationFiles(log), ["hyphen-german"]); +}, { + cwd: () => "unit/latexmk/" +})