diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec6681..85f7cc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Converting a document is now `in2lambda convert FILE FILTER`, with the same options as before (`-o/--out`, `-a/--answers`). `in2lambda FILE FILTER` still converts the file, printing one line to stderr naming the `in2lambda convert` command to run instead, so scripts and Docker invocations written before this keep working. A first argument that is neither a command nor a file is refused with that same line, rather than printing the usage and exiting successfully. - beartype is now `^0.22`. At 0.20.0 and below its import hook leaves `cli` a plain function rather than a group, so the command line either fails to import or runs `convert` whatever the arguments; 0.20.1 is the first version that works. - `in2lambda source add FILE` freezes a document: it converts .docx and .tex to markdown beside the file, and writes a `FILE.draft.json` beside it, holding the markdown's hash and every block in it with the lines it spans, so that another tool can quote the source by line range. The draft is named after the source, so a folder holding a term's worth of sheets holds a draft for each. `in2lambda source show` prints that markdown numbered with the block ids. Freezing a file that has changed since is refused unless `--start-over` says to discard the draft, and so is showing one, since its block ids would name lines they are not the ids of. Both need pandoc and the `convert` extra, as `convert` does. +- `in2lambda source add` freezes a converted .docx or .tex unwrapped, so a paragraph is one line however long it is and an inline `$ ... $` is never broken over two, and moves every `$$ ... $$` pandoc wrote on one line onto lines of its own, in a paragraph or anywhere in a list item, indented to the item's width where it stands in a list. A `$$` that opens or closes in a table cell, in a block quote or in a code block is left as written, and so is maths an unpaired `$$` elsewhere in the document - one in inline code, say - pairs with; `in2lambda validate` still reports the maths left as written. Both were habits of pandoc's writer rather than anything the author did, and neither is maths Lambda Feedback renders, so `in2lambda validate` reported them against every converted sheet. The markdown beside a document frozen before this differs, and so do its line ranges: `in2lambda source add --start-over` freezes it again. - A draft now holds a `log` of every command that changed it and a `fields` map of what those commands wrote, each field recording which layer wrote it (1 a spec, 2 a predicate, 3 a line range, 4 a literal), the source ranges it was copied from, whether it has been edited and by whom. `in2lambda draft mark ignore BLOCK` is the first such command, and `in2lambda draft replay` rebuilds the draft from the frozen markdown and the log, refusing unless what it builds is the draft that is there, byte for byte. A draft written before this has no `log` in it and is refused as one nothing here wrote; `in2lambda source add --start-over` freezes the document again. - A draft is filled in by `in2lambda draft question add`, `in2lambda draft part add QUESTION` and `in2lambda draft question solution QUESTION`. Each takes `--text` to copy the wording out of the frozen source, as a block id such as `b3` or as lines such as `s10:14`, or `--literal TEXT` where the source does not say it in a form the field can take, which records the field as edited and written by layer 4 rather than 3. Question and part numbers are worked out from the fields already written rather than given, so a replay arrives at the same ids. `in2lambda draft split block BLOCK AT` cuts a block the parser made one of two things into `b3a` and `b3b`, so that each half can be quoted on its own. A command writing a field that is already written, or quoting lines another field was taken from, is refused: the first naming the field, the second naming both. - `in2lambda draft field replace FIELD OLD NEW` changes the wording inside a field that is already written, for the faults only an edit can fix - a brace the OCR dropped out of some maths, which no range of the source says correctly. OLD has to occur in the field exactly once, or the command is refused saying how many times it occurs; `--regex` reads it as a regular expression and NEW as what to replace it with. The field is left quoting the lines it was taken from, at the layer that wrote it, but recorded as edited and by whoever replaced the wording, so the change can be shown against the source. diff --git a/in2lambda/source/__init__.py b/in2lambda/source/__init__.py index 6d17258..faa494e 100644 --- a/in2lambda/source/__init__.py +++ b/in2lambda/source/__init__.py @@ -249,13 +249,173 @@ def file_type(file: str) -> str: raise RuntimeError(f"Unsupported file extension: .{extension}") -def _pandoc(file: str, to: str) -> bytes: +def _pandoc(file: str, to: str, *options: str) -> bytes: """The given file, as pandoc writes it in the `to` format. Undecoded, because what is written to disk and what is hashed have to be the same bytes; whoever wants the text of it decodes it themselves. """ - return subprocess.check_output(["pandoc", file, "-f", file_type(file), "-t", to]) + return subprocess.check_output( + ["pandoc", file, "-f", file_type(file), "-t", to, *options] + ) + + +_DISPLAY_MATHS = re.compile(r"(? set[int]: + r"""The lines of some markdown whose ``$$`` is code rather than maths. + + ``commonmark_x`` fences a code block that carries a language and indents one that + carries nothing four spaces, and a ``$$ ... $$`` in either is text the document + shows rather than maths it renders. A list item's continuation paragraph is indented + four as well, so the indent is measured from the item the line stands in rather than + from the margin: a line four past the enclosing item's content column is code, and + display maths standing as an item's own paragraph is maths. The column is the one + :func:`dedented` takes off again, so a line this leaves alone is a line the field + quoting it reads as code too. + + Examples: + >>> from in2lambda.source import _verbatim_lines + >>> sorted(_verbatim_lines("Text\n\n $$x = y$$\n")) + [3] + >>> sorted(_verbatim_lines("1. Item\n\n $$x = y$$\n")) + [] + >>> sorted(_verbatim_lines("1. Item\n\n $$x = y$$\n")) + [3] + >>> sorted(_verbatim_lines("``` python\n$$x = y$$\n```\n")) + [1, 2, 3] + """ + verbatim = set() + fence = "" + items: list[int] = [] # The content column of each list item open at this line. + for number, line in enumerate(markdown.split("\n"), start=1): + stripped = line.lstrip(" ") + indent = len(line) - len(stripped) + if fence: + verbatim.add(number) + if stripped.startswith(fence): + fence = "" + elif not stripped: + # Commonmark closes an item at the next non-blank line indented less than + # its content column, not at the blank line before that one. + continue + else: + while items and indent < items[-1]: + items.pop() + base = items[-1] if items else 0 + if stripped[:3] in ("```", "~~~"): + fence = stripped[:3] + verbatim.add(number) + elif indent >= base + 4: + verbatim.add(number) + elif marker := _MARKER.match(line): + items.append(marker.end()) + return verbatim + + +def _display_maths_blocked(markdown: str) -> str: + r"""Markdown pandoc wrote, with its display maths moved onto lines of its own. + + ``commonmark_x`` writes ``$$F = p A$$`` on one line wherever in a paragraph the + maths stood, which is the one form the delimiter checks refuse and no real Lambda + Feedback export uses. Rewriting it at the freeze rather than at the field is what + makes every range quoted out of the markdown render, however the maths was written. + + The inserted lines take the indent of the line the maths began on - a list item's + marker width included, so maths in an item stays in the item - and whatever stood + either side of it on that line becomes a paragraph of its own. + + A ``$$`` that opens or closes on a pipe table's row, on a block quote's line or on a + code block's line is left as pandoc wrote it: a table cell cannot hold a block, an + inserted line carries the indent of the opening line but not a quote's ``> ``, and a + code block's ``$$`` is characters the document shows. A match holding a backtick, or + running across a blank line, is left as written as well: display maths holds + neither, so such a match is an unpaired ``$$`` - one in inline code, say - closed by + the opening ``$$`` of a later maths, and that later maths is then left as written + too. ``in2lambda validate`` reports the maths left in any of these. + + Examples: + >>> from in2lambda.source import _display_maths_blocked + >>> _display_maths_blocked("The load is $$F = pA$$ here.\n") + 'The load is\n\n$$\nF = pA\n$$\n\nhere.\n' + >>> _display_maths_blocked("1. Find $$F = pA$$\n") + '1. Find\n\n $$\n F = pA\n $$\n' + >>> _display_maths_blocked("A load $$F = pA$$\r\n") + 'A load\r\n\r\n$$\r\nF = pA\r\n$$\r\n' + >>> _display_maths_blocked("> The load is $$F = pA$$ here.\n") + '> The load is $$F = pA$$ here.\n' + >>> _display_maths_blocked("Type this:\n\n $$x = y$$\n") + 'Type this:\n\n $$x = y$$\n' + >>> _display_maths_blocked("``` python\nprint(\"$$x = y$$\")\n```\n") + '``` python\nprint("$$x = y$$")\n```\n' + >>> _display_maths_blocked("Type `$$` first.\n\nThe load is $$F = pA$$\n") + 'Type `$$` first.\n\nThe load is $$F = pA$$\n' + >>> _display_maths_blocked("Type `$$` then $$F = pA$$ ends.\n") + 'Type `$$` then $$F = pA$$ ends.\n' + >>> _display_maths_blocked("The load is $$F = pA\n> and $$ here.\n") + 'The load is $$F = pA\n> and $$ here.\n' + """ + if "\r\n" in markdown: + # Pandoc writes the line endings of whoever is running it, and the file on disk + # is hashed as it is written, so a Windows freeze stays a Windows file. + blocked = _display_maths_blocked(markdown.replace("\r\n", "\n")) + return blocked.replace("\n", "\r\n") + + verbatim = _verbatim_lines(markdown) + + def blocked(position: int) -> bool: + """Whether the `$$` at this offset stands in a table row, a quote or code.""" + opening = markdown[markdown.rfind("\n", 0, position) + 1 : position] + return opening.lstrip()[:1] in ("|", ">") or ( + markdown.count("\n", 0, position) + 1 in verbatim + ) + + written: list[str] = [] + end = 0 + for match in _DISPLAY_MATHS.finditer(markdown): + if "`" in match.group(1) or any( + not line.strip() for line in match.group().split("\n") + ): + # Display maths holds neither a backtick nor a blank line, so a match over + # one of the two is an unpaired `$$` - one in inline code, say - closed by + # the opening `$$` of a later maths. Rewriting it would make a maths block + # of the words standing between the two. + continue + if blocked(match.start()) or blocked(match.end()): + # A pipe table's cell cannot hold a block; an inserted line carries the + # indent of the line the maths began on but not a block quote's `> `, so the + # rewrite would put the maths and the words after it outside the quote; and + # a code block's `$$` is characters the document shows, not maths. Either + # delimiter standing in one of the three is enough to leave the match alone. + continue + before = markdown[markdown.rfind("\n", 0, match.start()) + 1 : match.start()] + marker = _MARKER.match(before) + indent = " " * ( + marker.end() if marker else len(before) - len(before.lstrip(" ")) + ) + # Maths that already starts its line - or the line's list item - needs no break + # before it, and the indent it would be given is in the line already. + opens_the_line = not before.strip() or ( + marker is not None and marker.end() == len(before) + ) + head = markdown[end : match.start()] + written.append(head if opens_the_line else f"{head.rstrip(' ')}\n\n{indent}") + body = "\n".join( + f"{indent}{line.strip()}" for line in match.group(1).strip().split("\n") + ) + written.append(f"$$\n{body}\n{indent}$$") + end = match.end() + rest = markdown[end:].split("\n", 1)[0] + if rest.strip(): + written.append(f"\n\n{indent}") + end += len(rest) - len(rest.lstrip(" ")) + written.append(markdown[end:]) + return "".join(written) def _digest(data: bytes) -> str: @@ -464,10 +624,6 @@ def blocks(markdown: str, source: int = 1) -> list[Block]: return [block for block, _ in _elements(markdown, source)] -_MARKER = re.compile(r" *(?:[-+*]|\(?(?:\d+|[ivxlcdm]+|[IVXLCDM]+|[A-Za-z])[.)]) {1,4}") -"""A list item's marker on its first line, as `commonmark_x` reads one.""" - - def dedented(text: str) -> str: r"""Some lines of a list item, with the item's own indentation off every one. @@ -609,7 +765,9 @@ def add( A .docx or .tex file is converted to markdown next to it; a markdown file is taken as it is and nothing is copied. Either way the markdown is hashed and its blocks written to ``FILE.draft.json``, so that whatever quotes a source by line range can - tell that the lines it was given still say what they said. + tell that the lines it was given still say what they said. A converted file is + written unwrapped - a paragraph is one line, however long - with each ``$$ ... $$`` + on lines of its own, which is the maths Lambda Feedback renders. The files are numbered in the order they are given, and a file already frozen into the draft beside them is checked against the hash it was frozen at rather than @@ -669,8 +827,13 @@ def add( raw, markdown = _source(path) frozen_path = path else: - raw = _pandoc(str(path), _MARKDOWN) - markdown = raw.decode("utf-8") + # Unwrapped, and with the display maths blocked out, before anything is + # hashed: both are habits of pandoc's writer rather than anything the author + # did, and both are what a field quoting these lines would have to render. + markdown = _display_maths_blocked( + _pandoc(str(path), _MARKDOWN, "--wrap=none").decode("utf-8") + ) + raw = markdown.encode("utf-8") frozen_path = path.with_suffix(".md") converted.append((frozen_path, raw)) digest = _digest(raw) diff --git a/tests/fixtures/sources/README.md b/tests/fixtures/sources/README.md index eb98fc2..7e7397f 100644 --- a/tests/fixtures/sources/README.md +++ b/tests/fixtures/sources/README.md @@ -7,7 +7,11 @@ formats the command takes, so that what a heading or a list item comes out as do which format an author brought it in; `empty_list_item` is a bullet with nothing in it, which has no position of its own and so no block; `unseparated_list` is a list with no blank line before it, which pandoc reports as part of the paragraph above, so that paragraph's block has to stop where -the list starts rather than where pandoc says it ends; `crlf` is the `markdown` case saved with +the list starts rather than where pandoc says it ends; `display_maths` is a `.tex` with display +maths standing alone, mid-sentence, on a list item's first line and as an item's own second +paragraph, each of which the `$$` rewrite below puts on three lines, and a paragraph well over +72 columns, whose block is the one line the unwrapped freeze leaves it as rather than the two +pandoc's own wrapping made of it; `crlf` is the `markdown` case saved with Windows line endings, which is what a document off a teacher's machine usually has, and it has to freeze to the same blocks and to a hash that `sha256sum source.md` reproduces. The `.gitattributes` at the top of the repository is what stops a checkout rewriting those endings away. @@ -19,6 +23,26 @@ The line ranges of the `.tex` and `.docx` cases are ranges in the markdown pando in the document itself, so they move if pandoc's `commonmark_x` writer changes. They were produced with **pandoc 3.9.0.2**. +That markdown is written with `--wrap=none`, so a paragraph is one line however long it is and +an inline `$ ... $` is never broken over two, and each `$$ ... $$` the writer put on one line is +moved onto lines of its own afterwards - indented to the item's content column where it is in a +list, so the item still holds it. Both are habits of pandoc's writer rather than anything the +author did, and neither is maths that Lambda Feedback renders, so a field quoted out of a freeze +that kept them would fail `in2lambda validate`. + +A `$$` that opens or closes on a pipe table's row, on a block quote's line or on a code block's +line is left as pandoc wrote it: a table cell cannot hold a block, the inserted lines would carry +no `> ` and so fall outside the quote, and a code block's `$$` is characters the document shows +rather than maths it renders. A code block is a line indented four past the content column of the +list item it stands in, which is how an item's own paragraph - indented four itself - is told from +code nested inside the item. + +A `$$ ... $$` holding a backtick, or running across a blank line, is left as pandoc wrote it as +well: display maths holds neither, so the two delimiters are an unpaired `$$` - one in inline +code, say - and the opening `$$` of a later maths, and rewriting them would make a maths block of +the words between. The later maths is then left as written too. `in2lambda validate` reports the +maths left as written in any of these places. + `docx/source.docx` was made from `markdown/source.md` with `pandoc source.md -o source.docx`, run beside a `figure.png` so that the image is embedded rather than dropped, and with the image's alt text removed: pandoc turns a captioned image into a figure, which `commonmark_x` diff --git a/tests/fixtures/sources/display_maths/expected.json b/tests/fixtures/sources/display_maths/expected.json new file mode 100644 index 0000000..bfdb1c2 --- /dev/null +++ b/tests/fixtures/sources/display_maths/expected.json @@ -0,0 +1,50 @@ +[ + { + "end": 1, + "id": "b1", + "start": 1, + "type": "heading" + }, + { + "end": 3, + "id": "b2", + "start": 3, + "type": "paragraph" + }, + { + "end": 7, + "id": "b3", + "start": 5, + "type": "display maths" + }, + { + "end": 9, + "id": "b4", + "start": 9, + "type": "paragraph" + }, + { + "end": 13, + "id": "b5", + "start": 11, + "type": "display maths" + }, + { + "end": 15, + "id": "b6", + "start": 15, + "type": "paragraph" + }, + { + "end": 23, + "id": "b7", + "start": 17, + "type": "list item" + }, + { + "end": 31, + "id": "b8", + "start": 25, + "type": "list item" + } +] diff --git a/tests/fixtures/sources/display_maths/source.tex b/tests/fixtures/sources/display_maths/source.tex new file mode 100644 index 0000000..1b52dd3 --- /dev/null +++ b/tests/fixtures/sources/display_maths/source.tex @@ -0,0 +1,21 @@ +\section{Display maths} + +The pressure is the same throughout the oil, so the load the large piston carries is $F = p A$ once the small piston has been pushed down. + +\[ +F = p A +\] + +The load is \[ F = p A \] once the pressure is known. + +\begin{enumerate} + \item Find the load, given that \[ F = p A \] and both areas are known. + + \item Find the load. + + \[ + F = p A + \] + + Both areas are known. +\end{enumerate} diff --git a/tests/fixtures/sources/docx/expected.json b/tests/fixtures/sources/docx/expected.json index 3b34c33..f190bf3 100644 --- a/tests/fixtures/sources/docx/expected.json +++ b/tests/fixtures/sources/docx/expected.json @@ -6,33 +6,33 @@ "type": "heading" }, { - "end": 4, + "end": 3, "id": "b2", "start": 3, "type": "paragraph" }, { - "end": 6, + "end": 5, "id": "b3", - "start": 6, + "start": 5, "type": "list item" }, { - "end": 7, + "end": 6, "id": "b4", - "start": 7, + "start": 6, "type": "list item" }, { - "end": 9, + "end": 10, "id": "b5", - "start": 9, + "start": 8, "type": "display maths" }, { "end": 12, "id": "b6", - "start": 11, + "start": 12, "type": "image" }, { diff --git a/tests/fixtures/sources/tex/expected.json b/tests/fixtures/sources/tex/expected.json index f4a5339..c4ac400 100644 --- a/tests/fixtures/sources/tex/expected.json +++ b/tests/fixtures/sources/tex/expected.json @@ -6,33 +6,33 @@ "type": "heading" }, { - "end": 4, + "end": 3, "id": "b2", "start": 3, "type": "paragraph" }, { - "end": 6, + "end": 5, "id": "b3", - "start": 6, + "start": 5, "type": "list item" }, { - "end": 8, + "end": 7, "id": "b4", - "start": 8, + "start": 7, "type": "list item" }, { - "end": 10, + "end": 11, "id": "b5", - "start": 10, + "start": 9, "type": "display maths" }, { - "end": 12, + "end": 13, "id": "b6", - "start": 12, + "start": 13, "type": "image" } ] diff --git a/tests/test_source.py b/tests/test_source.py index 08202d9..20c9197 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -16,7 +16,9 @@ from click.testing import CliRunner from conftest import SOURCES, SOURCES_DIR +from in2lambda.draft import _quoted from in2lambda.main import cli +from in2lambda.validation import MathDelimiterError, math_delimiter_checker MARKDOWN = SOURCES_DIR / "markdown" """The case the tests below happen to use; what they check holds for any of them.""" @@ -36,10 +38,20 @@ def test_source_add_finds_the_expected_blocks(folder: Path, tmp_path: Path) -> N result = CliRunner().invoke(cli, ["source", "add", str(_frozen(tmp_path))]) assert result.exit_code == 0, result.output - (source,) = json.loads((tmp_path / "source.draft.json").read_text())["sources"] + draft = json.loads((tmp_path / "source.draft.json").read_text()) + (source,) = draft["sources"] assert source["blocks"] == json.loads((folder / "expected.json").read_text()) - markdown = (tmp_path / source["source"]).read_bytes() - assert source["hash"] == f"sha256:{hashlib.sha256(markdown).hexdigest()}" + raw = (tmp_path / source["source"]).read_bytes() + assert source["hash"] == f"sha256:{hashlib.sha256(raw).hexdigest()}" + + # Every block is a range some command will quote, and what it quotes is a field + # `draft validate` runs the delimiter checks over. A freeze that kept pandoc's own + # wrapping, or the `$$ ... $$` its writer puts on one line, would hand those checks + # a finding about the writer rather than about the document. + markdown = raw.decode("utf-8") + for block in source["blocks"]: + quoted = _quoted(draft, markdown, 1, block["start"], block["end"]) + assert math_delimiter_checker(quoted) is MathDelimiterError.PASSED, quoted def test_freezing_again_is_refused_once_the_source_has_changed(