From e2cfac796005d27acfc18596bfd8ff71ac2417bd Mon Sep 17 00:00:00 2001 From: Siarhei Date: Fri, 11 Sep 2026 12:12:16 +0200 Subject: [PATCH 1/2] chore(translator): add the inline-mark codec and the container fragment collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for translating rich text one container at a time. Nothing imports these yet; the pipeline is wired to them in the next change, which is what makes them reachable and what earns a release. Typed `chore` deliberately: a release whose notes announce a feature no install can use is noise. Today a paragraph is translated one text node at a time, so word order stays pinned to the source language and inline formatting lands on whichever word happens to sit in that position. These two modules are the pieces needed to send a whole container instead. `collectInlineFragments` walks a Lexical tree and returns, per container, the fragments it is made of — each carrying the node to write text into and the node to place when the array is rebuilt. Those are two different things for a wrapper: a link around one word places the link, a link around two differently-formatted words gets a separate copy per fragment so the two can move independently. It refuses containers it cannot round-trip and says why: a source that already looks like marked-up text, a single leaf with nothing to reorder, no translatable text, or a wrapper shape it does not handle. `inlineMarks` serialises those fragments as `<1>text` and parses a reply back. Parsing is all-or-nothing by design: seven named failures, and any of them returns no fragments at all, because half a rebuilt paragraph written into a document is the failure nobody notices while untranslated text is obvious. The existing per-node walk is left untouched. It feeds both the per-node translation path and the provenance fingerprint, and the two walks disagree on whitespace — this one glues a whitespace-only node into its neighbour where the old one drops it — so widening the old walk would move stored fingerprint values. Verification: 1469 unit tests, 89 new across the two modules; check-types clean; lint 58 warnings and 0 errors, identical to main; declaration build passes. Four mutations, each red on its own cases: allowing a repeated mark, allowing a lost mark, allowing an unclosed mark, and placing a shared wrapper instead of a per-fragment copy. --- .../lexical/collectInlineFragments.test.ts | 650 ++++++++++++++++++ .../kernel/lexical/collectInlineFragments.ts | 239 +++++++ .../core/kernel/lexical/inlineMarks.test.ts | 334 +++++++++ .../src/core/kernel/lexical/inlineMarks.ts | 207 ++++++ 4 files changed, 1430 insertions(+) create mode 100644 packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts create mode 100644 packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts create mode 100644 packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts create mode 100644 packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts new file mode 100644 index 000000000..4c85d9561 --- /dev/null +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts @@ -0,0 +1,650 @@ +/** + * Contract tests for the container walk, written from + * `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (Emitting, Finding the + * container, Whitespace, Plain values) and the JSDoc in `collectInlineFragments.ts` — not from + * any implementation. Every assertion below quotes a sentence of that contract. + */ + +import { describe, it, expect } from "vitest"; +import { collectInlineFragments } from "./collectInlineFragments"; + +const createNode = (type: string, props?: Record, children?: any[]) => + ({ type, ...props, ...(children && { children }) }) as any; + +/** The children of a rebuilt-array entry, for the wrapper-copy assertions. */ +const childrenOf = (node: unknown): unknown[] => + ((node as { children?: unknown[] }).children ?? []) as unknown[]; + +describe("collectInlineFragments", () => { + describe("finding the container", () => { + it("takes the paragraph as the container when it has direct text children", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + createNode("text", { text: " today" }), + ]), + ]); + + expect(collectInlineFragments(root)).toHaveLength(1); + }); + + it("returns the container node itself, the one whose children the caller rebuilds", () => { + const paragraph = createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("text", { text: "now" }), + ]); + const root = createNode("root", {}, [paragraph]); + + expect(collectInlineFragments(root)[0]?.node).toBe(paragraph); + }); + + it("does not visit a container's nested wrapper as a container of its own", () => { + const link = createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("text", { text: "Buy " }), link]), + ]); + + const containers = collectInlineFragments(root); + + expect(containers.some((container) => container.node === link)).toBe(false); + }); + + it("returns containers in document order", () => { + const first = createNode("paragraph", {}, [createNode("text", { text: "First" })]); + const second = createNode("paragraph", {}, [createNode("text", { text: "Second" })]); + const root = createNode("root", {}, [first, second]); + + expect(collectInlineFragments(root).map((container) => container.node)).toEqual([ + first, + second, + ]); + }); + + // §4 diagram: "list ← no direct text, descend / listitem ← container (each item on its own, + // as it must be)". + it("takes each list item as its own container", () => { + const firstItem = createNode("listitem", {}, [createNode("text", { text: "first" })]); + const secondItem = createNode("listitem", {}, [createNode("text", { text: "second" })]); + const root = createNode("root", {}, [createNode("list", {}, [firstItem, secondItem])]); + + expect(collectInlineFragments(root).map((container) => container.node)).toEqual([ + firstItem, + secondItem, + ]); + }); + + // §4 diagram: "quote → paragraph → text ← the paragraph is the container". + it("takes a quote's paragraph as the container, not the quote", () => { + const paragraph = createNode("paragraph", {}, [createNode("text", { text: "quoted" })]); + const root = createNode("root", {}, [createNode("quote", {}, [paragraph])]); + + expect(collectInlineFragments(root).map((container) => container.node)).toEqual([paragraph]); + }); + + // A node with no text child anywhere below never qualifies, so it yields no container. + it("yields no container for a node with no text children at all", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("upload", { relationTo: "media" })]), + ]); + + expect(collectInlineFragments(root)).toEqual([]); + }); + + // §4 diagram: "block (fields, not children) — no text children, walked past". + it("walks past a block that carries fields instead of children", () => { + const first = createNode("paragraph", {}, [createNode("text", { text: "before" })]); + const second = createNode("paragraph", {}, [createNode("text", { text: "after" })]); + const root = createNode("root", {}, [ + first, + createNode("block", { fields: { blockType: "cta", label: "Buy" } }), + second, + ]); + + expect(collectInlineFragments(root).map((container) => container.node)).toEqual([ + first, + second, + ]); + }); + + it("returns no containers for an empty root", () => { + expect(collectInlineFragments(createNode("root", {}, []))).toEqual([]); + }); + + it("makes each link its own container in a paragraph of adjacent links", () => { + const firstLink = createNode("link", { url: "https://a.example" }, [ + createNode("text", { text: "first link" }), + ]); + const secondLink = createNode("link", { url: "https://b.example" }, [ + createNode("text", { text: "second link" }), + ]); + const root = createNode("root", {}, [createNode("paragraph", {}, [firstLink, secondLink])]); + + expect(collectInlineFragments(root).map((container) => container.node)).toEqual([ + firstLink, + secondLink, + ]); + }); + + it("gives each link container in that paragraph a single fragment", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("link", { url: "https://a.example" }, [ + createNode("text", { text: "first link" }), + ]), + createNode("link", { url: "https://b.example" }, [ + createNode("text", { text: "second link" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root).map((container) => container.fragments.length)).toEqual([ + 1, 1, + ]); + }); + + it("sends each link container in that paragraph down the per-node path", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("link", { url: "https://a.example" }, [ + createNode("text", { text: "first link" }), + ]), + createNode("link", { url: "https://b.example" }, [ + createNode("text", { text: "second link" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root).every((container) => container.skip !== undefined)).toBe( + true + ); + }); + }); + + describe("fragments", () => { + it("numbers fragments from 1 in document order within the container", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "a " }), + createNode("text", { text: "red" }), + createNode("text", { text: " car" }), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments.map((piece) => piece.markId)).toEqual([ + 1, 2, 3, + ]); + }); + + it("restarts numbering at 1 in the next container", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "a " }), + createNode("text", { text: "red" }), + ]), + createNode("paragraph", {}, [ + createNode("text", { text: "a " }), + createNode("text", { text: "blue" }), + ]), + ]); + + expect(collectInlineFragments(root)[1]?.fragments.map((piece) => piece.markId)).toEqual([ + 1, 2, + ]); + }); + + it("takes the fragment text from the leaf itself", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + createNode("text", { text: " today" }), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments.map((piece) => piece.text)).toEqual([ + "Buy ", + "our product", + " today", + ]); + }); + + // Emitting table, row 1: "text leaf, direct child of the container | the leaf | the same node". + it("points a direct text leaf's node at that very leaf", () => { + const leaf = createNode("text", { text: "Buy " }); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [leaf, createNode("text", { text: "now" })]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments[0]?.node).toBe(leaf); + }); + + it("uses the same node as top for a direct text leaf", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("text", { text: "now" }), + ]), + ]); + + const fragment = collectInlineFragments(root)[0]?.fragments[0]; + + expect(fragment?.top).toBe(fragment?.node); + }); + + // Emitting table, row 2: "the container's direct child holds exactly one leaf | the leaf | + // that direct child — any chain above the leaf rides along inside it". + it("uses the container's direct child as top when that wrapper holds one leaf", () => { + const link = createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("text", { text: "Buy " }), link]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.top).toBe(link); + }); + + it("points node at the leaf inside a single-leaf wrapper", () => { + const leaf = createNode("text", { text: "our product" }); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("link", { url: "https://example.com" }, [leaf]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.node).toBe(leaf); + }); + + it("keeps a wrapper chain above the leaf inside top rather than rebuilding it", () => { + const mark = createNode("mark", {}, [ + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + ]); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("text", { text: "Buy " }), mark]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.top).toBe(mark); + }); + + // Emitting table, row 3 + §4 Marks are flat: a link containing an emphasised word + const linkWithTwoLeaves = () => { + const emphasised = createNode("text", { text: "docs", format: 1 }); + const plain = createNode("text", { text: "read the " }); + const link = createNode("link", { url: "https://example.com" }, [plain, emphasised]); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("text", { text: "Please " }), link]), + ]); + return { root, link, plain, emphasised }; + }; + + it("emits one fragment per leaf of a multi-leaf wrapper", () => { + const { root } = linkWithTwoLeaves(); + + expect(collectInlineFragments(root)[0]?.fragments).toHaveLength(3); + }); + + it("gives a multi-leaf wrapper's fragment a copy of the wrapper as top, not the wrapper", () => { + const { root, link } = linkWithTwoLeaves(); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.top).not.toBe(link); + }); + + it("gives each leaf of a multi-leaf wrapper its own separate copy", () => { + const { root } = linkWithTwoLeaves(); + + const fragments = collectInlineFragments(root)[0]?.fragments ?? []; + + expect(fragments[1]?.top).not.toBe(fragments[2]?.top); + }); + + it("puts only this fragment's leaf inside the wrapper copy", () => { + const { root } = linkWithTwoLeaves(); + + const fragment = collectInlineFragments(root)[0]?.fragments[1]; + + expect(childrenOf(fragment?.top)).toHaveLength(1); + }); + + it("points node into the wrapper copy, not at the source leaf", () => { + const { root } = linkWithTwoLeaves(); + + const fragment = collectInlineFragments(root)[0]?.fragments[1]; + + expect(childrenOf(fragment?.top)[0]).toBe(fragment?.node); + }); + + // Emitting table, row 4: "non-text inline node (line break, inline block, upload) | ``", + // with no node of its own — a line break mid-paragraph. + const lineBreakMidParagraph = () => { + const lineBreak = createNode("linebreak"); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "first line" }), + lineBreak, + createNode("text", { text: "second line" }), + ]), + ]); + return { root, lineBreak }; + }; + + it("gives a node carrying no text a null text", () => { + const { root } = lineBreakMidParagraph(); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.text).toBeNull(); + }); + + it("gives a node carrying no text a null node", () => { + const { root } = lineBreakMidParagraph(); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.node).toBeNull(); + }); + + it("uses the text-free node itself as top", () => { + const { root, lineBreak } = lineBreakMidParagraph(); + + expect(collectInlineFragments(root)[0]?.fragments[1]?.top).toBe(lineBreak); + }); + + it("still numbers a fragment that carries no text", () => { + const { root } = lineBreakMidParagraph(); + + expect(collectInlineFragments(root)[0]?.fragments.map((piece) => piece.markId)).toEqual([ + 1, 2, 3, + ]); + }); + }); + + describe("whitespace-only nodes", () => { + // §4 Whitespace: "Buy" / " " / "our product" formatted. + const spaceBetweenWords = () => { + const buy = createNode("text", { text: "Buy" }); + const space = createNode("text", { text: " " }); + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + buy, + space, + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + ]), + ]); + return { root, buy, space }; + }; + + it("whitespace: a whitespace-only node becomes no fragment of its own", () => { + const { root } = spaceBetweenWords(); + + expect(collectInlineFragments(root)[0]?.fragments).toHaveLength(2); + }); + + it("whitespace: its text is glued onto the preceding fragment", () => { + const { root } = spaceBetweenWords(); + + expect(collectInlineFragments(root)[0]?.fragments[0]?.text).toBe("Buy "); + }); + + it("whitespace: its node drops out of the fragments", () => { + const { root, space } = spaceBetweenWords(); + + const fragments = collectInlineFragments(root)[0]?.fragments ?? []; + + expect(fragments.some((piece) => piece.node === space || piece.top === space)).toBe(false); + }); + + it("whitespace: the fragment it was glued onto keeps its own leaf as node", () => { + const { root, buy } = spaceBetweenWords(); + + expect(collectInlineFragments(root)[0]?.fragments[0]?.node).toBe(buy); + }); + + it("whitespace: numbering leaves no gap where the dropped node was", () => { + const { root } = spaceBetweenWords(); + + expect(collectInlineFragments(root)[0]?.fragments.map((piece) => piece.markId)).toEqual([ + 1, 2, + ]); + }); + + it("whitespace: glues backwards past a text-free fragment", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Hello" }), + createNode("linebreak"), + createNode("text", { text: " " }), + createNode("text", { text: "World" }), + ]), + ]); + + const fragments = collectInlineFragments(root)[0]?.fragments; + + expect(fragments?.map((fragment) => fragment.text)).toEqual(["Hello ", null, "World"]); + }); + + // Same guarantee at the end of a container: with nothing following, the glue has nowhere to + // go forwards, and dropping it would lose a character with no signal. + it("whitespace: does not lose trailing glue after a text-free fragment", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Hello" }), + createNode("linebreak"), + createNode("text", { text: " " }), + ]), + ]); + + const fragments = collectInlineFragments(root)[0]?.fragments; + + expect(fragments?.map((fragment) => fragment.text)).toEqual(["Hello ", null]); + }); + + it("whitespace: with no preceding fragment it is glued onto the following one", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: " " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + createNode("text", { text: " today" }), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments[0]?.text).toBe(" our product"); + }); + }); + + describe("containers that cannot use marks", () => { + // D3 / "mark-shaped-source": "Its source text contains a mark-shaped sequence, so parsing a + // reply would be ambiguous." The shapes named are `<12>`, ``, `<12/>`. + it("mark-shaped: an opening-shaped sequence in the source skips the container", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "the <12> sequence " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "explained" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBe("mark-shaped-source"); + }); + + it("mark-shaped: a closing-shaped sequence in the source skips the container", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "the sequence " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "explained" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBe("mark-shaped-source"); + }); + + it("mark-shaped: a self-closing-shaped sequence in the source skips the container", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "the <12/> sequence " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "explained" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBe("mark-shaped-source"); + }); + + it("mark-shaped: a skipped container still carries its fragments", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "the <12> sequence " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "explained" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.fragments).toHaveLength(2); + }); + + // D3: "Plain `
` ... unaffected — only digits between angle brackets collide". + it("mark-shaped: a plain tag like
is not mark-shaped", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "wrap it in " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "
" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); + }); + + // D3: "or `5 < 10` are unaffected". + it("mark-shaped: a comparison like 5 < 10 is not mark-shaped", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "5 < 10 is " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "always true" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); + }); + + // D6 / "single-leaf": "A single unformatted leaf: there is nothing to reorder and + // marks would be pure cost." + it("skips a container that is one plain text leaf", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [createNode("text", { text: "A plain paragraph." })]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBe("single-leaf"); + }); + + it("does not skip a container holding more than one fragment", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); + }); + }); + + describe("the source tree", () => { + it("leaves the input tree untouched", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Buy" }), + createNode("text", { text: " " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "our product" }), + ]), + createNode("linebreak"), + createNode("text", { text: " today" }), + ]), + ]); + const before = JSON.parse(JSON.stringify(root)); + + const containers = collectInlineFragments(root); + + // The positive half matters: an implementation that collected nothing would leave the + // tree untouched too, and pass a check that only compared before and after. + expect(containers[0]?.fragments).toHaveLength(4); + expect(root).toEqual(before); + }); + + it("leaves the input tree untouched when a wrapper has to be copied", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "Please " }), + createNode("link", { url: "https://example.com" }, [ + createNode("text", { text: "read the " }), + createNode("text", { text: "docs", format: 1 }), + ]), + ]), + ]); + const before = JSON.parse(JSON.stringify(root)); + + const containers = collectInlineFragments(root); + + // Same reason as above, plus the copy itself: `top` must not be the wrapper from the tree. + expect(containers[0]?.fragments).toHaveLength(3); + expect(containers[0]?.fragments[1]?.top).not.toBe( + (root as { children: { children: unknown[] }[] }).children[0]?.children[1] + ); + expect(root).toEqual(before); + }); + }); + + describe("wrapper shapes the copy cannot carry", () => { + // Copying a wrapper keeps only the path down to one leaf, so a sibling that is neither a leaf + // nor on that path — a line break between two formatted words inside one link — would vanish + // from every copy with no fragment and no warning. Skipping the container is the honest answer. + it("wrapper: skips a container whose multi-leaf wrapper holds a node the copy would drop", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "See " }), + createNode("link", { url: "/docs" }, [ + createNode("text", { text: "read the " }), + createNode("linebreak"), + createNode("text", { text: "docs", format: 1 }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBe("unsupported-wrapper"); + }); + + it("wrapper: a multi-leaf wrapper with only text inside is still handled", () => { + const root = createNode("root", {}, [ + createNode("paragraph", {}, [ + createNode("text", { text: "See " }), + createNode("link", { url: "/docs" }, [ + createNode("text", { text: "read the " }), + createNode("text", { text: "docs", format: 1 }), + ]), + ]), + ]); + + expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); + }); + }); +}); diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts new file mode 100644 index 000000000..65f43df1c --- /dev/null +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts @@ -0,0 +1,239 @@ +/** + * Container-level collection: groups a Lexical tree into the units that get translated as one + * string each, and the fragments inside them. + * + * Reads only `type`, `text` and `children` — the surface `types.ts` declares. Formatting + * (`format`, `style`, a link's `fields`) is never read: it travels inside the nodes themselves, + * which is what lets this layer reorder formatted pieces without understanding formatting. + * + * The existing per-node walk (`collectTextNodes.ts`) is left alone: it feeds the per-node + * translation path and the provenance fingerprint, and widening it would change stored + * fingerprint values. + * + * Design: `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (D1, D3, D6, D15, D17). + */ + +import { hasChildren, isSerializedLexicalTextNode } from "./guards"; +import type { SerializedLexicalNode, SerializedTextNode } from "./types"; + +/** + * One translatable piece of a container. + * + * `node` is where the translation is written; `top` is what goes into the container's rebuilt + * `children`. They are usually two views of the same subtree — `node` the leaf, `top` the + * container's direct child above it. + * + * When the container's direct child holds more than one leaf (a link with an emphasised word + * inside), `top` is instead a **copy** of that child holding only this fragment's leaf, and + * `node` points into that copy. Pushing one shared wrapper once per leaf would duplicate its + * whole text rather than reorder it. + * + * `text` and `node` are `null` together, for a fragment that carries no text at all. + */ +export type InlineFragment = { + markId: number; + text: string | null; + node: SerializedTextNode | null; + top: SerializedLexicalNode; +}; + +/** Why a container cannot use the marked format, when it cannot. */ +export type ContainerSkipReason = + /** Its source text contains a mark-shaped sequence, so parsing a reply would be ambiguous. */ + | "mark-shaped-source" + /** A single text leaf: nothing to reorder, so marks would be pure cost. */ + | "single-leaf" + | "no-translatable-text" + /** + * A wrapper holds several leaves *and* a node that is neither: copying the wrapper once per + * leaf keeps only the path down to that leaf, so the odd node would be dropped from every copy + * — silently, with no fragment of its own. Skipping is honest where a copy is not. + */ + | "unsupported-wrapper"; + +/** + * A container and its fragments. + * + * `node` is the node whose `children` the caller will rebuild. When `skip` is set the caller + * translates this container the per-node way instead, and `fragments` is still populated so + * the decision needs no second walk. + */ +export type InlineContainer = { + node: SerializedLexicalNode; + fragments: InlineFragment[]; + skip?: ContainerSkipReason; +}; + +const MARK_SHAPED = /<\s*\/?\s*\d+\s*\/?\s*>/u; + +const isBlank = (text: string): boolean => text.trim().length === 0; + +const hasDirectTextChild = (node: SerializedLexicalNode): boolean => + hasChildren(node) && node.children.some((child) => isSerializedLexicalTextNode(child)); + +/** A node with no children that is not text: a line break, an inline block, an upload. */ +const hasNonTextLeafInside = (node: SerializedLexicalNode): boolean => { + if (isSerializedLexicalTextNode(node)) return false; + if (!hasChildren(node)) return true; + return node.children.some(hasNonTextLeafInside); +}; + +const leavesOf = (node: SerializedLexicalNode): SerializedTextNode[] => { + if (isSerializedLexicalTextNode(node)) return [node]; + if (!hasChildren(node)) return []; + return node.children.flatMap(leavesOf); +}; + +/** Copies never share a mutable node: each leaf gets its own chain down from the container's child. */ +const copyChainToLeaf = ( + node: SerializedLexicalNode, + leaf: SerializedTextNode +): { copy: SerializedLexicalNode; leaf: SerializedTextNode } | null => { + if ((node as SerializedTextNode) === leaf) { + const copy = { ...leaf }; + return { copy, leaf: copy }; + } + if (!hasChildren(node)) return null; + + for (const child of node.children) { + const found = copyChainToLeaf(child, leaf); + if (found) { + const copy = { ...node, children: [found.copy] }; + return { copy, leaf: found.leaf }; + } + } + return null; +}; + +type Draft = + | { + kind: "fragment"; + text: string | null; + node: SerializedTextNode | null; + top: SerializedLexicalNode; + } + | { kind: "glue"; text: string } + | { kind: "unsupported" }; + +const draftsOf = (container: SerializedLexicalNode): Draft[] => { + if (!hasChildren(container)) return []; + + return container.children.flatMap((child): Draft[] => { + if (isSerializedLexicalTextNode(child)) { + return isBlank(child.text) + ? [{ kind: "glue", text: child.text }] + : [{ kind: "fragment", text: child.text, node: child, top: child }]; + } + + const leaves = leavesOf(child).filter((leaf) => !isBlank(leaf.text)); + + if (leaves.length === 0) { + return [{ kind: "fragment", text: null, node: null, top: child }]; + } + if (leaves.length === 1) { + const leaf = leaves[0]; + return leaf ? [{ kind: "fragment", text: leaf.text, node: leaf, top: child }] : []; + } + + if (hasNonTextLeafInside(child)) return [{ kind: "unsupported" }]; + + return leaves.flatMap((leaf): Draft[] => { + const copied = copyChainToLeaf(child, leaf); + return copied + ? [{ kind: "fragment", text: copied.leaf.text, node: copied.leaf, top: copied.copy }] + : []; + }); + }); +}; + +const glueWhitespace = (drafts: Draft[]): InlineFragment[] => { + const carried: string[] = []; + const fragments: InlineFragment[] = []; + + // Searching backwards has to skip text-free fragments: a line break between the text and the + // whitespace must not send the glue forwards, and at the end of a container it would drop it. + const lastWithText = (): InlineFragment | undefined => { + for (let index = fragments.length - 1; index >= 0; index -= 1) { + const candidate = fragments[index]; + if (candidate && candidate.text !== null) return candidate; + } + return undefined; + }; + + for (const draft of drafts) { + if (draft.kind === "unsupported") continue; + if (draft.kind === "glue") { + const previous = lastWithText(); + if (previous) previous.text += draft.text; + else carried.push(draft.text); + continue; + } + + const text = + draft.text !== null && carried.length > 0 + ? carried.splice(0).join("") + draft.text + : draft.text; + fragments.push({ markId: fragments.length + 1, text, node: draft.node, top: draft.top }); + } + + return fragments; +}; + +const skipReasonFor = (fragments: InlineFragment[]): ContainerSkipReason | undefined => { + const texts = fragments.flatMap((fragment) => (fragment.text === null ? [] : [fragment.text])); + + if (texts.some((text) => MARK_SHAPED.test(text))) return "mark-shaped-source"; + if (!texts.some((text) => !isBlank(text))) return "no-translatable-text"; + if (fragments.length === 1) return "single-leaf"; + return undefined; +}; + +/** + * Walks a serialized Lexical tree and returns its containers, in document order. + * + * A **container** is the nearest node with at least one direct text child. The walk descends + * until it finds one, then stops: everything below belongs to that container as its content. + * Naming no node types is deliberate — the rule holds for paragraphs, headings, list items and + * quotes alike, and for whatever is added later. The cost is that a node holding both its own + * inline text and a nested block cannot be told apart from a paragraph holding a link, so the + * nested block is treated as content. Real Lexical trees do not mix the two. + * + * Guarantees: + * - fragments are in document order, numbered from 1 within each container — the container is + * one string, so numbering restarts; + * - a leaf holding only whitespace (or nothing) never becomes a fragment: its text is glued + * onto the nearest fragment that carries text, searching backwards first, then forwards, so + * the gap between two words survives and its node drops out. Such a leaf still counts as a + * direct text child when deciding whether a node is a container; + * - copies duplicate the whole chain from the container's direct child down to the leaf, so two + * copies never share a mutable node; + * - **the input tree is not mutated** — the source nodes are read only; + * - a root with no `children` yields an empty list rather than throwing. + * + * @param root - the root node of a serialized Lexical value + */ +export function collectInlineFragments(root: SerializedLexicalNode): InlineContainer[] { + const containers: InlineContainer[] = []; + + const visit = (node: SerializedLexicalNode): void => { + if (!hasDirectTextChild(node)) { + if (hasChildren(node)) for (const child of node.children) visit(child); + return; + } + + const drafts = draftsOf(node); + const fragments = glueWhitespace(drafts); + const skip = drafts.some((draft) => draft.kind === "unsupported") + ? "unsupported-wrapper" + : skipReasonFor(fragments); + + containers.push({ + node, + fragments, + ...(skip ? { skip } : {}), + }); + }; + + visit(root); + return containers; +} diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts new file mode 100644 index 000000000..e64685e57 --- /dev/null +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts @@ -0,0 +1,334 @@ +/** + * Contract tests for the numbered inline mark format, written from + * `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (Emitting, Accepting a + * reply, Marks are flat, Whitespace) and the JSDoc in `inlineMarks.ts` — not from any + * implementation. Every assertion below quotes a sentence of that contract. + */ + +import { describe, it, expect } from "vitest"; +import type { MarkableFragment, MarkFailure, ParsedMark, ParseResult } from "./inlineMarks"; +import { serializeInlineMarks, parseInlineMarks } from "./inlineMarks"; + +const fragment = (markId: number, text: string | null): MarkableFragment => ({ markId, text }); + +/** Narrows an accepted reply, so an assertion below fails for its own reason and not for `ok`. */ +const acceptedFragments = (result: ParseResult): ParsedMark[] => { + if (!result.ok) { + throw new Error(`expected an accepted reply, got the failure "${result.reason}"`); + } + return result.fragments; +}; + +/** Narrows a rejected reply the same way. */ +const rejectionReason = (result: ParseResult): MarkFailure => { + if (result.ok) { + throw new Error("expected a rejected reply, got an accepted one"); + } + return result.reason; +}; + +describe("serializeInlineMarks", () => { + it("serialize wraps every fragment in its own numbered mark", () => { + const marked = serializeInlineMarks([ + fragment(1, "a "), + fragment(2, "red"), + fragment(3, " car"), + ]); + + expect(marked).toBe("<1>a <2>red<3> car"); + }); + + // D4: "Every fragment is wrapped, including ones carrying no formatting." + it("serialize wraps a lone fragment as well", () => { + const marked = serializeInlineMarks([fragment(1, "a car")]); + + expect(marked).toBe("<1>a car"); + }); + + it("serialize renders a text-free fragment self-closing", () => { + const marked = serializeInlineMarks([fragment(2, null)]); + + expect(marked).toBe("<2/>"); + }); + + // §4 Emitting, "non-text inline node (line break, inline block, upload) | ``" — + // a line break mid-paragraph. + it("serialize renders a line break between two texts as a self-closing mark", () => { + const marked = serializeInlineMarks([ + fragment(1, "first line"), + fragment(2, null), + fragment(3, "second line"), + ]); + + expect(marked).toBe("<1>first line<2/><3>second line"); + }); + + // §4 Marks are flat: a link with an emphasised word inside "emits two flat, adjacent marks: + // `<4>read the <5>docs`" — the number comes from the fragment, not its position. + it("serialize uses the numbers it was given, not the positions", () => { + const marked = serializeInlineMarks([fragment(4, "read the "), fragment(5, "docs")]); + + expect(marked).toBe("<4>read the <5>docs"); + }); + + it("serialize keeps the order given even when the numbers do not ascend", () => { + const marked = serializeInlineMarks([fragment(3, " car"), fragment(1, "a ")]); + + expect(marked).toBe("<3> car<1>a "); + }); +}); + +describe("parseInlineMarks", () => { + describe("reply order", () => { + // §1: sent `<1>a <2>red<3> car`, returned + // `<1>une <3>voiture <2>rouge` — "fragments come back in the reply's order". + const frenchSent = [fragment(1, "a "), fragment(2, "red"), fragment(3, " car")]; + const frenchReply = "<1>une <3>voiture <2>rouge"; + + it("reorder: the French adjective reply is accepted", () => { + const result = parseInlineMarks(frenchReply, frenchSent); + + expect(result.ok).toBe(true); + }); + + it("reorder: fragments come back in the reply's order, not the sent order", () => { + const result = parseInlineMarks(frenchReply, frenchSent); + + expect(acceptedFragments(result).map((f) => f.markId)).toEqual([1, 3, 2]); + }); + + it("reorder: each number keeps its own text after the move", () => { + const result = parseInlineMarks(frenchReply, frenchSent); + + expect(acceptedFragments(result).map((f) => f.text)).toEqual(["une ", "voiture ", "rouge"]); + }); + + // Same principle, another permutation: a German subordinate clause sends the verb to the end. + it("reorder: a German subordinate clause comes back in its own order", () => { + const sent = [ + fragment(1, "I know "), + fragment(2, "that he "), + fragment(3, "reads"), + fragment(4, " the book"), + ]; + + const result = parseInlineMarks( + "<1>Ich weiß <2>dass er <4>das Buch <3>liest", + sent + ); + + expect(acceptedFragments(result).map((f) => f.markId)).toEqual([1, 2, 4, 3]); + }); + + it("reorder: a reply left in the original order is accepted unchanged", () => { + const sent = [fragment(1, "a "), fragment(2, "red"), fragment(3, " car")]; + + const result = parseInlineMarks("<1>ein <2>rotes<3> Auto", sent); + + expect(acceptedFragments(result).map((f) => f.markId)).toEqual([1, 2, 3]); + }); + + it("reorder: a single-fragment reply parses back to that one fragment", () => { + const result = parseInlineMarks("<1>une voiture", [fragment(1, "a car")]); + + expect(acceptedFragments(result)).toEqual([{ markId: 1, text: "une voiture" }]); + }); + }); + + describe("verdicts", () => { + const mergeSent = [fragment(1, "a "), fragment(2, "red"), fragment(3, " car")]; + const mergeReply = "<1>une voiture rouge<2><3>"; + + it("verdict is ok when a merge is expressed as an empty mark", () => { + const result = parseInlineMarks(mergeReply, mergeSent); + + expect(result.ok).toBe(true); + }); + + it("verdict: a merged-away fragment comes back with an empty string", () => { + const result = parseInlineMarks(mergeReply, mergeSent); + + expect(acceptedFragments(result).map((f) => f.text)).toEqual(["une voiture rouge", "", ""]); + }); + + it("verdict is missing-mark when an issued number does not come back", () => { + const result = parseInlineMarks("<1>une <2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + fragment(3, " car"), + ]); + + expect(rejectionReason(result)).toBe("missing-mark"); + }); + + it("verdict is missing-mark for an empty reply", () => { + const result = parseInlineMarks("", [fragment(1, "a "), fragment(2, "red")]); + + expect(rejectionReason(result)).toBe("missing-mark"); + }); + + it("verdict is unknown-mark when a number nobody issued appears", () => { + const result = parseInlineMarks("<1>une <2>rouge<3>extra", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(rejectionReason(result)).toBe("unknown-mark"); + }); + + it("verdict is repeated-mark when an issued number comes back twice", () => { + const result = parseInlineMarks("<1>une <2>rouge<1>encore", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(rejectionReason(result)).toBe("repeated-mark"); + }); + + it("verdict is nested-marks when a mark opens inside another", () => { + const result = parseInlineMarks("<1><2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(rejectionReason(result)).toBe("nested-marks"); + }); + + it("verdict is crossed-marks when a mark closes with another number", () => { + const result = parseInlineMarks("<1>une voiture rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(rejectionReason(result)).toBe("crossed-marks"); + }); + + // Same row: "Unclosed ... marks | corrupt" → `unclosed-mark` + // ("A mark opened and never closed"). + it("verdict is unclosed-mark when a mark is never closed", () => { + const result = parseInlineMarks("<1>une <2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(rejectionReason(result)).toBe("unclosed-mark"); + }); + + it("verdict is no-text when every mark comes back empty", () => { + const result = parseInlineMarks("<1><2>", [fragment(1, "a "), fragment(2, "red")]); + + expect(rejectionReason(result)).toBe("no-text"); + }); + + it("verdict: a corrupt reply carries no fragments at all", () => { + const result = parseInlineMarks("<1>une ", [fragment(1, "a "), fragment(2, "red")]); + + expect((result as { fragments?: unknown }).fragments).toBeUndefined(); + }); + + it("verdict is ok with stray whitespace inside an opening mark", () => { + const result = parseInlineMarks("< 1 >une <2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(result.ok).toBe(true); + }); + + it("verdict is ok with stray whitespace inside a closing mark", () => { + const result = parseInlineMarks("<1>une <2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(result.ok).toBe(true); + }); + + it("verdict: stray whitespace inside a mark does not leak into the text", () => { + const result = parseInlineMarks("< 1 >une <2>rouge", [ + fragment(1, "a "), + fragment(2, "red"), + ]); + + expect(acceptedFragments(result).map((f) => f.text)).toEqual(["une ", "rouge"]); + }); + + it("verdict is ok when a text-free fragment comes back self-closing", () => { + const result = parseInlineMarks("<1>première<2/><3>seconde", [ + fragment(1, "first"), + fragment(2, null), + fragment(3, "second"), + ]); + + expect(result.ok).toBe(true); + }); + + it("verdict is ok when a text-free fragment comes back as an open/close pair", () => { + const result = parseInlineMarks("<1>première<2><3>seconde", [ + fragment(1, "first"), + fragment(2, null), + fragment(3, "second"), + ]); + + expect(result.ok).toBe(true); + }); + + it("verdict: a text-free fragment comes back with an empty string", () => { + const result = parseInlineMarks("<1>première<2/><3>seconde", [ + fragment(1, "first"), + fragment(2, null), + fragment(3, "second"), + ]); + + expect(acceptedFragments(result).map((f) => f.text)).toEqual(["première", "", "seconde"]); + }); + }); + + describe("restoring trimmed edges", () => { + it("edge whitespace: a trailing space the model trimmed is restored", () => { + const result = parseInlineMarks("<1>une<2>voiture", [ + fragment(1, "a "), + fragment(2, "car"), + ]); + + expect(acceptedFragments(result)[0]?.text).toBe("une "); + }); + + it("edge whitespace: a leading space the model trimmed is restored", () => { + const result = parseInlineMarks("<1>rouge<2>voiture", [ + fragment(1, "red"), + fragment(2, " car"), + ]); + + expect(acceptedFragments(result)[1]?.text).toBe(" voiture"); + }); + + it("edge whitespace: a space the model kept is not doubled", () => { + const result = parseInlineMarks("<1>une <2>voiture", [ + fragment(1, "a "), + fragment(2, "car"), + ]); + + expect(acceptedFragments(result)[0]?.text).toBe("une "); + }); + + // The rule fires only for a source that "began or ended with a space". + it("edge whitespace: a fragment whose source had no edge space gains none", () => { + const result = parseInlineMarks("<1>une<2>voiture", [ + fragment(1, "a"), + fragment(2, "car"), + ]); + + expect(acceptedFragments(result)[0]?.text).toBe("une"); + }); + + it("edge whitespace: an emptied fragment is left empty", () => { + const result = parseInlineMarks("<1>une voiture rouge<2>", [ + fragment(1, "a "), + fragment(2, " red car"), + ]); + + expect(acceptedFragments(result)[1]?.text).toBe(""); + }); + }); +}); diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts new file mode 100644 index 000000000..65f761b13 --- /dev/null +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts @@ -0,0 +1,207 @@ +/** + * Numbered inline marks: the wire format that lets a whole container be translated as one + * string while still saying which piece of it carried which formatting. + * + * Knows nothing about Lexical — or about any document format. It reads and writes strings and + * numbered fragments, and that is deliberate: the mark format changes for different reasons + * than a tree walk does. + * + * Design: `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (D2, D4, D13, D16). + */ + +/** + * A fragment as the mark format sees it: a number, and the text it holds. + * + * `text: null` means the fragment carries no text at all — a line break, an inline block. It + * still needs a number, because it still occupies a position the reply may move it to. + */ +export type MarkableFragment = { + markId: number; + text: string | null; +}; + +/** One fragment read back out of a reply. Text-free fragments come back with `text: ""`. */ +export type ParsedMark = { + markId: number; + text: string; +}; + +/** + * Why a reply could not be used. Corruption is an ordinary outcome here, not an anomaly — + * the caller retranslates the container per-node — so it is a returned value, never a throw. + */ +export type MarkFailure = + | "missing-mark" + | "unknown-mark" + | "repeated-mark" + | "nested-marks" + /** Closed with a different number than it opened with — not interleaving. */ + | "crossed-marks" + | "unclosed-mark" + | "no-text"; + +/** No partial success: half a container written into a document is the failure nobody notices. */ +export type ParseResult = + | { ok: true; fragments: ParsedMark[] } + | { ok: false; reason: MarkFailure }; + +/** `<1>`, ``, `<1/>` — with whitespace and leading zeros tolerated, because models add both. */ +const MARK_TOKEN = /<\s*(\/?)\s*(\d+)\s*(\/?)\s*>/gu; + +type Token = { kind: "open" | "close" | "self"; markId: number } | { kind: "text"; text: string }; + +const tokenize = (reply: string): Token[] => { + const tokens: Token[] = []; + let cursor = 0; + + MARK_TOKEN.lastIndex = 0; + let match = MARK_TOKEN.exec(reply); + while (match !== null) { + if (match.index > cursor) { + tokens.push({ kind: "text", text: reply.slice(cursor, match.index) }); + } + + const [, leadingSlash, digits, trailingSlash] = match; + const markId = Number(digits); + if (leadingSlash) tokens.push({ kind: "close", markId }); + else if (trailingSlash) tokens.push({ kind: "self", markId }); + else tokens.push({ kind: "open", markId }); + + cursor = match.index + match[0].length; + match = MARK_TOKEN.exec(reply); + } + + if (cursor < reply.length) tokens.push({ kind: "text", text: reply.slice(cursor) }); + return tokens; +}; + +/** Restores an edge the model trimmed, using the source's own characters (a tab is not a space). */ +const restoreEdges = (translated: string, source: string): string => { + if (!translated) return translated; + + const leading = source.match(/^\s+/u)?.[0] ?? ""; + const trailing = source.match(/\s+$/u)?.[0] ?? ""; + const needsLeading = leading && translated === translated.trimStart(); + const needsTrailing = trailing && translated === translated.trimEnd(); + + return `${needsLeading ? leading : ""}${translated}${needsTrailing ? trailing : ""}`; +}; + +/** + * Renders fragments as one marked string, in the order given. + * + * Every fragment is wrapped, including ones carrying no formatting: the caller then never has + * to build a node from scratch on the way back. A text-free fragment renders self-closing. + * + * Mark numbers are written as given — they identify fragments and need not be contiguous. Text + * is written verbatim: a fragment whose own text looks like a mark is refused one level up, by + * the collector. An empty fragment list renders an empty string. + */ +export function serializeInlineMarks(fragments: MarkableFragment[]): string { + return fragments + .map((fragment) => + fragment.text === null + ? `<${fragment.markId}/>` + : `<${fragment.markId}>${fragment.text}` + ) + .join(""); +} + +/** + * Reads a reply back into fragments. + * + * On success fragments come back **in the reply's order** — that is the point of the format, as + * the target language decides where each piece belongs. A mark returned empty is legitimate and + * yields `text: ""`; its caller drops that node. + * + * Accepted liberties: any order; whitespace inside a mark, newlines included (`< 1 >`); a + * leading zero; text sitting outside any mark, which is appended to the preceding mark (to the + * first when there is none) rather than discarded — a model that lost a boundary mid-sentence + * still returned the words. A fragment sent text-free that comes back carrying text has that + * text ignored: there is no leaf to write it into. + * + * Edge whitespace is restored **only when the reply has the same shape as the request** — same + * order, no mark returned empty. Once pieces move or merge their edges change legitimately, and + * restoring one then produces a double space or a space trailing a paragraph. When it does + * apply, the source's own characters come back, not a plain space: a tab or a non-breaking + * space was put there on purpose. + * + * `no-text` is judged on non-blank content. When a reply is corrupt several ways at once, the + * reason reported is the first of: structural (`nested-marks`, `crossed-marks`, + * `unclosed-mark`), then the mark set (`unknown-mark`, `repeated-mark`, `missing-mark`), then + * `no-text` — structure first, because a string that cannot be parsed unambiguously has no + * reliable mark set to compare. + * + * @param reply - the string a translator returned + * @param fragments - the fragments that were sent: which numbers were issued, and their source text + */ +export function parseInlineMarks(reply: string, fragments: MarkableFragment[]): ParseResult { + const collected: ParsedMark[] = []; + let openMarkId: number | null = null; + let openText = ""; + let strayBeforeFirst = ""; + + const appendStray = (text: string) => { + const last = collected.at(-1); + if (last) last.text += text; + else strayBeforeFirst += text; + }; + + for (const token of tokenize(reply)) { + if (token.kind === "text") { + if (openMarkId === null) appendStray(token.text); + else openText += token.text; + continue; + } + + if (token.kind === "self") { + if (openMarkId !== null) return { ok: false, reason: "nested-marks" }; + collected.push({ markId: token.markId, text: "" }); + continue; + } + + if (token.kind === "open") { + if (openMarkId !== null) return { ok: false, reason: "nested-marks" }; + openMarkId = token.markId; + openText = ""; + continue; + } + + if (openMarkId === null || openMarkId !== token.markId) { + return { ok: false, reason: "crossed-marks" }; + } + collected.push({ markId: openMarkId, text: openText }); + openMarkId = null; + openText = ""; + } + + if (openMarkId !== null) return { ok: false, reason: "unclosed-mark" }; + if (strayBeforeFirst && collected.length > 0) { + const first = collected[0]; + if (first) first.text = strayBeforeFirst + first.text; + } + + const issued = new Map(fragments.map((fragment) => [fragment.markId, fragment])); + const seen = new Set(); + for (const mark of collected) { + if (!issued.has(mark.markId)) return { ok: false, reason: "unknown-mark" }; + if (seen.has(mark.markId)) return { ok: false, reason: "repeated-mark" }; + seen.add(mark.markId); + } + if (seen.size !== issued.size) return { ok: false, reason: "missing-mark" }; + if (!collected.some((mark) => mark.text.trim())) return { ok: false, reason: "no-text" }; + + const sameShape = + collected.length === fragments.length && + collected.every((mark, index) => mark.markId === fragments[index]?.markId && mark.text !== ""); + + return { + ok: true, + fragments: sameShape + ? collected.map((mark) => { + const source = issued.get(mark.markId)?.text; + return source ? { ...mark, text: restoreEdges(mark.text, source) } : mark; + }) + : collected, + }; +} From af002af61ecd742bcedc78551dc390d80d82f4ba Mon Sep 17 00:00:00 2001 From: Siarhei Date: Fri, 11 Sep 2026 13:52:39 +0200 Subject: [PATCH 2/2] chore(translator): cut the comments back, and put two rules where they are enforced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up on the same modules after auditing the tests, the complexity and the comments. Behaviour is unchanged; three of the findings were defects rather than verbosity. **Citations to a file that does not exist.** Sixteen references to `docs/plans/2026-09-08-richtext-container-granularity-design.md` and its decision ids, mostly in the tests. That document was taken out of version control by 23fc243b and is on no branch, so none of them could be followed. Removed; the rationale they pointed at is now stated where it is needed or not at all. **A docblock that was already false.** `parseInlineMarks` claimed text returned for a fragment sent text-free "is ignored". It is not — the function returns it in `collected`. Corrected. **A guard for a state its caller rules out.** `restoreEdges` began with `if (!translated) return translated;`, but `sameShape` already excludes every mark whose text came back empty. Unexplained defensive code reads as evidence the case is reachable, so the next reader preserves it while changing the condition that made it dead. Removed. **A name that said the wrong thing.** `crossed-marks` reads as interleaving to everyone, and the comment beside it existed to say it is not: the case is a mark closed with a different number than it opened with. Renamed to `mismatched-close`, and the comment went with it. Two rules moved out of prose: - `text` and `node` were documented as "null together" and checked by nothing. They are now one discriminated union, so the pairing is a compile error rather than a promise — confirmed by building the impossible combination and reading TS2322. - The skip decision was computed from two sources — `drafts` in the walk, `fragments` in `skipReasonFor` — with an unwritten precedence between them. One function now holds the whole order, stated in one line. Comment lines: 71 → 43 and 63 → 42 in the two modules, 52 → 3 across their tests. Four comments survived the pass: the backwards glue search, the all-or-nothing rule, the mark pattern's tolerance of whitespace and leading zeros (models emit both), and the one test note explaining why a length assertion sits beside a non-mutation check. Verification: 1469 unit tests, unchanged in count and all green; check-types clean; lint 58 warnings and 0 errors, identical to main; declaration build passes. --- .../lexical/collectInlineFragments.test.ts | 33 +---- .../kernel/lexical/collectInlineFragments.ts | 120 +++++++----------- .../core/kernel/lexical/inlineMarks.test.ts | 24 +--- .../src/core/kernel/lexical/inlineMarks.ts | 71 ++++------- 4 files changed, 75 insertions(+), 173 deletions(-) diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts index 4c85d9561..cdbf6e0dc 100644 --- a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.test.ts @@ -1,17 +1,9 @@ -/** - * Contract tests for the container walk, written from - * `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (Emitting, Finding the - * container, Whitespace, Plain values) and the JSDoc in `collectInlineFragments.ts` — not from - * any implementation. Every assertion below quotes a sentence of that contract. - */ - import { describe, it, expect } from "vitest"; import { collectInlineFragments } from "./collectInlineFragments"; const createNode = (type: string, props?: Record, children?: any[]) => ({ type, ...props, ...(children && { children }) }) as any; -/** The children of a rebuilt-array entry, for the wrapper-copy assertions. */ const childrenOf = (node: unknown): unknown[] => ((node as { children?: unknown[] }).children ?? []) as unknown[]; @@ -65,8 +57,6 @@ describe("collectInlineFragments", () => { ]); }); - // §4 diagram: "list ← no direct text, descend / listitem ← container (each item on its own, - // as it must be)". it("takes each list item as its own container", () => { const firstItem = createNode("listitem", {}, [createNode("text", { text: "first" })]); const secondItem = createNode("listitem", {}, [createNode("text", { text: "second" })]); @@ -78,7 +68,6 @@ describe("collectInlineFragments", () => { ]); }); - // §4 diagram: "quote → paragraph → text ← the paragraph is the container". it("takes a quote's paragraph as the container, not the quote", () => { const paragraph = createNode("paragraph", {}, [createNode("text", { text: "quoted" })]); const root = createNode("root", {}, [createNode("quote", {}, [paragraph])]); @@ -86,7 +75,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root).map((container) => container.node)).toEqual([paragraph]); }); - // A node with no text child anywhere below never qualifies, so it yields no container. it("yields no container for a node with no text children at all", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [createNode("upload", { relationTo: "media" })]), @@ -95,7 +83,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root)).toEqual([]); }); - // §4 diagram: "block (fields, not children) — no text children, walked past". it("walks past a block that carries fields instead of children", () => { const first = createNode("paragraph", {}, [createNode("text", { text: "before" })]); const second = createNode("paragraph", {}, [createNode("text", { text: "after" })]); @@ -215,7 +202,6 @@ describe("collectInlineFragments", () => { ]); }); - // Emitting table, row 1: "text leaf, direct child of the container | the leaf | the same node". it("points a direct text leaf's node at that very leaf", () => { const leaf = createNode("text", { text: "Buy " }); const root = createNode("root", {}, [ @@ -238,8 +224,6 @@ describe("collectInlineFragments", () => { expect(fragment?.top).toBe(fragment?.node); }); - // Emitting table, row 2: "the container's direct child holds exactly one leaf | the leaf | - // that direct child — any chain above the leaf rides along inside it". it("uses the container's direct child as top when that wrapper holds one leaf", () => { const link = createNode("link", { url: "https://example.com" }, [ createNode("text", { text: "our product" }), @@ -276,7 +260,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root)[0]?.fragments[1]?.top).toBe(mark); }); - // Emitting table, row 3 + §4 Marks are flat: a link containing an emphasised word const linkWithTwoLeaves = () => { const emphasised = createNode("text", { text: "docs", format: 1 }); const plain = createNode("text", { text: "read the " }); @@ -323,8 +306,6 @@ describe("collectInlineFragments", () => { expect(childrenOf(fragment?.top)[0]).toBe(fragment?.node); }); - // Emitting table, row 4: "non-text inline node (line break, inline block, upload) | ``", - // with no node of its own — a line break mid-paragraph. const lineBreakMidParagraph = () => { const lineBreak = createNode("linebreak"); const root = createNode("root", {}, [ @@ -365,7 +346,6 @@ describe("collectInlineFragments", () => { }); describe("whitespace-only nodes", () => { - // §4 Whitespace: "Buy" / " " / "our product" formatted. const spaceBetweenWords = () => { const buy = createNode("text", { text: "Buy" }); const space = createNode("text", { text: " " }); @@ -430,8 +410,6 @@ describe("collectInlineFragments", () => { expect(fragments?.map((fragment) => fragment.text)).toEqual(["Hello ", null, "World"]); }); - // Same guarantee at the end of a container: with nothing following, the glue has nowhere to - // go forwards, and dropping it would lose a character with no signal. it("whitespace: does not lose trailing glue after a text-free fragment", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [ @@ -462,8 +440,6 @@ describe("collectInlineFragments", () => { }); describe("containers that cannot use marks", () => { - // D3 / "mark-shaped-source": "Its source text contains a mark-shaped sequence, so parsing a - // reply would be ambiguous." The shapes named are `<12>`, ``, `<12/>`. it("mark-shaped: an opening-shaped sequence in the source skips the container", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [ @@ -516,7 +492,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root)[0]?.fragments).toHaveLength(2); }); - // D3: "Plain `
` ... unaffected — only digits between angle brackets collide". it("mark-shaped: a plain tag like
is not mark-shaped", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [ @@ -530,7 +505,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); }); - // D3: "or `5 < 10` are unaffected". it("mark-shaped: a comparison like 5 < 10 is not mark-shaped", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [ @@ -544,8 +518,6 @@ describe("collectInlineFragments", () => { expect(collectInlineFragments(root)[0]?.skip).toBeUndefined(); }); - // D6 / "single-leaf": "A single unformatted leaf: there is nothing to reorder and - // marks would be pure cost." it("skips a container that is one plain text leaf", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [createNode("text", { text: "A plain paragraph." })]), @@ -605,7 +577,7 @@ describe("collectInlineFragments", () => { const containers = collectInlineFragments(root); - // Same reason as above, plus the copy itself: `top` must not be the wrapper from the tree. + // Length assertion as above: without it a collector returning [] would pass this test. expect(containers[0]?.fragments).toHaveLength(3); expect(containers[0]?.fragments[1]?.top).not.toBe( (root as { children: { children: unknown[] }[] }).children[0]?.children[1] @@ -615,9 +587,6 @@ describe("collectInlineFragments", () => { }); describe("wrapper shapes the copy cannot carry", () => { - // Copying a wrapper keeps only the path down to one leaf, so a sibling that is neither a leaf - // nor on that path — a line break between two formatted words inside one link — would vanish - // from every copy with no fragment and no warning. Skipping the container is the honest answer. it("wrapper: skips a container whose multi-leaf wrapper holds a node the copy would drop", () => { const root = createNode("root", {}, [ createNode("paragraph", {}, [ diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts index 65f43df1c..a342cf8b9 100644 --- a/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/collectInlineFragments.ts @@ -1,16 +1,8 @@ /** - * Container-level collection: groups a Lexical tree into the units that get translated as one - * string each, and the fragments inside them. + * Groups a Lexical tree into containers and the fragments inside them. * - * Reads only `type`, `text` and `children` — the surface `types.ts` declares. Formatting - * (`format`, `style`, a link's `fields`) is never read: it travels inside the nodes themselves, - * which is what lets this layer reorder formatted pieces without understanding formatting. - * - * The existing per-node walk (`collectTextNodes.ts`) is left alone: it feeds the per-node - * translation path and the provenance fingerprint, and widening it would change stored - * fingerprint values. - * - * Design: `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (D1, D3, D6, D15, D17). + * Formatting (`format`, `style`, a link's `fields`) is deliberately never read: it travels inside + * the nodes themselves, which is what lets this layer reorder formatted pieces at all. */ import { hasChildren, isSerializedLexicalTextNode } from "./guards"; @@ -20,43 +12,32 @@ import type { SerializedLexicalNode, SerializedTextNode } from "./types"; * One translatable piece of a container. * * `node` is where the translation is written; `top` is what goes into the container's rebuilt - * `children`. They are usually two views of the same subtree — `node` the leaf, `top` the - * container's direct child above it. - * - * When the container's direct child holds more than one leaf (a link with an emphasised word - * inside), `top` is instead a **copy** of that child holding only this fragment's leaf, and - * `node` points into that copy. Pushing one shared wrapper once per leaf would duplicate its - * whole text rather than reorder it. - * - * `text` and `node` are `null` together, for a fragment that carries no text at all. + * `children`. For a multi-leaf wrapper `top` is a copy holding only this leaf — pushing one + * shared wrapper once per leaf would duplicate its whole text instead of reordering it. */ -export type InlineFragment = { - markId: number; - text: string | null; - node: SerializedTextNode | null; - top: SerializedLexicalNode; -}; +export type InlineFragment = { markId: number; top: SerializedLexicalNode } & ( + | { text: string; node: SerializedTextNode } + | { text: null; node: null } +); + +/** A fragment that carries text, so `node` is present — what `glueWhitespace` can append to. */ +type TextFragment = Extract; -/** Why a container cannot use the marked format, when it cannot. */ export type ContainerSkipReason = - /** Its source text contains a mark-shaped sequence, so parsing a reply would be ambiguous. */ + /** Parsing the reply would be ambiguous — see `MARK_SHAPED`. */ | "mark-shaped-source" - /** A single text leaf: nothing to reorder, so marks would be pure cost. */ + /** Nothing to reorder, so marks would be pure cost — not a failure. */ | "single-leaf" | "no-translatable-text" /** - * A wrapper holds several leaves *and* a node that is neither: copying the wrapper once per - * leaf keeps only the path down to that leaf, so the odd node would be dropped from every copy - * — silently, with no fragment of its own. Skipping is honest where a copy is not. + * A multi-leaf wrapper also holding a non-text node: the per-leaf copy keeps only the path down + * to its leaf, so that node would vanish from every copy with no fragment and no signal. */ | "unsupported-wrapper"; /** - * A container and its fragments. - * - * `node` is the node whose `children` the caller will rebuild. When `skip` is set the caller - * translates this container the per-node way instead, and `fragments` is still populated so - * the decision needs no second walk. + * `node` is the node whose `children` the caller rebuilds. `fragments` is populated even when + * `skip` is set. */ export type InlineContainer = { node: SerializedLexicalNode; @@ -64,6 +45,7 @@ export type InlineContainer = { skip?: ContainerSkipReason; }; +// Must stay as wide as MARK_TOKEN in ./inlineMarks — a source this misses is a reply that mis-parses. const MARK_SHAPED = /<\s*\/?\s*\d+\s*\/?\s*>/u; const isBlank = (text: string): boolean => text.trim().length === 0; @@ -71,7 +53,6 @@ const isBlank = (text: string): boolean => text.trim().length === 0; const hasDirectTextChild = (node: SerializedLexicalNode): boolean => hasChildren(node) && node.children.some((child) => isSerializedLexicalTextNode(child)); -/** A node with no children that is not text: a line break, an inline block, an upload. */ const hasNonTextLeafInside = (node: SerializedLexicalNode): boolean => { if (isSerializedLexicalTextNode(node)) return false; if (!hasChildren(node)) return true; @@ -84,7 +65,6 @@ const leavesOf = (node: SerializedLexicalNode): SerializedTextNode[] => { return node.children.flatMap(leavesOf); }; -/** Copies never share a mutable node: each leaf gets its own chain down from the container's child. */ const copyChainToLeaf = ( node: SerializedLexicalNode, leaf: SerializedTextNode @@ -106,12 +86,10 @@ const copyChainToLeaf = ( }; type Draft = - | { - kind: "fragment"; - text: string | null; - node: SerializedTextNode | null; - top: SerializedLexicalNode; - } + | ({ kind: "fragment"; top: SerializedLexicalNode } & ( + | { text: string; node: SerializedTextNode } + | { text: null; node: null } + )) | { kind: "glue"; text: string } | { kind: "unsupported" }; @@ -152,7 +130,7 @@ const glueWhitespace = (drafts: Draft[]): InlineFragment[] => { // Searching backwards has to skip text-free fragments: a line break between the text and the // whitespace must not send the glue forwards, and at the end of a container it would drop it. - const lastWithText = (): InlineFragment | undefined => { + const lastWithText = (): TextFragment | undefined => { for (let index = fragments.length - 1; index >= 0; index -= 1) { const candidate = fragments[index]; if (candidate && candidate.text !== null) return candidate; @@ -169,17 +147,27 @@ const glueWhitespace = (drafts: Draft[]): InlineFragment[] => { continue; } - const text = - draft.text !== null && carried.length > 0 - ? carried.splice(0).join("") + draft.text - : draft.text; - fragments.push({ markId: fragments.length + 1, text, node: draft.node, top: draft.top }); + const markId = fragments.length + 1; + if (draft.text === null) { + fragments.push({ markId, text: null, node: null, top: draft.top }); + continue; + } + + const text = carried.length > 0 ? carried.splice(0).join("") + draft.text : draft.text; + fragments.push({ markId, text, node: draft.node, top: draft.top }); } return fragments; }; -const skipReasonFor = (fragments: InlineFragment[]): ContainerSkipReason | undefined => { +// Ordered: an unsupported shape and an ambiguous source are correctness problems, so they outrank +// the last two, which only decide whether marks would pay for themselves. +const skipReasonFor = ( + drafts: Draft[], + fragments: InlineFragment[] +): ContainerSkipReason | undefined => { + if (drafts.some((draft) => draft.kind === "unsupported")) return "unsupported-wrapper"; + const texts = fragments.flatMap((fragment) => (fragment.text === null ? [] : [fragment.text])); if (texts.some((text) => MARK_SHAPED.test(text))) return "mark-shaped-source"; @@ -191,26 +179,16 @@ const skipReasonFor = (fragments: InlineFragment[]): ContainerSkipReason | undef /** * Walks a serialized Lexical tree and returns its containers, in document order. * - * A **container** is the nearest node with at least one direct text child. The walk descends - * until it finds one, then stops: everything below belongs to that container as its content. + * A **container** is the nearest node with at least one direct text child; the walk stops there. * Naming no node types is deliberate — the rule holds for paragraphs, headings, list items and - * quotes alike, and for whatever is added later. The cost is that a node holding both its own - * inline text and a nested block cannot be told apart from a paragraph holding a link, so the - * nested block is treated as content. Real Lexical trees do not mix the two. + * whatever is added later. The cost: a node mixing its own inline text with a nested block is + * treated as a container, and the block becomes content. Real Lexical trees do not mix the two. * - * Guarantees: - * - fragments are in document order, numbered from 1 within each container — the container is - * one string, so numbering restarts; - * - a leaf holding only whitespace (or nothing) never becomes a fragment: its text is glued - * onto the nearest fragment that carries text, searching backwards first, then forwards, so - * the gap between two words survives and its node drops out. Such a leaf still counts as a - * direct text child when deciding whether a node is a container; - * - copies duplicate the whole chain from the container's direct child down to the leaf, so two - * copies never share a mutable node; - * - **the input tree is not mutated** — the source nodes are read only; - * - a root with no `children` yields an empty list rather than throwing. + * Fragments are numbered from 1 within each container — the container is one string, so the + * numbering restarts. * - * @param root - the root node of a serialized Lexical value + * The input tree is not mutated, but a fragment's `node` may BE a source node: writing to it + * writes into the caller's tree. */ export function collectInlineFragments(root: SerializedLexicalNode): InlineContainer[] { const containers: InlineContainer[] = []; @@ -223,9 +201,7 @@ export function collectInlineFragments(root: SerializedLexicalNode): InlineConta const drafts = draftsOf(node); const fragments = glueWhitespace(drafts); - const skip = drafts.some((draft) => draft.kind === "unsupported") - ? "unsupported-wrapper" - : skipReasonFor(fragments); + const skip = skipReasonFor(drafts, fragments); containers.push({ node, diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts index e64685e57..add9c4764 100644 --- a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.test.ts @@ -1,17 +1,9 @@ -/** - * Contract tests for the numbered inline mark format, written from - * `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (Emitting, Accepting a - * reply, Marks are flat, Whitespace) and the JSDoc in `inlineMarks.ts` — not from any - * implementation. Every assertion below quotes a sentence of that contract. - */ - import { describe, it, expect } from "vitest"; import type { MarkableFragment, MarkFailure, ParsedMark, ParseResult } from "./inlineMarks"; import { serializeInlineMarks, parseInlineMarks } from "./inlineMarks"; const fragment = (markId: number, text: string | null): MarkableFragment => ({ markId, text }); -/** Narrows an accepted reply, so an assertion below fails for its own reason and not for `ok`. */ const acceptedFragments = (result: ParseResult): ParsedMark[] => { if (!result.ok) { throw new Error(`expected an accepted reply, got the failure "${result.reason}"`); @@ -19,7 +11,6 @@ const acceptedFragments = (result: ParseResult): ParsedMark[] => { return result.fragments; }; -/** Narrows a rejected reply the same way. */ const rejectionReason = (result: ParseResult): MarkFailure => { if (result.ok) { throw new Error("expected a rejected reply, got an accepted one"); @@ -38,7 +29,6 @@ describe("serializeInlineMarks", () => { expect(marked).toBe("<1>a <2>red<3> car"); }); - // D4: "Every fragment is wrapped, including ones carrying no formatting." it("serialize wraps a lone fragment as well", () => { const marked = serializeInlineMarks([fragment(1, "a car")]); @@ -51,8 +41,6 @@ describe("serializeInlineMarks", () => { expect(marked).toBe("<2/>"); }); - // §4 Emitting, "non-text inline node (line break, inline block, upload) | ``" — - // a line break mid-paragraph. it("serialize renders a line break between two texts as a self-closing mark", () => { const marked = serializeInlineMarks([ fragment(1, "first line"), @@ -63,8 +51,6 @@ describe("serializeInlineMarks", () => { expect(marked).toBe("<1>first line<2/><3>second line"); }); - // §4 Marks are flat: a link with an emphasised word inside "emits two flat, adjacent marks: - // `<4>read the <5>docs`" — the number comes from the fragment, not its position. it("serialize uses the numbers it was given, not the positions", () => { const marked = serializeInlineMarks([fragment(4, "read the "), fragment(5, "docs")]); @@ -80,8 +66,6 @@ describe("serializeInlineMarks", () => { describe("parseInlineMarks", () => { describe("reply order", () => { - // §1: sent `<1>a <2>red<3> car`, returned - // `<1>une <3>voiture <2>rouge` — "fragments come back in the reply's order". const frenchSent = [fragment(1, "a "), fragment(2, "red"), fragment(3, " car")]; const frenchReply = "<1>une <3>voiture <2>rouge"; @@ -103,7 +87,6 @@ describe("parseInlineMarks", () => { expect(acceptedFragments(result).map((f) => f.text)).toEqual(["une ", "voiture ", "rouge"]); }); - // Same principle, another permutation: a German subordinate clause sends the verb to the end. it("reorder: a German subordinate clause comes back in its own order", () => { const sent = [ fragment(1, "I know "), @@ -194,17 +177,15 @@ describe("parseInlineMarks", () => { expect(rejectionReason(result)).toBe("nested-marks"); }); - it("verdict is crossed-marks when a mark closes with another number", () => { + it("verdict is mismatched-close when a mark closes with another number", () => { const result = parseInlineMarks("<1>une voiture rouge", [ fragment(1, "a "), fragment(2, "red"), ]); - expect(rejectionReason(result)).toBe("crossed-marks"); + expect(rejectionReason(result)).toBe("mismatched-close"); }); - // Same row: "Unclosed ... marks | corrupt" → `unclosed-mark` - // ("A mark opened and never closed"). it("verdict is unclosed-mark when a mark is never closed", () => { const result = parseInlineMarks("<1>une <2>rouge", [ fragment(1, "a "), @@ -312,7 +293,6 @@ describe("parseInlineMarks", () => { expect(acceptedFragments(result)[0]?.text).toBe("une "); }); - // The rule fires only for a source that "began or ended with a space". it("edge whitespace: a fragment whose source had no edge space gains none", () => { const result = parseInlineMarks("<1>une<2>voiture", [ fragment(1, "a"), diff --git a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts index 65f761b13..e21ba32a9 100644 --- a/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts +++ b/packages/payload-plugin-translator/src/core/kernel/lexical/inlineMarks.ts @@ -1,26 +1,18 @@ /** - * Numbered inline marks: the wire format that lets a whole container be translated as one - * string while still saying which piece of it carried which formatting. + * Numbered inline marks: the wire format that lets a whole container be translated as one string + * while still saying which piece carried which formatting. * - * Knows nothing about Lexical — or about any document format. It reads and writes strings and - * numbered fragments, and that is deliberate: the mark format changes for different reasons - * than a tree walk does. - * - * Design: `docs/plans/2026-09-08-richtext-container-granularity-design.md` §4 (D2, D4, D13, D16). + * Format-agnostic on purpose — importing `./types` here would couple the wire format to the tree + * walk, and the two change for different reasons. */ -/** - * A fragment as the mark format sees it: a number, and the text it holds. - * - * `text: null` means the fragment carries no text at all — a line break, an inline block. It - * still needs a number, because it still occupies a position the reply may move it to. - */ +/** `text: null` — a line break or inline block. It is still numbered: it occupies a position the reply may move it to. */ export type MarkableFragment = { markId: number; text: string | null; }; -/** One fragment read back out of a reply. Text-free fragments come back with `text: ""`. */ +/** A fragment sent with `text: null` comes back with `text: ""`. */ export type ParsedMark = { markId: number; text: string; @@ -35,8 +27,7 @@ export type MarkFailure = | "unknown-mark" | "repeated-mark" | "nested-marks" - /** Closed with a different number than it opened with — not interleaving. */ - | "crossed-marks" + | "mismatched-close" | "unclosed-mark" | "no-text"; @@ -75,10 +66,8 @@ const tokenize = (reply: string): Token[] => { return tokens; }; -/** Restores an edge the model trimmed, using the source's own characters (a tab is not a space). */ +/** Re-inserts the source's own edge characters, not a plain space: a tab or NBSP was put there on purpose. */ const restoreEdges = (translated: string, source: string): string => { - if (!translated) return translated; - const leading = source.match(/^\s+/u)?.[0] ?? ""; const trailing = source.match(/\s+$/u)?.[0] ?? ""; const needsLeading = leading && translated === translated.trimStart(); @@ -90,12 +79,10 @@ const restoreEdges = (translated: string, source: string): string => { /** * Renders fragments as one marked string, in the order given. * - * Every fragment is wrapped, including ones carrying no formatting: the caller then never has - * to build a node from scratch on the way back. A text-free fragment renders self-closing. - * - * Mark numbers are written as given — they identify fragments and need not be contiguous. Text - * is written verbatim: a fragment whose own text looks like a mark is refused one level up, by - * the collector. An empty fragment list renders an empty string. + * Every fragment is wrapped, including unformatted ones, so the caller never has to build a node + * from scratch on the way back. Numbers are written as given — they identify fragments and need + * not be contiguous. Text is written verbatim: a fragment whose own text looks like a mark is + * refused one level up, by the collector. */ export function serializeInlineMarks(fragments: MarkableFragment[]): string { return fragments @@ -108,31 +95,21 @@ export function serializeInlineMarks(fragments: MarkableFragment[]): string { } /** - * Reads a reply back into fragments. - * - * On success fragments come back **in the reply's order** — that is the point of the format, as - * the target language decides where each piece belongs. A mark returned empty is legitimate and - * yields `text: ""`; its caller drops that node. + * Reads a reply back into fragments, **in the reply's order** — the target language decides where + * each piece belongs. A mark returned empty is legitimate and yields `text: ""`. * - * Accepted liberties: any order; whitespace inside a mark, newlines included (`< 1 >`); a - * leading zero; text sitting outside any mark, which is appended to the preceding mark (to the - * first when there is none) rather than discarded — a model that lost a boundary mid-sentence - * still returned the words. A fragment sent text-free that comes back carrying text has that - * text ignored: there is no leaf to write it into. + * Text outside any mark is appended to the preceding mark (to the first when there is none) + * rather than dropped: a model that lost a boundary mid-sentence still returned the words. * - * Edge whitespace is restored **only when the reply has the same shape as the request** — same - * order, no mark returned empty. Once pieces move or merge their edges change legitimately, and - * restoring one then produces a double space or a space trailing a paragraph. When it does - * apply, the source's own characters come back, not a plain space: a tab or a non-breaking - * space was put there on purpose. + * Edge whitespace is restored only when the reply has the same shape as the request — same order, + * no mark emptied. Once pieces move or merge their edges change legitimately, and restoring one + * then produces a double space or a space trailing a paragraph. * - * `no-text` is judged on non-blank content. When a reply is corrupt several ways at once, the - * reason reported is the first of: structural (`nested-marks`, `crossed-marks`, - * `unclosed-mark`), then the mark set (`unknown-mark`, `repeated-mark`, `missing-mark`), then - * `no-text` — structure first, because a string that cannot be parsed unambiguously has no - * reliable mark set to compare. + * When a reply is corrupt several ways at once the reason reported is the first of: structural + * (`nested-marks`, `mismatched-close`, `unclosed-mark`), then the mark set (`unknown-mark`, + * `repeated-mark`, `missing-mark`), then `no-text` — a string that cannot be parsed unambiguously + * has no reliable mark set to compare. * - * @param reply - the string a translator returned * @param fragments - the fragments that were sent: which numbers were issued, and their source text */ export function parseInlineMarks(reply: string, fragments: MarkableFragment[]): ParseResult { @@ -168,7 +145,7 @@ export function parseInlineMarks(reply: string, fragments: MarkableFragment[]): } if (openMarkId === null || openMarkId !== token.markId) { - return { ok: false, reason: "crossed-marks" }; + return { ok: false, reason: "mismatched-close" }; } collected.push({ markId: openMarkId, text: openText }); openMarkId = null;