diff --git a/src/pdf_form_doc.py b/src/pdf_form_doc.py index 8b8e210b8..4bc1666cc 100644 --- a/src/pdf_form_doc.py +++ b/src/pdf_form_doc.py @@ -102,8 +102,10 @@ _PLAIN_FRONT_MATTER_RE = re.compile( # - **label:** value # - **label** [opts]: value # - [x] **label** +# Accept Unicode word characters for older saved markers, whose encoder left +# non-ASCII letters/numbers literal. Newly rendered names are ASCII-safe. _FIELD_BULLET_RE = re.compile( - r'^\s*-\s+(?P.*?)\s*\s*$' + r'^\s*-\s+(?P.*?)\s*\s*$' ) @@ -115,7 +117,7 @@ def _encode_name(name: str) -> str: """ out = [] for ch in name or "": - if ch.isalnum() or ch in ("_", ".", "-"): + if (ch.isascii() and ch.isalnum()) or ch in ("_", ".", "-"): out.append(ch) else: for b in ch.encode("utf-8"): diff --git a/tests/test_form_markdown_roundtrip.py b/tests/test_form_markdown_roundtrip.py index 94d4ae518..973f081ad 100644 --- a/tests/test_form_markdown_roundtrip.py +++ b/tests/test_form_markdown_roundtrip.py @@ -8,6 +8,8 @@ label with `[^*]+`, so they could not match a label like "Email *" / "State *" stayed empty and the exported PDF (and signed-reply attachment) came out blank for that field, with no error. """ +import pytest + from src.pdf_form_doc import render_form_as_markdown, parse_markdown_to_values @@ -38,3 +40,38 @@ def test_plain_labels_and_colon_values_unaffected(): vals = parse_markdown_to_values(md) assert vals["name"] == "Alice" assert vals["time"] == "9:00 sharp" + + +@pytest.mark.parametrize("name", ["Prénom", "Año", "İsim", "氏名", "اسم", "n²", "Cafe\u0301"]) +@pytest.mark.parametrize("field_type, value", [ + ("text", "Alice"), ("checkbox", True), ("choice", "Yes"), +]) +def test_unicode_field_names_survive_export_roundtrip(name, field_type, value): + fields = [{"name": name, "label": "Field *", "type": field_type, + "value": value, "options": ["Yes", "No"], "page": 1}] + markdown = render_form_as_markdown(fields, "upload", "Form") + marker_name = markdown.split("\n" + "- [x] **Consent** \n" + "- **Year** [2025 / 2026]: 2026 \n" + ) + assert parse_markdown_to_values(markdown) == { + "Prénom": "Alice", "同意": True, "Año": "2026", + } + + +def test_unicode_names_do_not_collide_with_literal_percent_or_ascii_names(): + names = ["Prénom", "Pr%C3%A9nom", "Prenom", "Section.1_name-2"] + fields = [{"name": name, "type": "text", "value": str(i), "page": 1} + for i, name in enumerate(names)] + markdown = render_form_as_markdown(fields, "upload", "Form") + assert "field=Section.1_name-2 " in markdown + assert parse_markdown_to_values(markdown) == {name: str(i) for i, name in enumerate(names)}