fix(forms): preserve Unicode field names during PDF export

src/pdf_form_doc.py

@@ -102,8 +102,10 @@ _PLAIN_FRONT_MATTER_RE = re.compile(
 #   - **label:** value <!-- field=NAME-ENC type=text -->
 #   - **label** [opts]: value <!-- field=NAME-ENC type=choice -->
 #   - [x] **label** <!-- field=NAME-ENC type=checkbox -->
+# Accept Unicode word characters for older saved markers, whose encoder left
+# non-ASCII letters/numbers literal. Newly rendered names are ASCII-safe.
 _FIELD_BULLET_RE = re.compile(
-    r'^\s*-\s+(?P<body>.*?)\s*<!--\s*field=(?P<name>[A-Za-z0-9_.%-]+)\s+type=(?P<type>\w+)\s*-->\s*$'
+    r'^\s*-\s+(?P<body>.*?)\s*<!--\s*field=(?P<name>[\w.%-]+)\s+type=(?P<type>\w+)\s*-->\s*$'
 )
 
 
@@ -115,7 +117,7 @@ def _encode_name(name: str) -> str:
     """
     out = []
     for ch in name or "":
-        if ch.isalnum() or ch in ("_", ".", "-"):
+        if (ch.isascii() and ch.isalnum()) or ch in ("_", ".", "-"):
             out.append(ch)
         else:
             for b in ch.encode("utf-8"):

tests/test_form_markdown_roundtrip.py

@@ -8,6 +8,8 @@ label with `[^*]+`, so they could not match a label like "Email *" / "State *"
 stayed empty and the exported PDF (and signed-reply attachment) came out blank
 for that field, with no error.
 """
+import pytest
+
 from src.pdf_form_doc import render_form_as_markdown, parse_markdown_to_values
 
 
@@ -38,3 +40,38 @@ def test_plain_labels_and_colon_values_unaffected():
     vals = parse_markdown_to_values(md)
     assert vals["name"] == "Alice"
     assert vals["time"] == "9:00 sharp"
+
+
+@pytest.mark.parametrize("name", ["Prénom", "Año", "İsim", "氏名", "اسم", "n²", "Cafe\u0301"])
+@pytest.mark.parametrize("field_type, value", [
+    ("text", "Alice"), ("checkbox", True), ("choice", "Yes"),
+])
+def test_unicode_field_names_survive_export_roundtrip(name, field_type, value):
+    fields = [{"name": name, "label": "Field *", "type": field_type,
+               "value": value, "options": ["Yes", "No"], "page": 1}]
+    markdown = render_form_as_markdown(fields, "upload", "Form")
+    marker_name = markdown.split("<!-- field=", 1)[1].split(" type=", 1)[0]
+    assert marker_name.isascii()
+    assert parse_markdown_to_values(markdown) == {name: value}
+
+
+def test_previously_saved_unicode_markers_remain_exportable():
+    # Older imports emitted Unicode letters literally. Their persisted
+    # markdown must also work after upgrading, without re-importing the PDF.
+    markdown = (
+        "- **First name:** Alice <!-- field=Prénom type=text -->\n"
+        "- [x] **Consent** <!-- field=同意 type=checkbox -->\n"
+        "- **Year** [2025 / 2026]: 2026 <!-- field=Año type=choice -->\n"
+    )
+    assert parse_markdown_to_values(markdown) == {
+        "Prénom": "Alice", "同意": True, "Año": "2026",
+    }
+
+
+def test_unicode_names_do_not_collide_with_literal_percent_or_ascii_names():
+    names = ["Prénom", "Pr%C3%A9nom", "Prenom", "Section.1_name-2"]
+    fields = [{"name": name, "type": "text", "value": str(i), "page": 1}
+              for i, name in enumerate(names)]
+    markdown = render_form_as_markdown(fields, "upload", "Form")
+    assert "field=Section.1_name-2 " in markdown
+    assert parse_markdown_to_values(markdown) == {name: str(i) for i, name in enumerate(names)}