docs: add example translating Mustache templates

Author: daniel-jones-dev

.gitlab-ci.yml

@@ -119,6 +119,18 @@ test:
         - test_report.xml
     when: always
 
+mustache example:
+  stage: test
+  extends: .test
+  script:
+    - cd examples/mustache
+    - pip install deepl
+    - python . --help
+    - python . --from en --to de "Hello {{user}}, your balance is {{{balance}}} dollars." > mustache_result.txt
+    - cat mustache_result.txt
+    - grep -q "{{user}}" mustache_result.txt
+    - grep -q "{{{balance}}}" mustache_result.txt
+
 # stage: publish -------------------------
 
 pypi upload:

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+* Add [example script](examples/mustache) to translate Mustache templates.
 ### Changed
 ### Deprecated
 ### Removed

examples/mustache/README.md

@@ -0,0 +1,72 @@
+# Example: Translation of Mustache templates
+
+An example showing how to translate [Mustache templates][mustache-docs] using
+XML tag-handling. Mustache templates embed tags in literal text and HTML tags;
+the tags refer to keys in hashes, so they should not be translated.
+
+For example:
+
+```
+Hello {{name}}. You have just won <b>{{value}} dollars</b>!
+```
+
+could be translated into German as:
+
+```
+Hallo {{name}}. Sie haben gerade <b>{{value}} Dollar</b> gewonnen!
+```
+
+## Usage
+
+Install the [`deepl` Python library](../../README.md).
+
+Define your DeepL auth key as an environment variable `DEEPL_AUTH_KEY`.
+
+```
+export DEEPL_AUTH_KEY=f63c02c5-f056-...
+```
+
+Run the Mustache translator by running Python on this directory:
+
+```
+python examples/mustache --to de "Hello {{name}}"
+```
+
+For an explanation of the command line arguments, provide the `--help` option:
+
+```
+python examples/mustache --help
+```
+
+## How it works
+
+This Mustache translator uses XML tag-handling to preserve the Mustache tags 
+while using the DeepL API to translate. It also handles HTML tags embedded in
+the Mustache template.
+
+1. The input Mustache template is parsed to separate the literal text from the
+   Mustache tags.
+   ```
+   Hello {{name}}. You have just won <b>{{value}} dollars</b>!
+         ^^^^^^^^                       ^^^^^^^^^
+   ```
+2. The template is modified to replace all Mustache tags with placeholder XML
+   tags. Unique IDs are attached to each placeholder tag to identify them in the
+   translated XML.
+   ```
+   Hello <m id=0 />. You have just won <b><m id=1 /> dollars</b>!
+         ^^^^^^^^^^                       ^^^^^^^^^^
+   ```
+3. The XML template is translated using DeepL API.
+   ```
+   Hallo <m id=0 />. Sie haben gerade <b><m id=1 /> Dollar</b> gewonnen!
+   ```
+4. The translated XML is parsed to identify placeholder tags and replace them 
+   with the original Mustache tags.
+   ```
+   Hallo {{name}}. Sie haben gerade <b>{{value}} Dollar</b> gewonnen!
+         ^^^^^^^^                      ^^^^^^^^^
+   ```
+   
+
+[mustache-docs]: https://mustache.github.io/mustache.5.html

examples/mustache/__main__.py

@@ -0,0 +1,91 @@
+# Copyright 2023 DeepL SE (https://www.deepl.com)
+# Use of this source code is governed by an MIT
+# license that can be found in the LICENSE file.
+
+import argparse
+import deepl
+import os
+
+from mustache import translate_mustache
+
+
+env_auth_key = "DEEPL_AUTH_KEY"
+env_server_url = "DEEPL_SERVER_URL"
+
+
+def get_parser(prog_name):
+    """Constructs and returns the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog=prog_name,
+        description="Translate Mustache templates using the DeepL API "
+        "(https://www.deepl.com/docs-api).",
+        epilog="If you encounter issues while using this example, please "
+        "report them at https://github.com/DeepLcom/deepl-python/issues",
+    )
+
+    parser.add_argument(
+        "--auth-key",
+        default=None,
+        help="authentication key as given in your DeepL account; the "
+        f"{env_auth_key} environment variable is used as secondary fallback",
+    )
+    parser.add_argument(
+        "--server-url",
+        default=None,
+        metavar="URL",
+        help=f"alternative server URL for testing; the {env_server_url} "
+        f"environment variable may be used as secondary fallback",
+    )
+    parser.add_argument(
+        "--to",
+        "--target-lang",
+        dest="target_lang",
+        required=True,
+        help="language into which the template should be translated",
+    )
+    parser.add_argument(
+        "--from",
+        "--source-lang",
+        dest="source_lang",
+        help="language of the template to be translated",
+    )
+    parser.add_argument(
+        "template",
+        nargs="+",
+        type=str,
+        help="template to be translated. Wrap template in quotes to prevent "
+        "the shell from splitting on whitespace.",
+    )
+
+    return parser
+
+
+def main():
+    # Create a parser, reusing most of the arguments from the main CLI
+    parser = get_parser(prog_name=None)
+    args = parser.parse_args()
+    auth_key = args.auth_key or os.getenv(env_auth_key)
+    server_url = args.server_url or os.getenv(env_server_url)
+    if auth_key is None:
+        raise Exception(
+            f"Please provide authentication key via the {env_auth_key} "
+            "environment variable or --auth_key argument"
+        )
+
+    # Create a Translator object, and call get_usage() to validate connection
+    translator = deepl.Translator(auth_key, server_url=server_url)
+    translator.get_usage()
+
+    for template in args.template:
+        # Call translate_mustache() to translate the Mustache template
+        output = translate_mustache(
+            template,
+            translator=translator,
+            source_lang=args.source_lang,
+            target_lang=args.target_lang,
+        )
+        print(output)
+
+
+if __name__ == "__main__":
+    main()

examples/mustache/html_parsing.py

@@ -0,0 +1,96 @@
+# Copyright 2023 DeepL SE (https://www.deepl.com)
+# Use of this source code is governed by an MIT
+# license that can be found in the LICENSE file.
+
+from html.parser import HTMLParser
+import html
+from typing import List, Tuple, Dict
+
+
+class PassthroughHTMLParser(HTMLParser):
+    """
+    An HTML parser that accumulates parsed HTML in the original HTML.
+    The original HTML can be accessed using parsed().
+
+    Note: some HTML features are not correctly handled and reproduced: entity
+    references, character references, declarations, processing instructions.
+    """
+
+    def __init__(self):
+        super().__init__(convert_charrefs=False)
+        self._result = ""
+
+    def parsed(self):
+        return self._result
+
+    def _append(self, text):
+        self._result += text
+
+    def handle_startendtag(self, tag, attrs):
+        self._append(f"<{tag}{self._encode_attrs(attrs)} />")
+
+    def handle_starttag(self, tag, attrs):
+        self._append(f"<{tag}{self._encode_attrs(attrs)}>")
+
+    def handle_endtag(self, tag):
+        self._append(f"</{tag}>")
+
+    def handle_charref(self, name):
+        self._append(f"&#{name};")
+
+    def handle_entityref(self, name):
+        self._append(f"&{name};")
+
+    def handle_data(self, data):
+        self._append(data)
+
+    def handle_comment(self, data):
+        self._append(f"<!--{data}-->")
+
+    def handle_decl(self, decl):
+        self._append(f"<!{decl}>")
+
+    def handle_pi(self, data):
+        self._append(f"<?{data}>")
+
+    @staticmethod
+    def _encode_attrs(attrs: List[Tuple[str, str]]) -> str:
+        if attrs:
+            return " " + " ".join(
+                f'{key}="{html.escape(value)}"' for key, value in attrs
+            )
+        else:
+            return ""
+
+
+class TagReplacerHTMLParser(PassthroughHTMLParser):
+    """
+    An HTML parser that accumulates parsed HTML unmodified, except that
+    matching start-end tags are replaced by prepared content.
+
+    :param replace_tags: A dictionary of tags to replace, with keys matching
+        tag names ("tag") or tag names with IDs ("tag#id"), and values the text
+        to replace matching tags with.
+    """
+
+    def __init__(self, replace_tags: Dict[str, str]):
+        super().__init__()
+        self._replace_tags = replace_tags
+
+    def handle_startendtag(self, tag, attrs):
+        if self._replace_tag_if_matching(tag):
+            return
+
+        id = next((value for key, value in attrs if key == "id"), None)
+        if id is not None and self._replace_tag_if_matching(f"{tag}#{id}"):
+            return
+
+        # If no replacement is found, fall back to passthrough parser
+        super().handle_starttag(tag, attrs)
+
+    def _replace_tag_if_matching(self, tag_and_id):
+        if tag_and_id in self._replace_tags:
+            stored_tag = self._replace_tags[tag_and_id]
+            self._append(stored_tag)
+            return True
+        return False