rewrite api.py, update builder.py, add unit tests for them

Author: kkozik-amplify

hcl2/__init__.py

@@ -8,10 +8,21 @@
 from .api import (
     load,
     loads,
+    dump,
+    dumps,
     parse,
     parses,
+    parse_to_tree,
+    parses_to_tree,
+    from_dict,
+    from_json,
+    reconstruct,
     transform,
-    writes,
+    serialize,
 )
 
 from .builder import Builder
+from .deserializer import DeserializerOptions
+from .formatter import FormatterOptions
+from .rules.base import StartRule
+from .utils import SerializationOptions

hcl2/__main__.py

@@ -19,6 +19,7 @@
 from lark import UnexpectedCharacters, UnexpectedToken
 
 from . import load
+from .utils import SerializationOptions
 from .version import __version__
 
 
@@ -58,7 +59,8 @@ def main():
                 else open(args.OUT_PATH, "w", encoding="utf-8")
             )
             print(args.PATH, file=sys.stderr, flush=True)
-            json.dump(load(in_file, with_meta=args.with_meta), out_file)
+            options = SerializationOptions(with_meta=True) if args.with_meta else None
+            json.dump(load(in_file, serialization_options=options), out_file)
             if args.OUT_PATH is None:
                 out_file.write("\n")
                 out_file.close()

hcl2/api.py

@@ -1,60 +1,205 @@
-"""The API that will be exposed to users of this package"""
-from typing import TextIO
+"""The API that will be exposed to users of this package.
+
+Follows the json module convention: load/loads for reading, dump/dumps for writing.
+Also exposes intermediate pipeline stages for advanced usage.
+"""
+
+import json as _json
+from typing import TextIO, Optional
 
 from lark.tree import Tree
-from hcl2.parser import parser
+
+from hcl2.deserializer import BaseDeserializer, DeserializerOptions
+from hcl2.formatter import BaseFormatter, FormatterOptions
+from hcl2.parser import parser as _get_parser
 from hcl2.reconstructor import HCLReconstructor
+from hcl2.rules.base import StartRule
 from hcl2.transformer import RuleTransformer
+from hcl2.utils import SerializationOptions
+
+
+# ---------------------------------------------------------------------------
+# Primary API: load / loads / dump / dumps
+# ---------------------------------------------------------------------------
+
+
+def load(
+    file: TextIO,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Load a HCL2 file and return a Python dict.
+
+    :param file: File with HCL2 content.
+    :param serialization_options: Options controlling serialization behavior.
+    """
+    return loads(file.read(), serialization_options=serialization_options)
+
+
+def loads(
+    text: str,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Load HCL2 from a string and return a Python dict.
 
+    :param text: HCL2 text.
+    :param serialization_options: Options controlling serialization behavior.
+    """
+    tree = parses(text)
+    return serialize(tree, serialization_options=serialization_options)
+
+
+def dump(
+    data: dict,
+    file: TextIO,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+) -> None:
+    """Write a Python dict as HCL2 to a file.
 
-def load(file: TextIO, with_meta=False) -> dict:
-    """Load a HCL2 file.
-    :param file: File with hcl2 to be loaded as a dict.
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-    parameters to the output dict. Default to false.
+    :param data: Python dict (as produced by :func:`load`).
+    :param file: Writable text file.
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
     """
-    return loads(file.read(), with_meta=with_meta)
+    file.write(dumps(data, deserializer_options=deserializer_options, formatter_options=formatter_options))
 
 
-def loads(text: str, with_meta=False) -> dict:
-    """Load HCL2 from a string.
-    :param text: Text with hcl2 to be loaded as a dict.
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-    parameters to the output dict. Default to false.
+def dumps(
+    data: dict,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+) -> str:
+    """Convert a Python dict to an HCL2 string.
+
+    :param data: Python dict (as produced by :func:`load`).
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
     """
-    # append new line as a workaround for https://github.com/lark-parser/lark/issues/237
+    tree = from_dict(data, deserializer_options=deserializer_options, formatter_options=formatter_options)
+    return reconstruct(tree)
+
+
+# ---------------------------------------------------------------------------
+# Parsing: HCL text -> LarkElement tree or raw Lark tree
+# ---------------------------------------------------------------------------
+
+
+def parse(file: TextIO, *, discard_comments: bool = False) -> StartRule:
+    """Parse a HCL2 file into a LarkElement tree.
+
+    :param file: File with HCL2 content.
+    :param discard_comments: If True, discard comments during transformation.
+    """
+    return parses(file.read(), discard_comments=discard_comments)
+
+
+def parses(text: str, *, discard_comments: bool = False) -> StartRule:
+    """Parse a HCL2 string into a LarkElement tree.
+
+    :param text: HCL2 text.
+    :param discard_comments: If True, discard comments during transformation.
+    """
+    lark_tree = parses_to_tree(text)
+    return transform(lark_tree, discard_comments=discard_comments)
+
+
+def parse_to_tree(file: TextIO) -> Tree:
+    """Parse a HCL2 file into a raw Lark parse tree.
+
+    :param file: File with HCL2 content.
+    """
+    return parses_to_tree(file.read())
+
+
+def parses_to_tree(text: str) -> Tree:
+    """Parse a HCL2 string into a raw Lark parse tree.
+
+    :param text: HCL2 text.
+    """
+    # Append newline as workaround for https://github.com/lark-parser/lark/issues/237
     # Lark doesn't support EOF token so our grammar can't look for "new line or end of file"
-    # This means that all blocks must end in a new line even if the file ends
-    # Append a new line as a temporary fix
-    tree = parser().parse(text + "\n")
-    return RuleTransformer().transform(tree)
+    return _get_parser().parse(text + "\n")
 
 
-def parse(file: TextIO) -> Tree:
-    """Load HCL2 syntax tree from a file.
-    :param file: File with hcl2 to be loaded as a dict.
+# ---------------------------------------------------------------------------
+# Intermediate pipeline stages
+# ---------------------------------------------------------------------------
+
+
+def from_dict(
+    data: dict,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+    format: bool = True,
+) -> StartRule:
+    """Convert a Python dict into a LarkElement tree.
+
+    :param data: Python dict (as produced by :func:`load`).
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    :param format: If True (default), apply formatting to the tree.
+    """
+    deserializer = BaseDeserializer(deserializer_options)
+    tree = deserializer.load_python(data)
+    if format:
+        formatter = BaseFormatter(formatter_options)
+        formatter.format_tree(tree)
+    return tree
+
+
+def from_json(
+    text: str,
+    *,
+    deserializer_options: Optional[DeserializerOptions] = None,
+    formatter_options: Optional[FormatterOptions] = None,
+    format: bool = True,
+) -> StartRule:
+    """Convert a JSON string into a LarkElement tree.
+
+    :param text: JSON string.
+    :param deserializer_options: Options controlling deserialization behavior.
+    :param formatter_options: Options controlling formatting behavior.
+    :param format: If True (default), apply formatting to the tree.
     """
-    return parses(file.read())
+    data = _json.loads(text)
+    return from_dict(data, deserializer_options=deserializer_options, formatter_options=formatter_options, format=format)
+
 
+def reconstruct(tree) -> str:
+    """Convert a LarkElement tree (or raw Lark tree) to an HCL2 string.
 
-def parses(text: str) -> Tree:
-    """Load HCL2 syntax tree from a string.
-    :param text: Text with hcl2 to be loaded as a dict.
+    :param tree: A :class:`StartRule` (LarkElement tree) or :class:`lark.Tree`.
     """
-    return parser().parse(text)
+    reconstructor = HCLReconstructor()
+    if isinstance(tree, StartRule):
+        tree = tree.to_lark()
+    return reconstructor.reconstruct(tree)
 
 
-def transform(ast: Tree, with_meta=False) -> dict:
-    """Convert an HCL2 AST to a dictionary.
-    :param ast: HCL2 syntax tree, output from `parse` or `parses`
-    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
-        parameters to the output dict. Default to false.
+def transform(lark_tree: Tree, *, discard_comments: bool = False) -> StartRule:
+    """Transform a raw Lark parse tree into a LarkElement tree.
+
+    :param lark_tree: Raw Lark tree from :func:`parse_to_tree` or :func:`parse_string_to_tree`.
+    :param discard_comments: If True, discard comments during transformation.
     """
-    return RuleTransformer().transform(ast)
+    return RuleTransformer(discard_new_line_or_comments=discard_comments).transform(lark_tree)
+
 
+def serialize(
+    tree: StartRule,
+    *,
+    serialization_options: Optional[SerializationOptions] = None,
+) -> dict:
+    """Serialize a LarkElement tree to a Python dict.
 
-def writes(ast: Tree) -> str:
-    """Convert an HCL2 syntax tree to a string.
-    :param ast: HCL2 syntax tree, output from `parse` or `parses`
+    :param tree: A :class:`StartRule` (LarkElement tree).
+    :param serialization_options: Options controlling serialization behavior.
     """
-    return HCLReconstructor().reconstruct(ast)
+    if serialization_options is not None:
+        return tree.serialize(options=serialization_options)
+    return tree.serialize()

hcl2/builder.py

@@ -3,18 +3,16 @@
 
 from collections import defaultdict
 
-from hcl2.const import START_LINE_KEY, END_LINE_KEY
+from hcl2.const import IS_BLOCK
 
 
 class Builder:
     """
     The `hcl2.Builder` class produces a dictionary that should be identical to the
-    output of `hcl2.load(example_file, with_meta=True)`. The `with_meta` keyword
-    argument is important here. HCL "blocks" in the Python dictionary are
-    identified by the presence of `__start_line__` and `__end_line__` metadata
-    within them. The `Builder` class handles adding that metadata. If that metadata
-    is missing, the `hcl2.reconstructor.HCLReverseTransformer` class fails to
-    identify what is a block and what is just an attribute with an object value.
+    output of `hcl2.load(example_file)`. HCL "blocks" in the Python dictionary are
+    identified by the presence of `__is_block__: True` markers within them.
+    The `Builder` class handles adding that marker. If that marker is missing,
+    the deserializer fails to distinguish blocks from regular object attributes.
     """
 
     def __init__(self, attributes: Optional[dict] = None):
@@ -49,8 +47,7 @@ def build(self):
 
         body.update(
             {
-                START_LINE_KEY: -1,
-                END_LINE_KEY: -1,
+                IS_BLOCK: True,
                 **self.attributes,
             }
         )
@@ -79,7 +76,7 @@ def _add_nested_blocks(
         """Add nested blocks defined within another `Builder` instance to the `block` dictionary"""
         nested_block = nested_blocks_builder.build()
         for key, value in nested_block.items():
-            if key not in (START_LINE_KEY, END_LINE_KEY):
+            if key != IS_BLOCK:
                 if key not in block.keys():
                     block[key] = []
                 block[key].extend(value)

hcl2/deserializer.py

@@ -6,7 +6,7 @@
 
 from regex import regex
 
-from hcl2 import parses
+from hcl2.parser import parser as _get_parser
 from hcl2.const import IS_BLOCK
 from hcl2.rules.abstract import LarkElement, LarkRule
 from hcl2.rules.base import (
@@ -217,7 +217,7 @@ def _deserialize_expression(self, value: str) -> ExprTermRule:
         # create HCL2 snippet
         value = f"temp = {value}"
         # parse the above
-        parsed_tree = parses(value)
+        parsed_tree = _get_parser().parse(value)
         # transform parsed tree into LarkElement tree
         rules_tree = self._transformer.transform(parsed_tree)
         # extract expression from the tree