Merge pull request #53 from erezsh/lark_cache

Use Lark with its cache feature, instead of creating a standalone parser

Author: htorianik

.gitignore

@@ -120,3 +120,4 @@ node_modules/
 
 # Don't commit the generated parser
 lark_parser.py
+.lark_cache.bin

hcl2/api.py

@@ -15,4 +15,7 @@ def loads(text: str) -> dict:
     # Lark doesn't support a EOF token so our grammar can't look for "new line or end of file"
     # This means that all blocks must end in a new line even if the file ends
     # Append a new line as a temporary fix
-    return hcl2.parse(text + "\n")
+    # Ignoring type as the type-annotation of Lark.parse() claims that it always returns a Tree,
+    # but in the docs of the parse() said that it returns whatever the supplied transformer returns.
+    # We supply DictTransformer so the return type is Dict.
+    return hcl2.parse(text + "\n")  # type: ignore

hcl2/parser.py

@@ -1,42 +1,18 @@
 """A parser for HCL2 implemented using the Lark parser"""
 from pathlib import Path
 
-from hcl2.transformer import DictTransformer
-
-THIS_DIR = Path(__file__).absolute().resolve().parent
-PARSER_FILE = THIS_DIR / "lark_parser.py"
-
+from lark import Lark
 
-def create_parser_file(parser_file: Path = PARSER_FILE) -> None:
-    """
-    Parsing the Lark grammar takes about 0.5 seconds. In order to improve performance we can cache the parser
-    file. The below code caches the entire python file which is generated by Lark's standalone parser feature
-    See: https://github.com/lark-parser/lark/blob/master/lark/tools/standalone.py
-
-    Lark also supports serializing the parser config but the deserialize function did not work for me.
-    The lark state contains dicts with numbers as keys which is not supported by json so the serialized
-    state can't be written to a json file. Exporting to other file types would have required
-    adding additional dependencies or writing a lot more code. Lark's standalone parser
-    feature works great but it expects to be run as a separate shell command
-    The below code copies some of the standalone parser generator code in a way that we can use
-    """
-    # This function is needed only if standalone parser is not yet compiled.
-    # pylint: disable=import-outside-toplevel
-    from lark import Lark
-    from lark.tools.standalone import gen_standalone
-
-    lark_file = THIS_DIR / "hcl2.lark"
-    with open(parser_file, "w", encoding="utf-8") as parser_file_stream:
-        lark_inst = Lark(lark_file.read_text(), parser="lalr", lexer="contextual")
-        parser_file_stream.write("# mypy: ignore-errors\n")
-        gen_standalone(lark_inst, out=parser_file_stream)
+from hcl2.transformer import DictTransformer
 
 
-if not PARSER_FILE.exists():
-    create_parser_file(PARSER_FILE)
+PARSER_FILE = Path(__file__).absolute().resolve().parent / ".lark_cache.bin"
 
-# pylint: disable=wrong-import-position
-# Lark_StandAlone needs to be imported after the above block of code because lark_parser.py might not exist
-from hcl2.lark_parser import Lark_StandAlone
 
-hcl2 = Lark_StandAlone(transformer=DictTransformer())
+hcl2 = Lark.open(
+    "hcl2.lark",
+    parser="lalr",
+    cache=str(PARSER_FILE),  # Disable/Delete file to effect changes to the grammar
+    rel_to=__file__,
+    transformer=DictTransformer(),
+)

test/unit/test_load.py

@@ -1,10 +1,11 @@
 """ Test parsing a variety of hcl files"""
+
 import json
 from pathlib import Path
 from unittest import TestCase
 
+from hcl2.parser import PARSER_FILE
 import hcl2
-from hcl2.parser import PARSER_FILE, create_parser_file
 
 
 HELPERS_DIR = Path(__file__).absolute().parent.parent / "helpers"
@@ -18,9 +19,8 @@ class TestLoad(TestCase):
 
     def test_load_terraform(self):
         """Test parsing a set of hcl2 files and force recreating the parser file"""
-        parser_file = Path(hcl2.__file__).absolute().parent / PARSER_FILE
-        parser_file.unlink()
-        create_parser_file()
+        # delete the parser file to force it to be recreated
+        PARSER_FILE.unlink()
         for hcl_path in HCL2_FILES:
             yield self.check_terraform, hcl_path