- support for standard template library

- '--full-plain' command line flag that returns full plain text source with all templates applied.

Author: dusano

file_utils.py

@@ -2,8 +2,10 @@
 
 import difflib
 
-from liquid import Environment, FileSystemLoader, StrictUndefined
-from liquid.exceptions import UndefinedError
+from liquid2 import Environment, FileSystemLoader, StrictUndefined
+from liquid2.exceptions import UndefinedError
+
+import plain_spec
 
 BINARY_FILE_EXTENSIONS = ['.pyc']
 
@@ -198,8 +200,8 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.loaded_templates = {}
 
-    def get_source(self, environment, template_name):
-        source = super().get_source(environment, template_name)
+    def get_source(self, environment, template_name, **kwargs):
+        source = super().get_source(environment, template_name, **kwargs)
         self.loaded_templates[template_name] = source.source
         return source
 
@@ -212,10 +214,13 @@ def get_loaded_templates(source_path, plain_source):
         loader=liquid_loader,
         undefined=StrictUndefined
     )
+
+    liquid_env.filters["code_variable"] = plain_spec.code_variable_liquid_filter
+
     plain_source_template = liquid_env.from_string(plain_source)
     try:
-        plain_source_template.render()
+        plain_source = plain_source_template.render()
     except UndefinedError as e:
         raise Exception(f"Undefined liquid variable: {str(e)}")
 
-    return liquid_loader.loaded_templates
+    return plain_source, liquid_loader.loaded_templates

plain2code.py

@@ -16,6 +16,7 @@
 DEFAULT_BUILD_FOLDER = 'build'
 DEFAULT_CONFORMANCE_TESTS_FOLDER = "conformance_tests"
 CONFORMANCE_TESTS_DEFINITION_FILE_NAME = "conformance_tests.json"
+DEFAULT_TEMPLATE_DIRS = "standard_template_library"
 
 MAX_UNITTEST_FIX_ATTEMPTS = 10
 MAX_CONFORMANCE_TEST_FIX_ATTEMPTS = 10
@@ -130,8 +131,8 @@ def run_unittests(args, codeplainAPI, frid, plain_source_tree, linked_resources,
     return existing_files, changed_files
 
 
-def generate_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, plain_source_tree, linked_resources, existing_files, conformance_tests_folder_name):
-    specifications = plain_spec.get_specifications_for_frid(plain_source_tree, functional_requirement_id)
+def generate_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, plain_source_tree, linked_resources, existing_files_content, conformance_tests_folder_name):
+    specifications, _ = plain_spec.get_specifications_for_frid(plain_source_tree, functional_requirement_id)
     if args.verbose:
         # TODO: Print the definitions.
         print(f"\nImplementing test requirements:")
@@ -156,8 +157,6 @@ def generate_conformance_tests(args, codeplainAPI, frid, functional_requirement_
 
     file_utils.delete_files_and_subfolders(conformance_tests_folder_name, args.verbose)
 
-    existing_files_content = file_utils.get_existing_files_content(args.build_folder, existing_files)
-
     response_files = codeplainAPI.render_conformance_tests(frid, functional_requirement_id, plain_source_tree, linked_resources, existing_files_content)
 
     file_utils.store_response_files(conformance_tests_folder_name, response_files, [])
@@ -195,7 +194,7 @@ def run_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, p
 
             print("Recreating conformance tests.")
 
-            generate_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, plain_source_tree, linked_resources, existing_files, conformance_tests_folder_name)
+            generate_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, plain_source_tree, linked_resources, existing_files_content, conformance_tests_folder_name)
 
             recreated_conformance_tests = True
             conformance_test_fix_count = 0
@@ -240,7 +239,7 @@ def run_conformance_tests(args, codeplainAPI, frid, functional_requirement_id, p
 
 def conformance_testing(args, codeplainAPI, frid, plain_source_tree, linked_resources, existing_files, conformance_tests):
     conformance_tests_run_count = 0
-    specifications = plain_spec.get_specifications_for_frid(plain_source_tree, frid)
+    specifications, _ = plain_spec.get_specifications_for_frid(plain_source_tree, frid)
     while conformance_tests_run_count < MAX_CONFORMANCE_TEST_RUNS:
         conformance_tests_run_count += 1
         implementation_code_has_changed = False
@@ -269,7 +268,7 @@ def conformance_testing(args, codeplainAPI, frid, plain_source_tree, linked_reso
                 else:
                     conformance_tests_folder_name = None
 
-                conformance_tests[frid] = generate_conformance_tests(args, codeplainAPI, frid, frid, plain_source_tree, linked_resources, existing_files, conformance_tests_folder_name)
+                conformance_tests[frid] = generate_conformance_tests(args, codeplainAPI, frid, frid, plain_source_tree, linked_resources, existing_files_content, conformance_tests_folder_name)
 
             conformance_tests_folder_name = conformance_tests[functional_requirement_id]['folder_name']
 
@@ -315,7 +314,7 @@ def render_functional_requirement(args, codeplainAPI, plain_source_tree, frid, a
 
         return
 
-    specifications = plain_spec.get_specifications_for_frid(plain_source_tree, frid)
+    specifications, _ = plain_spec.get_specifications_for_frid(plain_source_tree, frid)
 
     if args.verbose:
         print(f"\n-------------------------------------")
@@ -407,8 +406,13 @@ def render_functional_requirement(args, codeplainAPI, plain_source_tree, frid, a
     print('\n'.join(["- " + file_name for file_name in response_files.keys()]))
 
     [existing_files, tmp_changed_files] = run_unittests(args, codeplainAPI, frid, plain_source_tree, linked_resources, existing_files)
-    
-    changed_files.update(tmp_changed_files)
+
+    for file_name in tmp_changed_files:
+        if file_name not in existing_files:
+            if file_name in changed_files:
+                changed_files.remove(file_name)
+        else:
+            changed_files.add(file_name)
 
     num_refactoring_iterations = 0
     while num_refactoring_iterations < MAX_REFACTORING_ITERATIONS:
@@ -439,7 +443,13 @@ def render_functional_requirement(args, codeplainAPI, plain_source_tree, frid, a
         print('\n'.join(response_files.keys()))
 
         [existing_files, tmp_changed_files] = run_unittests(args, codeplainAPI, frid, plain_source_tree, linked_resources, existing_files)
-        changed_files.update(tmp_changed_files)
+
+        for file_name in tmp_changed_files:
+            if file_name not in existing_files:
+                if file_name in changed_files:
+                    changed_files.remove(file_name)
+            else:
+                changed_files.add(file_name)
 
     if args.conformance_tests_script and plain_spec.TEST_REQUIREMENTS in specifications and specifications[plain_spec.TEST_REQUIREMENTS]:
         conformance_tests = conformance_testing(args, codeplainAPI, frid, plain_source_tree, linked_resources, existing_files, conformance_tests)
@@ -490,7 +500,19 @@ def render(args):
     with open(args.filename, "r") as fin:
         plain_source = fin.read()
 
-    loaded_templates = file_utils.get_loaded_templates(os.path.dirname(args.filename), plain_source)
+    template_dirs = [
+        os.path.dirname(args.filename),
+        os.path.join(os.path.dirname( os.path.abspath(__file__)), DEFAULT_TEMPLATE_DIRS)
+    ]
+
+    [full_plain_source, loaded_templates] = file_utils.get_loaded_templates(template_dirs, plain_source)
+
+    if args.full_plain:
+        if args.verbose:
+            print("Full plain text:\n")
+
+        print(full_plain_source)
+        return
 
     codeplainAPI = codeplain_api.CodeplainAPI(args.api_key)
     codeplainAPI.debug = args.debug
@@ -531,6 +553,7 @@ def render(args):
     parser.add_argument('--conformance-tests-script', type=str, help='a script to run conformance tests')
     parser.add_argument('--api', type=str, nargs='?', const="https://api.codeplain.ai", help='force using the API (for internal use)')
     parser.add_argument('--api-key', type=str, default=CLAUDE_API_KEY, help='API key used to access the API. If not provided, the CLAUDE_API_KEY environment variable is used.')
+    parser.add_argument('--full-plain', action='store_true', help='full plain text to render')
 
     args = parser.parse_args()
 
@@ -543,7 +566,9 @@ def render(args):
         print(f"Running plain2code using REST API at {args.api }\n")
         import codeplain_REST_api as codeplain_api
     else:
-        print(f"Running plain2code using local API.\n")
+        if args.verbose or not args.full_plain:
+            print(f"Running plain2code using local API.\n")
+
         codeplain_api = importlib.import_module(codeplain_api_module_name)
 
     try:

plain_spec.py

@@ -1,4 +1,8 @@
 import copy
+import uuid
+import hashlib
+import json
+from liquid2.filter import with_context
 
 
 DEFINITIONS = 'Definitions:'
@@ -118,7 +122,23 @@ def get_previous_frid(plain_source_tree, frid):
     raise Exception(f"Functional requirement {frid} does not exist.")
 
 
-def get_specifications_from_plain_source_tree(frid, plain_source_tree, definitions, non_functional_requirements, test_requirements, functional_requirements, section_id=None):
+def get_specification_item_markdown(specification_item, code_variables, replace_code_variables):
+    markdown = specification_item['markdown']
+    if 'code_variables' in specification_item:
+        for code_variable in specification_item['code_variables']:
+            if code_variable['name'] in code_variables:
+                if code_variables[code_variable['name']] != code_variable['value']:
+                    raise Exception(f"Code variable {code_variable['name']} has multiple values: {code_variables[code_variable['name']]} and {code_variable['value']}")
+            else:
+                code_variables[code_variable['name']] = code_variable['value']
+
+            if replace_code_variables:
+                markdown = markdown.replace(f"{{{{ {code_variable['name']} }}}}", code_variable['value'])
+
+    return markdown
+
+
+def get_specifications_from_plain_source_tree(frid, plain_source_tree, definitions, non_functional_requirements, test_requirements, functional_requirements, code_variables, replace_code_variables, section_id=None):
     return_frid = None
     if FUNCTIONAL_REQUIREMENTS in plain_source_tree and len(plain_source_tree[FUNCTIONAL_REQUIREMENTS]) > 0:
         functional_requirement_count = 0
@@ -129,43 +149,77 @@ def get_specifications_from_plain_source_tree(frid, plain_source_tree, definitio
             else:
                 current_frid = section_id + "." + str(functional_requirement_count)
 
-            functional_requirements.append(functional_requirement['markdown'])
+            functional_requirements.append(get_specification_item_markdown(functional_requirement, code_variables, replace_code_variables))
 
             if current_frid == frid:
                 return_frid = current_frid
                 break
 
     if 'sections' in plain_source_tree:
         for section in plain_source_tree['sections']:
-            sub_frid = get_specifications_from_plain_source_tree(frid, section, definitions, non_functional_requirements, test_requirements, functional_requirements, section['ID'])
+            sub_frid = get_specifications_from_plain_source_tree(frid, section, definitions, non_functional_requirements, test_requirements, functional_requirements, code_variables, replace_code_variables, section['ID'])
             if sub_frid is not None:
                 return_frid = sub_frid
                 break
 
     if return_frid is not None:
         if DEFINITIONS in plain_source_tree and plain_source_tree[DEFINITIONS] is not None:
-            definitions[0:0] = [specification['markdown'] for specification in plain_source_tree[DEFINITIONS]]
+            definitions[0:0] = [get_specification_item_markdown(specification, code_variables, replace_code_variables) for specification in plain_source_tree[DEFINITIONS]]
         if NON_FUNCTIONAL_REQUIREMENTS in plain_source_tree and plain_source_tree[NON_FUNCTIONAL_REQUIREMENTS] is not None:
-            non_functional_requirements[0:0] = [specification['markdown'] for specification in plain_source_tree[NON_FUNCTIONAL_REQUIREMENTS]]
+            non_functional_requirements[0:0] = [get_specification_item_markdown(specification, code_variables, replace_code_variables) for specification in plain_source_tree[NON_FUNCTIONAL_REQUIREMENTS]]
         if TEST_REQUIREMENTS in plain_source_tree and plain_source_tree[TEST_REQUIREMENTS] is not None:
-            test_requirements[0:0] = [specification['markdown'] for specification in plain_source_tree[TEST_REQUIREMENTS]]
+            test_requirements[0:0] = [get_specification_item_markdown(specification, code_variables, replace_code_variables) for specification in plain_source_tree[TEST_REQUIREMENTS]]
 
     return return_frid
 
 
-def get_specifications_for_frid(plain_source_tree, frid):
+def get_specifications_for_frid(plain_source_tree, frid, replace_code_variables=True):
     definitions = []
     non_functional_requirements = []
     test_requirements = []
     functional_requirements = []
 
-    result = get_specifications_from_plain_source_tree(frid, plain_source_tree, definitions, non_functional_requirements, test_requirements, functional_requirements)
+    code_variables = {}
+
+    result = get_specifications_from_plain_source_tree(frid, plain_source_tree, definitions, non_functional_requirements, test_requirements, functional_requirements, code_variables, replace_code_variables)
     if result is None:
         raise Exception(f"Functional requirement {frid} does not exist.")
 
-    return {
+    specifications = {
         DEFINITIONS: definitions,
         NON_FUNCTIONAL_REQUIREMENTS: non_functional_requirements,
         TEST_REQUIREMENTS: test_requirements,
         FUNCTIONAL_REQUIREMENTS: functional_requirements
-    }
+    }
+
+    if code_variables:
+        return specifications, code_variables
+    else:
+        return specifications, None
+
+
+@with_context
+def code_variable_liquid_filter(value, *, context):
+    if len(context.scope) == 0:
+        raise Exception("Invalid use of code_variable filter!")
+
+    if 'code_variables' in context.globals:
+        code_variables = context.globals['code_variables']
+
+        variable = next(iter(context.scope.items()))
+
+        unique_str = uuid.uuid4().hex
+
+        code_variables[unique_str] = {variable[0]: value}
+
+        return unique_str
+    else:
+        return value
+
+
+def hash_text(text):
+    return hashlib.sha256(text.encode()).hexdigest()
+
+
+def get_hash_value(specifications):
+    return hash_text(json.dumps(specifications, indent=4))

requirements.txt

@@ -1,2 +1,2 @@
 requests
-python-liquid
+python-liquid2==0.3.0

standard_template_library/golang-console-app-template.plain

@@ -0,0 +1,30 @@
+***Definitions:***
+
+- The App is a console application.
+
+
+***Non-Functional Requirements:***
+
+- Implementation should be in Go lang.
+
+- The Implementation should include unit tests using Go's built-in testing package.
+
+- The main executable code file of The App should be called "{{ main_executable_file_name | code_variable }}". The main executable code file does not have any semantic meaning and it should be completely disregarded when generating the code (except for the file name).
+
+
+***Test Requirements:***
+
+- The Conformance Tests of The App should be implemented in Go lang using Go's built-in testing package.
+
+- The main executable code file of The App is "{{ main_executable_file_name | code_variable }}".
+
+- The current working directory contains the file "{{ main_executable_file_name | code_variable }}".
+
+- The App can be executed using the command "go run {{ main_executable_file_name | code_variable }}".
+
+
+# The Go lang console application boilerplate
+
+***Functional Requirements:***
+
+- Implement the entry point for The App.

standard_template_library/python-console-app-template.plain

@@ -0,0 +1,32 @@
+***Definitions:***
+
+- The App is a console application.
+
+
+***Non-Functional Requirements:***
+
+- The App should be implemented in Python (The Implementation).
+
+- The Implementation should include unit tests using Unittest framework (The Unittests)! If the unittests are put in the subfolder, make sure to include __init__.py to make them discoverable.
+
+- The main executable code file of The App should be called "{{ main_executable_file_name | code_variable }}". The main executable code file does not have any semantic meaning and it should be completely disregarded when generating the code (except for the file name).
+
+
+***Test Requirements:***
+
+- The Conformance Tests of The App should be implemented in Python using Unittest framework. The Conformance Tests will be run using "python -m unittest discover" command. Therefore, if the unittests are put in the subfolder, make sure to include __init__.py to make them discoverable. 
+
+- The Conformance Tests must be implemented and executed - do not use unittest.skip() or “any other test skipping functionality.
+
+- The main executable code file of The App is "{{ main_executable_file_name | code_variable }}".
+
+- The current working directory contains the file "{{ main_executable_file_name | code_variable }}".
+
+- The App can be executed using the command "python {{ main_executable_file_name | code_variable }}".
+
+
+# The Python console application boilerplate
+
+***Functional Requirements:***
+
+- Implement the entry point for The App.

standard_template_library/typescript-react-app-template.plain

@@ -0,0 +1,22 @@
+***Definitions:***
+
+- The App is a web application.
+
+
+***Non-Functional Requirements:***
+
+- The App should be implemented in TypeScript, using React as a web framework.
+
+- Please note that The App will be executed using npm.
+
+
+***Test Requirements:***
+
+- The Conformance Tests of The App should be written in TypeScript, using Cypress as the framework for The Conformance Tests.
+
+
+# The React boilerplate
+
+***Functional Requirements:***
+
+- Implement the entry point for The App.