diff --git a/.gitignore b/.gitignore index 1d4e738..4cc4eed 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,7 @@ dmypy.json # Pyre type checker .pyre/ -.idea/ \ No newline at end of file +.idea/ + +#DS Store Mac OS +.DS_Store \ No newline at end of file diff --git a/servicex_codegen/__init__.py b/servicex_codegen/__init__.py index 083859c..a0f731a 100644 --- a/servicex_codegen/__init__.py +++ b/servicex_codegen/__init__.py @@ -58,11 +58,11 @@ def create_app(test_config=None, provided_translator=None): if test_config: app.config.from_mapping(test_config) else: - if 'APP_CONFIG_FILE' in os.environ: - app.config.from_envvar('APP_CONFIG_FILE') if 'CODEGEN_CONFIG_FILE' in os.environ: app.config.from_envvar('CODEGEN_CONFIG_FILE') + app.config['TRANSFORMER_SCIENCE_IMAGE'] = os.environ.get('TRANSFORMER_SCIENCE_IMAGE') + with app.app_context(): translator = provided_translator diff --git a/servicex_codegen/post_operation.py b/servicex_codegen/post_operation.py index bdcc2f7..0d300dd 100644 --- a/servicex_codegen/post_operation.py +++ b/servicex_codegen/post_operation.py @@ -26,16 +26,17 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # - +import json # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # import os + import zipfile from tempfile import TemporaryDirectory -from flask import Response, current_app, request +from flask import Response, request, current_app from flask_restful import Resource from requests_toolbelt import MultipartEncoder @@ -84,6 +85,10 @@ def stream_generated_code(self, generated_code_result: GeneratedFileResult) -> b def post(self): try: + with open("transformer_capabilities.json") as capabilities_file: + capabilities = json.load(capabilities_file) + print("capable", capabilities['language'], capabilities['command']) + with TemporaryDirectory() as tempdir: body = request.get_json() generated_code_result = self.code_generator.generate_code( @@ -92,12 +97,14 @@ def post(self): zip_data = self.stream_generated_code(generated_code_result) # code gen transformer returns the default transformer image mentioned in # the config file - transformer_image = current_app.config.get("TRANSFORMER_SCIENCE_IMAGE") + transformer_image = current_app.config['TRANSFORMER_SCIENCE_IMAGE'] # MultipartEncoder library takes multiple types of data fields and merge # them into a multipart mime data type m = MultipartEncoder( fields={'transformer_image': transformer_image, + 'language': capabilities['language'], + 'command': capabilities['command'], 'zip_data': zip_data} ) diff --git a/tests/test_post_operation.py b/tests/test_post_operation.py index a868792..7db7b75 100644 --- a/tests/test_post_operation.py +++ b/tests/test_post_operation.py @@ -66,7 +66,8 @@ def test_post_good_query_with_params(self, mocker): ) config = { - 'TARGET_BACKEND': 'uproot' + 'TARGET_BACKEND': 'uproot', + 'TRANSFORMER_SCIENCE_IMAGE': "foo/bar:latest" } app = create_app(config, provided_translator=mock_ast_translator) client = app.test_client() @@ -85,7 +86,7 @@ def test_post_good_query_with_params(self, mocker): decoder_parts = decoder.MultipartDecoder(response.data, content_type) transformer_image = str(decoder_parts.parts[0].content, 'utf-8') - zip_file = decoder_parts.parts[1].content + zip_file = decoder_parts.parts[3].content print("Transformer Image: ", transformer_image) print("Zip File: ", zip_file) @@ -130,7 +131,7 @@ def test_post_good_query_without_params(self, mocker): decoder_parts = decoder.MultipartDecoder(response.data, content_type) transformer_image = str(decoder_parts.parts[0].content, 'utf-8') - zip_file = decoder_parts.parts[1].content + zip_file = decoder_parts.parts[3].content print("Transformer Image: ", transformer_image) print("Zip File: ", zip_file) diff --git a/transformer_capabilities.json b/transformer_capabilities.json new file mode 100644 index 0000000..10ff7c4 --- /dev/null +++ b/transformer_capabilities.json @@ -0,0 +1,9 @@ +{ + "name": "FuncADL based uproot transformer", + "description": "Extracts data from flat ntuple style root files.", + "limitations": "Would be good to note what isn't implemented", + "file-formats": ["parquet"], + "stats-parser": "UprootStats", + "language": "python", + "command": "/generated/transform_single_file.py" +} \ No newline at end of file