diff --git a/Dockerfile b/Dockerfile index ae0b52d76ef..d76cb9b868b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,7 +96,7 @@ ENV CLASSPATH /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-j COPY docker/logging.properties /usr/local/tomcat/conf/logging.properties RUN sed -i -e 's/Valve/Disabled/' /usr/local/tomcat/conf/server.xml -# add our scripts +# add our scripts and configuration COPY docker /scripts RUN chmod -R +x /scripts diff --git a/docker/start.py b/docker/start.py index 7c484412b23..75a76101cea 100755 --- a/docker/start.py +++ b/docker/start.py @@ -76,6 +76,7 @@ OPENGROK_JAR = os.path.join(OPENGROK_LIB_DIR, 'opengrok.jar') NOMIRROR_ENV_NAME = 'NOMIRROR' +API_TIMEOUT_ENV_NAME = 'API_TIMEOUT' expected_token = None periodic_timer = None @@ -481,16 +482,16 @@ def main(): setup_redirect_source(logger, url_root) api_timeout = 8 - if os.environ.get('API_TIMEOUT'): - api_timeout = int(os.environ.get('API_TIMEOUT')) - extra_indexer_options = "--connectTimeout " + str(api_timeout) + if os.environ.get(API_TIMEOUT_ENV_NAME): + api_timeout = int(os.environ.get(API_TIMEOUT_ENV_NAME)) + else: + os.environ[API_TIMEOUT_ENV_NAME] = str(api_timeout) env = {} - indexer_opt = os.environ.get('INDEXER_OPT', '') - if indexer_opt: - extra_indexer_options += " " + indexer_opt - logger.info("extra indexer options: '{}'".format(extra_indexer_options)) - env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = extra_indexer_options + extra_indexer_options = os.environ.get('INDEXER_OPT', '') + if extra_indexer_options: + logger.info("extra indexer options: '{}'".format(extra_indexer_options)) + env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = extra_indexer_options if os.environ.get(NOMIRROR_ENV_NAME): env[OPENGROK_NO_MIRROR_ENV] = os.environ.get(NOMIRROR_ENV_NAME) diff --git a/docker/sync.yml b/docker/sync.yml index 2cfba140a92..f2eae94dcff 100644 --- a/docker/sync.yml +++ b/docker/sync.yml @@ -7,12 +7,17 @@ commands: duration: PT1H tags: ['%PROJECT%'] text: resync + reindex in progress -- command: [opengrok-mirror, -c, '/opengrok/etc/mirror.yml', -I, -U, '%URL%', '%PROJECT%'] -- command: [opengrok-reindex-project, --printoutput, - --jar, /opengrok/lib/opengrok.jar, -U, '%URL%', -P, '%PROJECT%', --, - -r, dirbased, -G, -m, '256', --leadingWildCards, 'on', - -c, /usr/local/bin/ctags, -U, '%URL%', -H, '%PROJECT%'] - limits: {RLIMIT_NOFILE: 1024} +- command: + args: [opengrok-mirror, -c, '/opengrok/etc/mirror.yml', -I, -U, '%URL%', '%PROJECT%'] +- command: + args: [opengrok-reindex-project, --printoutput, + --api_timeout, '%API_TIMEOUT%', + --jar, /opengrok/lib/opengrok.jar, -U, '%URL%', -P, '%PROJECT%', --, + --connectTimeout, '%API_TIMEOUT%', + -r, dirbased, -G, -m, '256', --leadingWildCards, 'on', + -c, /usr/local/bin/ctags, -U, '%URL%', -H, '%PROJECT%'] + limits: {RLIMIT_NOFILE: 1024} + args_subst: {"%API_TIMEOUT%": "$API_TIMEOUT"} - call: uri: '%URL%api/v1/messages?tag=%PROJECT%' method: DELETE diff --git a/tools/src/main/python/opengrok_tools/sync.py b/tools/src/main/python/opengrok_tools/sync.py index b5fc0aa09ee..9106aa730d2 100755 --- a/tools/src/main/python/opengrok_tools/sync.py +++ b/tools/src/main/python/opengrok_tools/sync.py @@ -19,7 +19,7 @@ # # -# Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. # """ diff --git a/tools/src/main/python/opengrok_tools/utils/command.py b/tools/src/main/python/opengrok_tools/utils/command.py index 9ab7987d54c..1351664c171 100644 --- a/tools/src/main/python/opengrok_tools/utils/command.py +++ b/tools/src/main/python/opengrok_tools/utils/command.py @@ -18,7 +18,7 @@ # # -# Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. # import logging @@ -297,7 +297,7 @@ def close(self): finally: if self.timeout != 0 and timeout_thread: with time_condition: - time_condition.notifyAll() + time_condition.notify_all() # The subprocess module does not close the write pipe descriptor # it fetched via OutputThread's fileno() so in order to gracefully @@ -344,10 +344,14 @@ def fill_arg(self, args_append=None, args_subst=None): newarg = cmdarg for pattern in args_subst.keys(): if pattern in newarg and args_subst[pattern]: + value = args_subst[pattern] + if value.startswith("$"): + self.logger.debug(f"treating {value} as environment variable") + value = os.environ.get(value[1:], "") self.logger.debug("replacing '{}' in '{}' with '{}'". format(pattern, newarg, - args_subst[pattern])) - newarg = newarg.replace(pattern, args_subst[pattern]) + value)) + newarg = newarg.replace(pattern, value) self.logger.debug("replaced argument with {}". format(newarg)) subst_done = i diff --git a/tools/src/main/python/opengrok_tools/utils/commandsequence.py b/tools/src/main/python/opengrok_tools/utils/commandsequence.py index d456b194b48..66250d21876 100644 --- a/tools/src/main/python/opengrok_tools/utils/commandsequence.py +++ b/tools/src/main/python/opengrok_tools/utils/commandsequence.py @@ -18,7 +18,7 @@ # # -# Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. # import logging @@ -41,6 +41,10 @@ HEADERS_PROPERTY = "headers" METHOD_PROPERTY = "method" URI_PROPERTY = "uri" +ARGS_SUBST_PROPERTY = "args_subst" +LIMITS_PROPERTY = "limits" +ENV_PROPERTY = "env" +ARGS_PROPERTY = "args" class CommandConfigurationException(Exception): @@ -97,8 +101,8 @@ def check_command_property(command): if command_value is None and call_value is None: raise CommandConfigurationException(f"command dictionary has unknown key: {command}") - if command_value and not isinstance(command_value, list): - raise CommandConfigurationException("command value not a list: {}". + if command_value and not isinstance(command_value, dict): + raise CommandConfigurationException("command value not a dictionary: {}". format(command_value)) if call_value: check_call_config(call_value) @@ -179,6 +183,9 @@ def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None, self.url = url + self.args_subst = {PROJECT_SUBST: self.name, + URL_SUBST: self.url} + def __str__(self): return str(self.name) @@ -245,8 +252,7 @@ def run(self): if command.get(CALL_PROPERTY): try: call_rest_api(ApiCall(command.get(CALL_PROPERTY)), - {PROJECT_SUBST: self.name, - URL_SUBST: self.url}, + self.args_subst, self.http_headers, self.api_timeout, self.async_api_timeout) @@ -258,13 +264,16 @@ def run(self): break elif command.get(COMMAND_PROPERTY): - command_args = command.get(COMMAND_PROPERTY) + command = command.get(COMMAND_PROPERTY) + command_args = command.get(ARGS_PROPERTY) + args_subst = self.args_subst + if command.get(ARGS_SUBST_PROPERTY): + args_subst.update(command.get(ARGS_SUBST_PROPERTY)) command = Command(command_args, - env_vars=command.get("env"), + env_vars=command.get(ENV_PROPERTY), logger=self.logger, - resource_limits=command.get("limits"), - args_subst={PROJECT_SUBST: self.name, - URL_SUBST: self.url}, + resource_limits=command.get(LIMITS_PROPERTY), + args_subst=args_subst, args_append=[self.name], excl_subst=True) ret_code = self.run_command(command) @@ -316,13 +325,13 @@ def run_cleanup(self): self.logger.error("API call {} failed: {}". format(cleanup_cmd, e)) elif cleanup_cmd.get(COMMAND_PROPERTY): - command_args = cleanup_cmd.get(COMMAND_PROPERTY) + cleanup_cmd = cleanup_cmd.get(COMMAND_PROPERTY) + command_args = cleanup_cmd.get(ARGS_PROPERTY) self.logger.debug("Running cleanup command '{}'". format(command_args)) cmd = Command(command_args, logger=self.logger, - args_subst={PROJECT_SUBST: self.name, - URL_SUBST: self.url}, + args_subst=self.args_subst, args_append=[self.name], excl_subst=True) cmd.execute() if cmd.getretcode() != SUCCESS_EXITVAL: diff --git a/tools/src/test/python/test_command.py b/tools/src/test/python/test_command.py index 69fa457dd10..93c51c60f50 100755 --- a/tools/src/test/python/test_command.py +++ b/tools/src/test/python/test_command.py @@ -20,7 +20,7 @@ # # -# Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. # Portions Copyright (c) 2020, Krystof Tulinger # @@ -75,6 +75,17 @@ def test_subst_multiple(): assert cmd.cmd == ['foo', 'aaablahbbbxyzccc', 'bar'] +def test_subst_env(): + """ + Test substitution with value treated as environment variable. + """ + os.environ["FOO"] = "foo" + cmd = Command(['foo', 'aaa%ARG%bbb', 'bar'], + args_subst={"%ARG%": "$FOO"}) + assert cmd.cmd == ['foo', 'aaafoobbb', 'bar'] + os.environ.pop("FOO") + + # On Windows the return code is actually 1. @pytest.mark.skipif(platform.system() == 'Windows', reason="broken on Windows") diff --git a/tools/src/test/python/test_command_sequence.py b/tools/src/test/python/test_command_sequence.py index 78e8de5265f..e6806a1cf41 100755 --- a/tools/src/test/python/test_command_sequence.py +++ b/tools/src/test/python/test_command_sequence.py @@ -20,7 +20,7 @@ # # -# Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. # import os @@ -38,8 +38,8 @@ def test_str(): cmds = CommandSequence(CommandSequenceBase("opengrok-master", - [{"command": ['foo']}, - {"command": ["bar"]}])) + [{"command": {"args": ['foo']}}, + {"command": {"args": ["bar"]}}])) assert str(cmds) == "opengrok-master" @@ -50,32 +50,25 @@ def test_invalid_configuration_commands_none(): assert str(exc_info.value) == "commands is None" -def test_invalid_configuration_commands_not_list(): - with pytest.raises(CommandConfigurationException) as exc_info: - CommandSequence(CommandSequenceBase("foo", {"foo": "bar"})) - - assert str(exc_info.value) == "commands is not a list" - - def test_invalid_configuration_commands_no_command(): with pytest.raises(CommandConfigurationException) as exc_info: - CommandSequence(CommandSequenceBase("foo", [{"command": ['foo']}, + CommandSequence(CommandSequenceBase("foo", [{"command": {"args": ['foo']}}, {"foo": "bar"}])) assert str(exc_info.value).startswith("command dictionary has unknown key") -def test_invalid_configuration_commands_no_list(): +def test_invalid_configuration_commands_no_dict1(): with pytest.raises(CommandConfigurationException) as exc_info: - CommandSequence(CommandSequenceBase("foo", [{"command": ['foo']}, - {"command": "bar"}])) + CommandSequence(CommandSequenceBase("foo", [{"command": {"args": ['foo']}}, + {"command": ["bar"]}])) - assert str(exc_info.value).startswith("command value not a list") + assert str(exc_info.value).startswith("command value not a dictionary") -def test_invalid_configuration_commands_no_dict(): +def test_invalid_configuration_commands_no_dict2(): with pytest.raises(CommandConfigurationException) as exc_info: - CommandSequence(CommandSequenceBase("foo", [{"command": ['foo']}, + CommandSequence(CommandSequenceBase("foo", [{"command": {"args": ['foo']}}, "command"])) assert str(exc_info.value).find("is not a dictionary") != -1 @@ -87,7 +80,7 @@ def test_timeout_propagation(): """ expected_timeout = 11 expected_api_timeout = 22 - cmd_seq_base = CommandSequenceBase("foo", [{"command": ['foo']}], + cmd_seq_base = CommandSequenceBase("foo", [{"command": {"args": ['foo']}}], api_timeout=expected_timeout, async_api_timeout=expected_api_timeout) cmd_seq = CommandSequence(cmd_seq_base) @@ -99,11 +92,11 @@ def test_timeout_propagation(): or not os.path.exists('/bin/echo'), reason="requires Unix") def test_run_retcodes(): - cmd_list = [{"command": ["/bin/echo"]}, - {"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + "; exit 0"]}, - {"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + "; exit 1"]}] + cmd_list = [{"command": {"args": ["/bin/echo"]}}, + {"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + "; exit 0"]}}, + {"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + "; exit 1"]}}] cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list)) cmds.run() assert cmds.retcodes == { @@ -117,9 +110,9 @@ def test_run_retcodes(): or not os.path.exists('/bin/echo'), reason="requires Unix") def test_terminate_after_non_zero_code(): - cmd_list = [{"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + "; exit 255"]}, - {"command": ["/bin/echo"]}] + cmd_list = [{"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + "; exit 255"]}}, + {"command": {"args": ["/bin/echo"]}}] cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list)) cmds.run() assert cmds.retcodes == { @@ -131,9 +124,9 @@ def test_terminate_after_non_zero_code(): or not os.path.exists('/bin/echo'), reason="requires Unix") def test_exit_2_handling(): - cmd_list = [{"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + "; exit 2"]}, - {"command": ["/bin/echo"]}] + cmd_list = [{"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + "; exit 2"]}}, + {"command": {"args": ["/bin/echo"]}}] cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list)) cmds.run() assert cmds.retcodes == { @@ -146,14 +139,14 @@ def test_exit_2_handling(): or not os.path.exists('/bin/echo'), reason="requires Unix") def test_driveon_flag(): - cmd_list = [{"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + "; exit 2"]}, - {"command": ["/bin/echo"]}, - {"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST + - "; exit 1"]}, - {"command": ["/bin/sh", "-c", - "echo " + PROJECT_SUBST]}] + cmd_list = [{"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + "; exit 2"]}}, + {"command": {"args": ["/bin/echo"]}}, + {"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST + + "; exit 1"]}}, + {"command": {"args": ["/bin/sh", "-c", + "echo " + PROJECT_SUBST]}}] cmds = CommandSequence(CommandSequenceBase("opengrok-master", cmd_list, driveon=True)) cmds.run() @@ -168,13 +161,37 @@ def test_driveon_flag(): @pytest.mark.skipif(not os.path.exists('/bin/echo'), reason="requires Unix") def test_project_subst(): - cmd_list = [{"command": ["/bin/echo", PROJECT_SUBST]}] + cmd_list = [{"command": {"args": ["/bin/echo", PROJECT_SUBST]}}] cmds = CommandSequence(CommandSequenceBase("test-subst", cmd_list)) cmds.run() assert cmds.outputs['/bin/echo test-subst'] == ['test-subst\n'] +@pytest.mark.skipif(not os.path.exists('/bin/echo'), + reason="requires Unix") +def test_args_subst(): + cmd_list = [{"command": {"args": ["/bin/echo", "%PATTERN%"], + "args_subst": {"%PATTERN%": "foo"}}}] + cmds = CommandSequence(CommandSequenceBase("test-subst", cmd_list)) + cmds.run() + + assert cmds.outputs['/bin/echo foo'] == ['foo\n'] + + +@pytest.mark.skipif(not os.path.exists('/bin/echo'), + reason="requires Unix") +def test_args_subst_env(): + cmd_list = [{"command": {"args": ["/bin/echo", "%PATTERN%"], + "args_subst": {"%PATTERN%": "$FOO"}}}] + os.environ["FOO"] = "bar" + cmds = CommandSequence(CommandSequenceBase("test-subst", cmd_list)) + cmds.run() + os.environ.pop("FOO") + + assert cmds.outputs['/bin/echo bar'] == ['bar\n'] + + def test_cleanup_exception(): """ If cleanup is not a list, exception should be thrown when initializing @@ -195,12 +212,12 @@ def test_cleanup(): with tempfile.TemporaryDirectory() as tmpdir: file_foo = os.path.join(tmpdir, "foo") file_bar = os.path.join(tmpdir, "bar") - cleanup_list = [{"command": ["/usr/bin/touch", file_foo]}, - {"command": ["/bin/cat", "/totallynonexistent"]}, - {"command": ["/usr/bin/touch", file_bar]}] + cleanup_list = [{"command": {"args": ["/usr/bin/touch", file_foo]}}, + {"command": {"args": ["/bin/cat", "/totallynonexistent"]}}, + {"command": {"args": ["/usr/bin/touch", file_bar]}}] # Running 'cat' on non-existing entry causes it to return 1. cmd = ["/bin/cat", "/foobar"] - cmd_list = [{"command": cmd}] + cmd_list = [{"command": {"args": cmd}}] commands = CommandSequence(CommandSequenceBase("test-cleanup-list", cmd_list, cleanup=cleanup_list))