config: switch to RO in isolated mode

The isolated instance shouldn't produce new transactions, including ones
from a background fiber started by an application or a role. So, the
isolated instance forcefully goes to the read-only mode disregarding any
other configuration options.

Part of tarantool#10776

NO_DOC=tarantool/doc#4632
NO_CHANGELOG=added together with the configuration option

Author: Totktonada

src/box/lua/config/applier/box_cfg.lua

@@ -61,6 +61,39 @@ local function peer_uris(configdata)
     return uris
 end
 
+-- Modify box-level configuration values and perform other actions
+-- to enable the isolated mode (if configured).
+local function switch_isolated_mode(configdata, box_cfg)
+    -- If the isolated mode is not enabled, there is nothing to do.
+    if not configdata:get('isolated', {use_default = true}) then
+        return
+    end
+
+    -- An application or a role may perform background database
+    -- modification if the instance is in the RW mode: for
+    -- example, a role may perform eviction of stale records.
+    -- If the instance is in the isolated mode, it should be in RO
+    -- to don't produce any new transactions.
+    --
+    -- The reason is that these transactions will be sent to other
+    -- replicaset members and applied on them, when the instance
+    -- goes from the isolated mode. At the same time, the
+    -- non-isolated part of the replicaset may serve requests and
+    -- perform data modifications. An attempt to modify the same
+    -- data from two instances may break data integrity[^1].
+    --
+    -- It is recommended to extract all the needed data from the
+    -- isolated instance and perform the modifications on the
+    -- current leader (in the non-isolated part of the
+    -- replicaset).
+    --
+    -- [^1]: Unless the data operations are carefully designed to
+    --       be idempotent to use in the master-master mode.
+    --
+    -- TODO(gh-10404): Set ro_reason=isolated.
+    box_cfg.read_only = true
+end
+
 local function log_destination(log)
     if log.to == 'stderr' or log.to == 'devnull' then
         return box.NULL
@@ -754,6 +787,10 @@ local function apply(config)
             box_cfg.read_only = true
         end
 
+        -- RO may be enforced by the isolated mode, so we call the
+        -- function after all the other RO/RW logic is done.
+        switch_isolated_mode(configdata, box_cfg)
+
         log.debug('box_cfg.apply (startup)')
         box.cfg(box_cfg)
 
@@ -762,6 +799,10 @@ local function apply(config)
         return {needs_retry = startup_may_be_long}
     end
 
+    -- RO may be enforced by the isolated mode, so we call the
+    -- function after all the other RO/RW logic is done.
+    switch_isolated_mode(configdata, box_cfg)
+
     -- If it is reload, just apply the new configuration.
     log.debug('box_cfg.apply')
     box.cfg(box_cfg)

test/config-luatest/isolated_mode_test.lua

@@ -93,3 +93,62 @@ g.test_startup_with_snap = function(g)
     g.it:roundtrip("box.space._schema:get({'replicaset_name'})[2]",
         'replicaset-001')
 end
+
+-- Verify that the instance goes to RO in the isolated mode
+-- disregarding of any other configuration options.
+--
+-- The test case verifies that it occurs on a runtime
+-- reconfiguration as well as on a startup.
+g.test_read_only = function(g)
+    local function assert_isolated(exp)
+        g.it:roundtrip("require('config'):get('isolated')", exp)
+    end
+
+    local function assert_read_only(exp)
+        g.it:roundtrip('box.info.ro', exp)
+    end
+
+    local config = cbuilder:new()
+        :set_replicaset_option('replication.failover', 'manual')
+        :set_replicaset_option('leader', 'i-001')
+        :add_instance('i-001', {})
+        :add_instance('i-002', {})
+        :add_instance('i-003', {})
+        :config()
+
+    local cluster = cluster.new(g, config)
+    cluster:start()
+
+    -- Mark i-001 as isolated, reload the configuration.
+    local config_2 = cbuilder:new(config)
+        :set_instance_option('i-001', 'isolated', true)
+        :config()
+    cluster:reload(config_2)
+
+    -- Use the console connection, because an instance in the
+    -- isolated mode doesn't accept iproto requests.
+    g.it = it.connect(cluster['i-001'])
+
+    -- The isolated mode is applied, the instance goes to RO.
+    assert_isolated(true)
+    assert_read_only(true)
+
+    -- Restart i-001, reconnect the console.
+    g.it:close()
+    cluster['i-001']:restart()
+    g.it = it.connect(cluster['i-001'])
+
+    -- Still in the isolated mode and RO.
+    assert_isolated(true)
+    assert_read_only(true)
+
+    -- Disable the isolated mode on i-001.
+    local config_3 = cbuilder:new(config_2)
+        :set_instance_option('i-001', 'isolated', nil)
+        :config()
+    cluster:reload(config_3)
+
+    -- Goes to RW.
+    assert_isolated(false)
+    assert_read_only(false)
+end