feat(chain-simulator): make block generation configurable with AUTO_GENERATE_BLOCKS

docs/source/dev_documentation/changelog.md

@@ -13,9 +13,11 @@
 - `set_states_batched` helper for pushing multiple account states with automatic payload-size grouping
 - `_set_state_with_retry` helper for transient failure resilience on `set_state` calls
 - `PROXY_TIMEOUT` configuration parameter (default 10s, mainnet 30s, chain simulator 60s) to prevent read timeouts when fetching large contract storage
+- `AUTO_GENERATE_BLOCKS` chain simulator configuration parameter (default `true`) to control whether MxOps drives block production itself; set it to `false` when the simulator auto-generates blocks so MxOps behaves like on other networks
 
 ### Changed
 
+- Chain simulator block production is now gated by the `AUTO_GENERATE_BLOCKS` flag: transaction waits, `WaitStep` (`for_blocks`) and `ChainSimulatorSetTokenBalanceStep` only drive blocks when it is enabled (the default). With it disabled, MxOps waits for the auto-producing simulator instead, polling at a clamped rate
 - Chain simulator default `API_RATE_LIMIT` increased to 100 (from 2) and `STORAGE_ITERATION_BATCH_SIZE` to 5000 (from 1000) for faster localhost operations
 - `AccountCloneStep` internal methods extracted into reusable module-level functions
 

docs/source/user_documentation/chain_simulator.md

@@ -35,3 +35,27 @@ Once you are done, don't forget to stop the chain-simulator:
 ```bash
 mxops chain-simulator stop
 ```
+
+## Block generation
+
+By default, the chain-simulator does not produce blocks on its own: MxOps drives
+block production for you. After sending a transaction it asks the simulator to
+generate blocks until the transaction is processed, and steps like `Wait` with
+`for_blocks` generate the requested blocks directly.
+
+The chain-simulator can also be configured (via its own `config.toml`) to
+auto-generate blocks at a fixed interval, behaving like a real network. There is
+no endpoint to detect this mode, so MxOps exposes a config option to control it:
+
+```ini
+[CHAIN_SIMULATOR]
+AUTO_GENERATE_BLOCKS=true
+```
+
+- `true` (default): MxOps generates blocks itself (the behavior described above).
+- `false`: MxOps does not generate any block and simply waits for the simulator
+  to produce them on its own, exactly like on devnet or mainnet. Set this when
+  you point MxOps at a simulator started with `auto-generate-blocks` enabled.
+
+This option only applies to the chain-simulator network and has no effect on
+other networks.

mxops/common/providers.py

@@ -4,6 +4,7 @@
 This module contains derived classes from api or proxy providers
 """
 
+from configparser import NoOptionError, NoSectionError
 import logging
 from time import sleep
 
@@ -19,12 +20,50 @@
 from requests.exceptions import HTTPError, Timeout
 
 from mxops.config.config import Config
-from mxops.enums import LogGroupEnum
-from mxops.errors import MaxIterationError, StorageIterationError
+from mxops.enums import LogGroupEnum, NetworkEnum
+from mxops.errors import (
+    InvalidConfigValue,
+    MaxIterationError,
+    StorageIterationError,
+)
 from mxops.utils.logger import get_logger
 from mxops.utils.progress import ProgressLogger
 
 
+def should_generate_blocks() -> bool:
+    """
+    Tell whether MxOps should manually drive block production.
+
+    True only on the chain simulator when AUTO_GENERATE_BLOCKS is enabled
+    (the default). False on every other network, and on the chain simulator
+    when AUTO_GENERATE_BLOCKS is disabled (the simulator auto-produces blocks
+    on its own and MxOps should behave like on any other network).
+
+    This helper lives here (rather than in ``config``) only to avoid an import
+    cycle: ``config`` is imported by nearly everything, while every call site
+    of this helper already imports from ``providers``.
+
+    :return: whether MxOps should generate blocks itself
+    :rtype: bool
+    """
+    config = Config.get_config()
+    if config.get_network() != NetworkEnum.CHAIN_SIMULATOR:
+        return False
+    try:
+        raw = config.get("AUTO_GENERATE_BLOCKS")
+    except (NoOptionError, NoSectionError):
+        return True
+    normalized = raw.strip().lower()
+    if normalized in ("true", "1", "yes", "on"):
+        return True
+    if normalized in ("false", "0", "no", "off", ""):
+        return False
+    raise InvalidConfigValue(
+        f"Invalid AUTO_GENERATE_BLOCKS value: {raw!r}. "
+        "Expected a boolean-like value (e.g. true/false)."
+    )
+
+
 def set_state_with_batching(
     proxy: ProxyNetworkProvider,
     account_state: dict,

mxops/errors.py

@@ -521,6 +521,12 @@ class InvalidDataFormat(Exception):
     """
 
 
+class InvalidConfigValue(Exception):
+    """
+    to be raised when a configuration option holds an invalid value
+    """
+
+
 class WrongFuzzTestFile(Exception):
     """
     to be raised when the file given for fuzz testing in not correctly formatted

mxops/execution/network.py

@@ -10,12 +10,18 @@
 from multiversx_sdk import TransactionOnNetwork
 from multiversx_sdk.core.constants import EGLD_IDENTIFIER_FOR_MULTI_ESDTNFT_TRANSFER
 
-from mxops.common.providers import MyProxyNetworkProvider
+from mxops.common.providers import MyProxyNetworkProvider, should_generate_blocks
 from mxops.config.config import Config
 from mxops import errors
-from mxops.enums import NetworkEnum
 from mxops.execution.msc import OnChainTokenTransfer
 
+# Minimum polling interval (seconds) used when MxOps does not drive block
+# production. The chain-simulator TX_REFRESH_PERIOD (0.001s) is tuned for the
+# generate-blocks-then-return path, where the polling loop returns on its first
+# iteration. When the simulator auto-produces blocks, that same period would
+# busy-poll the proxy thousands of times per transaction, so it is clamped here.
+MIN_TX_REFRESH_PERIOD = 0.2
+
 
 def send(tx: Transaction) -> str:
     """
@@ -49,10 +55,12 @@ def send_and_wait_for_result(
     refresh_period = float(config.get("TX_REFRESH_PERIOD"))
 
     tx_hash = proxy.send_transaction(tx).hex()
-    num_periods_to_wait = int(timeout / refresh_period)
-    if config.get_network() == NetworkEnum.CHAIN_SIMULATOR:
+    if should_generate_blocks():
         proxy.generate_blocks_until_tx_completion(tx_hash)
+    else:
+        refresh_period = max(refresh_period, MIN_TX_REFRESH_PERIOD)
 
+    num_periods_to_wait = int(timeout / refresh_period)
     for _ in range(0, num_periods_to_wait):
         time.sleep(refresh_period)
 

mxops/execution/steps/msc.py

@@ -13,11 +13,10 @@
 import numpy as np
 
 from mxops import errors
-from mxops.common.providers import MyProxyNetworkProvider
-from mxops.config.config import Config
+from mxops.common.providers import MyProxyNetworkProvider, should_generate_blocks
 from mxops.data.execution_data import ScenarioData
 from mxops.data.utils import json_dumps
-from mxops.enums import LogGroupEnum, NetworkEnum
+from mxops.enums import LogGroupEnum
 from mxops.execution import utils
 from mxops.smart_values import (
     SmartBool,
@@ -160,10 +159,9 @@ def _execute(self):
             return
         if self.for_blocks is not None:
             for_blocks = self.for_blocks.get_evaluated_value()
-            network = Config.get_config().get_network()
             shard = self.shard.get_evaluated_value()
             logger.info(f"Waiting for {for_blocks} blocks on shard {shard}")
-            if network == NetworkEnum.CHAIN_SIMULATOR:
+            if should_generate_blocks():
                 MyProxyNetworkProvider().generate_blocks(for_blocks)
             else:
                 utils.wait_for_n_blocks(shard, for_blocks)

mxops/execution/steps/setup.py

@@ -15,6 +15,7 @@
 from typing import ClassVar
 
 from multiversx_sdk import AccountStorage, Address, ProxyNetworkProvider
+from multiversx_sdk.core.constants import METACHAIN_ID
 from multiversx_sdk.network_providers.config import NetworkProviderConfig
 import requests
 
@@ -25,6 +26,7 @@
     get_account_storage_with_fallback,
     set_state_with_batching,
     set_states_batched,
+    should_generate_blocks,
 )
 from mxops.config.config import Config
 from mxops.data.data_cache import (
@@ -1144,7 +1146,17 @@ def _execute(self):
             logger.debug(f"Pushing {len(pairs)} ESDT balance key(s) to {bech32}")
             proxy.set_address_state(bech32, pairs)
 
-        # Phase 5: generate a block so the new state is committed and visible
-        # via the standard proxy endpoints (e.g. address/.../esdt/...). Without
-        # this, get_token_of_account can keep returning the pre-write balance.
-        proxy.generate_blocks(1)
+        # Phase 5: ensure a block is produced so the new state is committed and
+        # visible via the standard proxy endpoints (e.g. address/.../esdt/...).
+        # Without this, get_token_of_account can keep returning the pre-write
+        # balance. generate_blocks(1) advances every shard at once; when the
+        # simulator auto-produces blocks we instead wait for each shard to
+        # advance — the receivers' user shards (where the ESDT balances live)
+        # and the metachain (where the ESDT module was written) — since a single
+        # metachain block does not prove a user shard has committed its state.
+        if should_generate_blocks():
+            proxy.generate_blocks(1)
+        else:
+            num_shards = Config.get_config().get_network_config().num_shards
+            for shard in (*range(num_shards), METACHAIN_ID):
+                utils.wait_for_n_blocks(shard, 1)

mxops/resources/default_config.ini

@@ -22,6 +22,7 @@ TX_REFRESH_PERIOD=0.001
 API_RATE_LIMIT=100
 STORAGE_ITERATION_BATCH_SIZE=5000
 PROXY_TIMEOUT=60
+AUTO_GENERATE_BLOCKS=true
 
 [LOCAL]
 PROXY=http://localhost:7950

mxops/utils/chain_simulator.py

@@ -303,7 +303,12 @@ def start_chain_simulator(
         )
         return
 
-    # Only generate first epoch if chain-simulator service is included
+    # Only generate first epoch if chain-simulator service is included.
+    # This is intentionally not gated by AUTO_GENERATE_BLOCKS: the bundled
+    # simulator is always started in manual mode, this runs once during
+    # bootstrap before any scene, and the retry loop doubles as a readiness
+    # probe (it keeps retrying until the node answers). The auto-generate case
+    # targets an externally-running simulator, which is not started here.
     if "chain-simulator" in resolved_services:
         retry_count = 0
         proxy = MyProxyNetworkProvider()

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mxops"
-version = "3.2.0-dev2"
+version = "3.2.0-dev3"
 authors = [
   {name="Etienne Wallet"},
 ]

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.2.0-dev2
+current_version = 3.2.0-dev3
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-(?P<release>[^-0-9]+)(?P<build>\d+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}{build}

tests/test_block_generation.py

@@ -0,0 +1,61 @@
+"""
+Unit tests for should_generate_blocks, the helper that decides whether MxOps
+should manually drive block production on the chain simulator.
+"""
+
+from configparser import NoOptionError
+
+import pytest
+import pytest_mock
+
+from mxops import errors
+from mxops.common.providers import should_generate_blocks
+from mxops.config.config import Config
+
+
+def test_should_generate_blocks_false_on_other_networks():
+    """Default test network is LOCAL: MxOps never generates blocks."""
+    assert should_generate_blocks() is False
+
+
+def test_should_generate_blocks_true_on_chain_simulator_by_default(
+    chain_simulator_network,
+):
+    """Chain simulator with the default config keeps today's behavior."""
+    assert should_generate_blocks() is True
+
+
+def test_should_generate_blocks_false_when_disabled(chain_simulator_network):
+    """AUTO_GENERATE_BLOCKS=false means the simulator auto-produces blocks."""
+    config = Config.get_config()
+    prev = config.get("AUTO_GENERATE_BLOCKS")
+    config.set_option("AUTO_GENERATE_BLOCKS", "false")
+    try:
+        assert should_generate_blocks() is False
+    finally:
+        config.set_option("AUTO_GENERATE_BLOCKS", prev)
+
+
+def test_should_generate_blocks_rejects_invalid_value(chain_simulator_network):
+    """A malformed value must raise rather than silently disabling blocks."""
+    config = Config.get_config()
+    prev = config.get("AUTO_GENERATE_BLOCKS")
+    config.set_option("AUTO_GENERATE_BLOCKS", "ture")  # typo
+    try:
+        with pytest.raises(errors.InvalidConfigValue, match="AUTO_GENERATE_BLOCKS"):
+            should_generate_blocks()
+    finally:
+        config.set_option("AUTO_GENERATE_BLOCKS", prev)
+
+
+def test_should_generate_blocks_defaults_true_when_option_missing(
+    chain_simulator_network, mocker: pytest_mock.MockerFixture
+):
+    """A user config without the option must keep working (defaults to True)."""
+    config = Config.get_config()
+    mocker.patch.object(
+        config,
+        "get",
+        side_effect=NoOptionError("AUTO_GENERATE_BLOCKS", "CHAIN_SIMULATOR"),
+    )
+    assert should_generate_blocks() is True

tests/test_chain_simulator_set_token_balance.py

@@ -2,11 +2,13 @@
 Unit tests for ChainSimulatorSetTokenBalanceStep and its protobuf/key helpers.
 """
 
-from unittest.mock import patch
+from unittest.mock import call, patch
 
+from multiversx_sdk.core.constants import METACHAIN_ID
 import pytest
 
 from mxops import errors
+from mxops.config.config import Config
 from mxops.enums import NetworkEnum
 from mxops.execution.steps import ChainSimulatorSetTokenBalanceStep
 from mxops.execution.steps.setup import (
@@ -254,6 +256,45 @@ def test_step_success_groups_by_receiver(chain_simulator_scenario):
     mock_blocks.assert_called_once_with(1)
 
 
+def test_step_auto_generate_blocks_disabled_waits(chain_simulator_scenario):
+    """When AUTO_GENERATE_BLOCKS is off, the simulator produces blocks on its
+    own: the step must not force a block and must instead wait for every shard
+    (the receivers' user shards and the metachain) to advance, so the set-state
+    is committed there before returning. num_shards is 3 in the mocked config."""
+    config = Config.get_config()
+    prev = config.get("AUTO_GENERATE_BLOCKS")
+    config.set_option("AUTO_GENERATE_BLOCKS", "false")
+    step = _make_step(
+        [{"receiver": MOCK_BECH32_A, "token_identifier": USDC, "amount": 100}]
+    )
+    try:
+        with patch(
+            "mxops.execution.steps.setup._get_esdt_module_clone_data",
+            return_value=({"address": "esdt-mod", "pairs": {}}, set()),
+        ), patch(
+            "mxops.execution.steps.setup._insert_tokens_in_elasticsearch"
+        ), patch(
+            "mxops.execution.steps.setup.MyProxyNetworkProvider.set_state"
+        ), patch(
+            "mxops.execution.steps.setup.MyProxyNetworkProvider.set_address_state"
+        ), patch(
+            "mxops.execution.steps.setup.MyProxyNetworkProvider.generate_blocks"
+        ) as mock_blocks, patch(
+            "mxops.execution.steps.setup.utils.wait_for_n_blocks"
+        ) as mock_wait:
+            step.execute()
+
+        mock_blocks.assert_not_called()
+        assert mock_wait.call_args_list == [
+            call(0, 1),
+            call(1, 1),
+            call(2, 1),
+            call(METACHAIN_ID, 1),
+        ]
+    finally:
+        config.set_option("AUTO_GENERATE_BLOCKS", prev)
+
+
 def test_step_pushes_esdt_module_only_for_newly_cloned(chain_simulator_scenario):
     """If _get_esdt_module_clone_data signals that some tokens were cloned
     (newly_cloned non-empty), the step must push the ESDT-module state via