diff --git a/deployments/sequencer/src/app.py b/deployments/sequencer/src/app.py index 4f51ca92099..2aa023b95eb 100644 --- a/deployments/sequencer/src/app.py +++ b/deployments/sequencer/src/app.py @@ -50,7 +50,6 @@ def main(): monitoring_configs["enabled"], args.layout, args.overlay or [], - args.config_format, ) _create_monitoring_chart(app, namespace, args.cluster, monitoring_configs) @@ -189,7 +188,6 @@ def _create_service_charts( monitoring_enabled: bool, layout: str, overlays: list[str], - config_format: str, ): """Create SequencerNodeChart for each service.""" for service_cfg in deployment_config.services: @@ -201,7 +199,6 @@ def _create_service_charts( service_config=service_cfg, layout=layout, overlays=overlays, - config_format=config_format, ) diff --git a/deployments/sequencer/src/charts/node.py b/deployments/sequencer/src/charts/node.py index 3dfd92309ce..97f96984d39 100644 --- a/deployments/sequencer/src/charts/node.py +++ b/deployments/sequencer/src/charts/node.py @@ -31,13 +31,11 @@ def __init__( service_config: ServiceConfig, layout: str, overlays: list[str], - config_format: str, ): super().__init__(scope, name, disable_resource_name_hashes=True, namespace=namespace) self.monitoring = monitoring self.service_config = service_config - self.config_format = config_format # Create labels dictionary from service config + service name # Base labels from shared config (metaLabels) - now merged into service_config @@ -57,7 +55,6 @@ def __init__( monitoring_endpoint_port, layout, overlays, - config_format=config_format, ) # Create ServiceAccount if enabled @@ -104,7 +101,6 @@ def __init__( service_config=self.service_config, labels=labels, monitoring_endpoint_port=monitoring_endpoint_port, - config_format=config_format, ) else: self.controller = DeploymentConstruct( @@ -113,7 +109,6 @@ def __init__( service_config=self.service_config, labels=labels, monitoring_endpoint_port=monitoring_endpoint_port, - config_format=config_format, ) # Create BackendConfig if enabled diff --git a/deployments/sequencer/src/cli.py b/deployments/sequencer/src/cli.py index d23aa119c45..5d37a9f4318 100644 --- a/deployments/sequencer/src/cli.py +++ b/deployments/sequencer/src/cli.py @@ -75,16 +75,6 @@ def argument_parser(): action=UniqueStoreAction, help="Override image for all services. Format: 'repository:tag' or 'repository' (defaults to 'latest' tag).", ) - parser.add_argument( - "--config-format", - type=str, - action=UniqueStoreAction, - choices=["preset", "native"], - default="preset", - help="Node config format to emit and pass to the node container via --config_format. " - "'preset' (default): flat dotted-key placeholder fill (legacy). " - "'native': nested SequencerNodeConfig assembled via jsonnet build().", - ) parser.add_argument( "-v", "--verbose", diff --git a/deployments/sequencer/src/config/loaders.py b/deployments/sequencer/src/config/loaders.py index ae6fe8a991c..b9bd1930e09 100644 --- a/deployments/sequencer/src/config/loaders.py +++ b/deployments/sequencer/src/config/loaders.py @@ -1,13 +1,10 @@ import json import os -import sys from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Dict, List, Optional import yaml -from rich.console import Console -from rich.panel import Panel from src.config.overlay import ( merge_common_with_overlay_strict, merge_service_overlay, @@ -189,645 +186,6 @@ def load(self) -> dict: return wrapped -class NodeConfigLoader(Config): - ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../../") - - def __init__(self, config_list_json_path: str): - """ - Initialize NodeConfigLoader. - - Args: - config_list_json_path: Path to JSON file containing a list of config paths - """ - self.config_list_json_path = config_list_json_path - self._validate() - - def _validate(self): - # Validate the config list JSON file - full_path = os.path.join(self.ROOT_DIR, self.config_list_json_path) - self._validate_file(full_path) - - def load(self) -> dict: - # Load the JSON file containing the list of config paths - config_list_full_path = os.path.join(self.ROOT_DIR, self.config_list_json_path) - config_data = self._try_load_json(file_path=config_list_full_path) - - # Validate that it's a list of strings - if not isinstance(config_data, list): - raise ValueError( - f"Config list JSON file '{self.config_list_json_path}' must contain a JSON array. Got: {type(config_data)}" - ) - config_list: List[str] = config_data - if not all(isinstance(item, str) for item in config_list): - raise ValueError( - f"Config list JSON file '{self.config_list_json_path}' must contain a JSON array of strings." - ) - - # Load and merge all config files in the list - result = {} - for config_path in config_list: - # Use the full path as provided in the config list - config_full_path = os.path.join(self.ROOT_DIR, config_path) - try: - data = self._try_load_json(file_path=config_full_path) - if not isinstance(data, dict): - raise ValueError( - f"Config file '{config_path}' must contain a JSON object (dict), got: {type(data)}" - ) - result.update(data) # later values overwrite previous - except FileNotFoundError: - # Fail fast if a specified config file doesn't exist - raise FileNotFoundError(f"Config file not found: {config_full_path}") - except ValueError as e: - # Fail fast if a config file has invalid JSON - raise ValueError(f"Invalid JSON in config file {config_full_path}: {e}") - - # Return a lexicographically sorted dict to ensure consistent ordering and simpler CM diffs. - return dict[Any, Any](sorted(result.items())) - - @staticmethod - def _set_nested_dotted_key(data: dict, dotted_key: str, value: Any) -> None: - """Set value in nested dict using dotted key notation, creating structure if needed. - - Examples: - _set_nested_dotted_key({}, 'a.b.c', 123) -> {'a': {'b': {'c': 123}}} - _set_nested_dotted_key({'a': {'x': 1}}, 'a.b.c', 123) -> {'a': {'x': 1, 'b': {'c': 123}}} - """ - keys = dotted_key.split(".") - current = data - for key in keys[:-1]: - if key not in current or not isinstance(current[key], dict): - current[key] = {} - current = current[key] - current[keys[-1]] = value - - @staticmethod - def _replace_placeholder_value(obj: Any, placeholder: str, replacement: Any) -> Any: - """Recursively search for a placeholder value and replace it with the replacement value. - - Args: - obj: The object to search (dict, list, or primitive) - placeholder: The placeholder string to find (e.g., '$$$_CHAIN_ID_$$$') - replacement: The value to replace it with - - Returns: - The object with placeholder replaced (if found) - """ - if isinstance(obj, dict): - return { - k: NodeConfigLoader._replace_placeholder_value(v, placeholder, replacement) - for k, v in obj.items() - } - elif isinstance(obj, list): - return [ - NodeConfigLoader._replace_placeholder_value(item, placeholder, replacement) - for item in obj - ] - elif isinstance(obj, str) and obj == placeholder: - return replacement - elif isinstance(obj, (int, float)) and str(obj) == placeholder: - return replacement - else: - return obj - - @staticmethod - def _placeholder_exists(obj: Any, placeholder: str) -> bool: - """Check if a placeholder value exists anywhere in the config object. - - Args: - obj: The object to search (dict, list, or primitive) - placeholder: The placeholder string to find (e.g., '$$$_CHAIN_ID_$$$') - - Returns: - True if placeholder is found, False otherwise - """ - if isinstance(obj, dict): - return any(NodeConfigLoader._placeholder_exists(v, placeholder) for v in obj.values()) - elif isinstance(obj, list): - return any(NodeConfigLoader._placeholder_exists(item, placeholder) for item in obj) - elif isinstance(obj, str) and obj == placeholder: - return True - elif isinstance(obj, (int, float)) and str(obj) == placeholder: - return True - else: - return False - - @staticmethod - def _normalize_placeholder(placeholder: str) -> str: - """Normalize placeholder by replacing dashes with underscores. - - Only affects values matching the $$$_..._$$$ pattern. This ensures all - placeholders use underscores consistently for matching. - - Args: - placeholder: The placeholder string (e.g., '$$$_COMPONENTS-SIERRA-COMPILER-URL_$$$') - - Returns: - Normalized placeholder with dashes replaced by underscores - (e.g., '$$$_COMPONENTS_SIERRA_COMPILER_URL_$$$') - - Examples: - '$$$_COMPONENTS-SIERRA-COMPILER-URL_$$$' -> '$$$_COMPONENTS_SIERRA_COMPILER_URL_$$$' - '$$$_CHAIN_ID_$$$' -> '$$$_CHAIN_ID_$$$' (no change) - 'regular_string' -> 'regular_string' (no change) - """ - if ( - isinstance(placeholder, str) - and placeholder.startswith("$$$_") - and placeholder.endswith("_$$$") - ): - # Replace dashes with underscores in the middle part (between $$$_ and _$$$) - middle = placeholder[4:-4] # Remove $$$_ prefix and _$$$ suffix - normalized_middle = middle.replace("-", "_") - return f"$$$_{normalized_middle}_$$$" - return placeholder - - @staticmethod - def _normalize_placeholders_in_config(obj: Any) -> Any: - """Recursively normalize all placeholders in a config object. - - Traverses dictionaries, lists, and primitive values, replacing dashes - with underscores in all placeholder values matching $$$_..._$$$ pattern. - - Args: - obj: The config object to normalize (dict, list, or primitive) - - Returns: - The normalized config object with all placeholders normalized - """ - if isinstance(obj, dict): - return { - k: NodeConfigLoader._normalize_placeholders_in_config(v) for k, v in obj.items() - } - elif isinstance(obj, list): - return [NodeConfigLoader._normalize_placeholders_in_config(item) for item in obj] - elif isinstance(obj, str): - return NodeConfigLoader._normalize_placeholder(obj) - else: - # For int, float, bool, etc., check if string representation is a placeholder - if isinstance(obj, (int, float)): - str_repr = str(obj) - if str_repr.startswith("$$$_") and str_repr.endswith("_$$$"): - normalized = NodeConfigLoader._normalize_placeholder(str_repr) - # Try to preserve original type if possible - try: - if isinstance(obj, int): - return int(normalized) - elif isinstance(obj, float): - return float(normalized) - except (ValueError, TypeError): - return normalized - return obj - - @staticmethod - def _yaml_key_to_placeholder(yaml_key: str) -> str: - """Convert a YAML key to the full placeholder format. - - YAML keys use hierarchical structure with dots (e.g., 'components.batcher.port'), - which map to placeholders with hyphens (e.g., '$$$_COMPONENTS-BATCHER-PORT_$$$'). - - The transformation: - - Dots (.) in YAML → Hyphens (-) in placeholder - - Lowercase → Uppercase - - Special keys with '.#is_none' → '.#is_none' becomes '-IS_NONE' (hash removed) - - Args: - yaml_key: The YAML key with dots (e.g., 'components.batcher.port', 'chain_id') - - Returns: - The full placeholder format (e.g., '$$$_COMPONENTS-BATCHER-PORT_$$$', '$$$_CHAIN_ID_$$$') - - Examples: - 'components.batcher.port' -> '$$$_COMPONENTS-BATCHER-PORT_$$$' - 'components.sierra_compiler.url' -> '$$$_COMPONENTS-SIERRA_COMPILER-URL_$$$' - 'consensus_manager_config.context_config.override_eth_to_fri_rate.#is_none' -> '$$$_CONSENSUS_MANAGER_CONFIG-CONTEXT_CONFIG-OVERRIDE_ETH_TO_FRI_RATE-IS_NONE_$$$' - 'chain_id' -> '$$$_CHAIN_ID_$$$' (single-level keys still work) - """ - # Convert dots to hyphens, then uppercase - # Special handling: '.#is_none' becomes '-IS_NONE' (remove the #) - placeholder = yaml_key.replace(".#is_none", "-is_none").replace(".", "-").upper() - return f"$$$_{placeholder}_$$$" - - @staticmethod - def _placeholder_to_yaml_key(placeholder: str) -> str: - """Convert a placeholder back to YAML key format. - - This is the inverse of _yaml_key_to_placeholder. - - Args: - placeholder: The placeholder string (e.g., '$$$_COMPONENTS-BATCHER-PORT_$$$') - - Returns: - The YAML key format (e.g., 'components.batcher.port') - - Examples: - '$$$_COMPONENTS-BATCHER-PORT_$$$' -> 'components.batcher.port' - '$$$_CONSENSUS_MANAGER_CONFIG-CONTEXT_CONFIG-OVERRIDE_ETH_TO_FRI_RATE-IS_NONE_$$$' -> 'consensus_manager_config.context_config.override_eth_to_fri_rate.#is_none' - """ - # Remove $$$_ prefix and _$$$ suffix - middle = placeholder[4:-4] - # Convert hyphens to dots and lowercase - yaml_key = middle.replace("-", ".").lower() - # Special handling: convert '-is_none' back to '.#is_none' - yaml_key = yaml_key.replace(".is_none", ".#is_none") - return yaml_key - - @staticmethod - def _path_relative_to_root(absolute_path: Path) -> str: - """Return path relative to ROOT_DIR if under it, otherwise the path as-is.""" - root = Path(NodeConfigLoader.ROOT_DIR).resolve() - path = absolute_path.resolve() - try: - return str(path.relative_to(root)) - except ValueError: - return str(path) - - @staticmethod - def _print_file_paths_section( - console: Console, - config_list_path: Optional[str], - layout: Optional[str] = None, - overlays: Optional[List[str]] = None, - service_name: Optional[str] = None, - ) -> None: - """Print the file paths section (relative to repo root so paths stay short and visible). - - Args: - console: Rich Console instance - config_list_path: Optional path to the config list JSON file - layout: Optional layout name (e.g., "hybrid") - overlays: Optional list of overlay flag values (e.g., ["hybrid.mainnet", "hybrid.mainnet.apollo-mainnet-0"]) - service_name: Optional service name (e.g., "core") to show overlay service YAML path(s) - """ - lines: List[tuple[str, str]] = [] - - if config_list_path: - full_config_path = Path(NodeConfigLoader.ROOT_DIR) / config_list_path - lines.append( - ( - "application_config_json_path:", - NodeConfigLoader._path_relative_to_root(full_config_path), - ) - ) - else: - lines.append(("application_config_json_path:", "")) - - if layout: - layout_path = ( - Path(NodeConfigLoader.ROOT_DIR) / "configs" / "layouts" / layout - ).resolve() - lines.append( - ("config_layout_path:", NodeConfigLoader._path_relative_to_root(layout_path)) - ) - else: - lines.append(("config_layout_path:", "")) - - if overlays: - for idx, overlay in enumerate(overlays): - overlay_path_segments = overlay.split(".") - if overlay_path_segments and layout and overlay_path_segments[0] == layout: - overlay_base_path = ( - Path(NodeConfigLoader.ROOT_DIR) / "configs" / "overlays" / layout - ).resolve() - for segment in overlay_path_segments[1:]: - overlay_base_path = overlay_base_path / segment - label = ( - "config_overlay_path:" if idx == 0 else f"config_overlay_path_{idx + 1}:" - ) - lines.append( - (label, NodeConfigLoader._path_relative_to_root(overlay_base_path)) - ) - if service_name: - service_yaml_path = overlay_base_path / "services" / f"{service_name}.yaml" - svc_label = ( - "config_overlay_service_yaml_path:" - if idx == 0 - else f"config_overlay_service_yaml_path_{idx + 1}:" - ) - lines.append( - (svc_label, NodeConfigLoader._path_relative_to_root(service_yaml_path)) - ) - else: - label = ( - "config_overlay_path:" if idx == 0 else f"config_overlay_path_{idx + 1}:" - ) - lines.append((label, "")) - else: - lines.append(("config_overlay_path:", "")) - - console.print("\n[bold]File Paths:[/bold]") - # Use plain print so paths are never wrapped or cropped by Rich's terminal width - for label, value in lines: - print(f" {label} {value}", file=console.file) - - @staticmethod - def _print_unused_config_keys_section( - console: Console, unmatched_keys: List[tuple[str, str]] - ) -> None: - """Print the unused config keys section using rich formatting. - - Args: - console: Rich Console instance - unmatched_keys: List of (yaml_key, placeholder) tuples - """ - if not unmatched_keys: - return - - console.print() - console.print( - Panel.fit( - f"[yellow]Found {len(unmatched_keys)} config key(s) in your YAML file that don't have\n" - "corresponding placeholders in the application config JSON file[/yellow]", - title="[bold red]Unused Config Keys[/bold red]", - border_style="red", - ) - ) - - # Simple list format: config key on one line, placeholder on next line - for yaml_key, placeholder in unmatched_keys: - console.print(f"[yellow]{yaml_key}[/yellow]") - console.print(f"[dim]{placeholder}[/dim]") - console.print() # Empty line between items - - @staticmethod - def _print_missing_placeholders_section( - console: Console, remaining_placeholders: set[str], config: dict - ) -> None: - """Print the missing placeholders section using rich formatting. - - Args: - console: Rich Console instance - remaining_placeholders: Set of placeholder strings that weren't overridden - config: The config dictionary to find placeholder locations in - """ - if not remaining_placeholders: - return - - sorted_placeholders = sorted(remaining_placeholders) - console.print() - console.print( - Panel.fit( - f"[yellow]The following {len(sorted_placeholders)} placeholder(s) were found in the\n" - "application config but were not overridden in your YAML overlay[/yellow]", - title="[bold red]Missing Placeholders[/bold red]", - border_style="red", - ) - ) - - # Simple list format: placeholder on one line, expected config key on next line - for placeholder in sorted_placeholders: - # Convert placeholder back to YAML key format for suggestion - yaml_key_suggestion = NodeConfigLoader._placeholder_to_yaml_key(placeholder) - console.print(f"[red]{placeholder}[/red]") - console.print(f"[green]{yaml_key_suggestion}[/green]") - console.print() # Empty line between items - - @staticmethod - def apply_sequencer_overrides( - merged_json_config: dict, - sequencer_config: Dict[str, Any], - service_name: str = "unknown", - config_list_path: Optional[str] = None, - layout: Optional[str] = None, - overlays: Optional[List[str]] = None, - ) -> dict: - """Apply sequencerConfig overrides from YAML to merged JSON config. - - Overrides are applied by placeholder value, not by JSON key. This makes the - deployment resilient to JSON key changes as long as the placeholder values remain the same. - - YAML keys use hierarchical structure with dots (e.g., 'components.batcher.port'), - which are automatically converted to placeholder format (e.g., '$$$_COMPONENTS-BATCHER-PORT_$$$') - for matching against placeholders in the JSON config. - - Args: - merged_json_config: The merged JSON config dictionary from all config files - sequencer_config: Dictionary from YAML with hierarchical keys: - { - 'components.batcher.port': 55000, - 'components.sierra_compiler.url': 'sequencer-sierracompiler-service', - 'chain_id': '123' - } - These are converted to placeholder format for matching. - service_name: Name of the service (for error messages) - config_list_path: Optional path to the config list JSON file (for error messages) - layout: Optional layout name (e.g., "hybrid") - overlays: Optional list of overlay flag values (e.g., ["hybrid.mainnet", "hybrid.mainnet.apollo-mainnet-0"]) - - Returns: - Updated config dictionary with overrides applied - - Raises: - ValueError: If any YAML key doesn't match any placeholder in the config - - Examples: - JSON: {'components.batcher.port': '$$$_COMPONENTS-BATCHER-PORT_$$$'} - YAML: {'components.batcher.port': 55000} - Result: {'components.batcher.port': 55000} - """ - # Use config as-is (no normalization needed - placeholders already use hyphens) - result = merged_json_config - - # Step 1: Validate that all YAML keys match existing placeholders - # Collect unmatched keys but don't raise yet - we'll check after applying overrides - unmatched_keys = [] - matched_keys = [] - for yaml_key, replacement_value in sequencer_config.items(): - # Convert YAML key to placeholder format (dots -> hyphens, uppercase) - placeholder = NodeConfigLoader._yaml_key_to_placeholder(yaml_key) - # Check if placeholder exists in the config - exists = NodeConfigLoader._placeholder_exists(result, placeholder) - if not exists: - unmatched_keys.append((yaml_key, placeholder)) - else: - matched_keys.append((yaml_key, placeholder, replacement_value)) - - # Step 2: Apply overrides for matched keys only - for yaml_key, placeholder, replacement_value in matched_keys: - # Replace the placeholder value wherever it appears in the config - result = NodeConfigLoader._replace_placeholder_value( - result, placeholder, replacement_value - ) - - # Step 3: Check for remaining placeholders - remaining_placeholders = NodeConfigLoader._find_all_placeholders(result) - - # Step 4: If there are any issues, raise a combined error - if unmatched_keys or remaining_placeholders: - # Use stderr for error output to ensure it's visible even when stdout is captured - console = Console(file=sys.stderr, force_terminal=True) - total_issues = len(unmatched_keys) + len(remaining_placeholders) - - # Determine error title - if unmatched_keys and remaining_placeholders: - error_title = ( - "CONFIGURATION ERRORS DETECTED (Unused Config Keys & Unhandled Placeholders)" - ) - elif unmatched_keys: - error_title = "UNUSED CONFIG KEY(S) DETECTED" - else: - error_title = "UNHANDLED PLACEHOLDER(S) DETECTED" - - # Print formatted error message - console.print() - console.print( - Panel.fit( - f"[bold red]{error_title}[/bold red]\n\n" - f"Found [yellow]{total_issues}[/yellow] issue(s):\n" - + ( - f" - [cyan]{len(remaining_placeholders)}[/cyan] unhandled placeholder(s)\n" - if remaining_placeholders - else "" - ) - + ( - f" - [cyan]{len(unmatched_keys)}[/cyan] unused config key(s)\n" - if unmatched_keys - else "" - ), - border_style="red", - ) - ) - - NodeConfigLoader._print_file_paths_section( - console, config_list_path, layout, overlays, service_name=service_name - ) - NodeConfigLoader._print_missing_placeholders_section( - console, remaining_placeholders, result - ) - NodeConfigLoader._print_unused_config_keys_section(console, unmatched_keys) - - # Ensure output is flushed before raising exception - sys.stderr.flush() - - # Build plain text version for ValueError - error_message = f"{error_title}\n\n" - error_message += f"Found {total_issues} issue(s):\n" - if remaining_placeholders: - error_message += f" - {len(remaining_placeholders)} unhandled placeholder(s)\n" - if unmatched_keys: - error_message += f" - {len(unmatched_keys)} unused config key(s)\n" - error_message += "\nSee formatted output above for details." - - raise ConfigValidationError(error_message) - - # Re-sort after modifications - return dict[Any, Any](sorted(result.items())) - - @staticmethod - def _find_all_placeholders(obj: Any) -> set[str]: - """Recursively find all placeholder values ($$$_..._$$$) in the config object. - - Args: - obj: The object to search (dict, list, or primitive) - - Returns: - Set of all placeholder values found (e.g., {'$$$_CHAIN_ID_$$$', '$$$_STARKNET_URL_$$$'}) - """ - placeholders: set[str] = set() - - if isinstance(obj, dict): - for value in obj.values(): - placeholders.update(NodeConfigLoader._find_all_placeholders(value)) - elif isinstance(obj, list): - for item in obj: - placeholders.update(NodeConfigLoader._find_all_placeholders(item)) - elif isinstance(obj, str) and obj.startswith("$$$_") and obj.endswith("_$$$"): - placeholders.add(obj) - elif isinstance(obj, (int, float)): - str_repr = str(obj) - if str_repr.startswith("$$$_") and str_repr.endswith("_$$$"): - placeholders.add(str_repr) - - return placeholders - - @staticmethod - def _find_placeholder_locations(obj: Any, placeholder: str, path: str = "") -> List[str]: - """Recursively find all key paths where a placeholder value appears in the config object. - - Args: - obj: The object to search (dict, list, or primitive) - placeholder: The placeholder string to find (e.g., '$$$_CHAIN_ID_$$$') - path: Current path in the object (for building full paths) - - Returns: - List of key paths where the placeholder was found (e.g., ['chain_id', 'components.batcher.port']) - """ - locations: List[str] = [] - - if isinstance(obj, dict): - for key, value in obj.items(): - current_path = f"{path}.{key}" if path else key - locations.extend( - NodeConfigLoader._find_placeholder_locations(value, placeholder, current_path) - ) - elif isinstance(obj, list): - for idx, item in enumerate(obj): - current_path = f"{path}[{idx}]" if path else f"[{idx}]" - locations.extend( - NodeConfigLoader._find_placeholder_locations(item, placeholder, current_path) - ) - elif isinstance(obj, str) and obj == placeholder: - locations.append(path if path else "") - elif isinstance(obj, (int, float)) and str(obj) == placeholder: - locations.append(path if path else "") - - return locations - - @staticmethod - def validate_no_remaining_placeholders( - config: dict, - config_list_path: Optional[str] = None, - layout: Optional[str] = None, - overlays: Optional[List[str]] = None, - service_name: Optional[str] = None, - ) -> None: - """Validate that no placeholder values remain in the final config. - - Args: - config: The final config dictionary after all overrides are applied - config_list_path: Optional path to the config list JSON file (for error messages) - layout: Optional layout name (e.g., "hybrid") - overlays: Optional list of overlay flag values (e.g., ["hybrid.mainnet", "hybrid.mainnet.apollo-mainnet-0"]) - service_name: Optional service name (e.g., "core") to show overlay service YAML path in error output - - Raises: - ValueError: If any placeholder values ($$$_..._$$$) are found in the config - """ - remaining_placeholders = NodeConfigLoader._find_all_placeholders(config) - - if remaining_placeholders: - # Use stderr for error output to ensure it's visible even when stdout is captured - console = Console(file=sys.stderr, force_terminal=True) - sorted_placeholders = sorted(remaining_placeholders) - - # Print formatted error message - console.print() - console.print( - Panel.fit( - f"[bold red]UNHANDLED PLACEHOLDERS DETECTED[/bold red]\n\n" - f"Found [yellow]{len(sorted_placeholders)}[/yellow] unhandled placeholder(s) in the final config.", - border_style="red", - ) - ) - - NodeConfigLoader._print_file_paths_section( - console, config_list_path, layout, overlays, service_name=service_name - ) - NodeConfigLoader._print_missing_placeholders_section( - console, remaining_placeholders, config - ) - - # Ensure output is flushed before raising exception - sys.stderr.flush() - - # Build plain text version for ValueError - error_message = "UNHANDLED PLACEHOLDERS DETECTED\n\n" - error_message += f"Found {len(sorted_placeholders)} unhandled placeholder(s) in the final config.\n\n" - error_message += "See formatted output above for details." - - raise ConfigValidationError(error_message) - - class GrafanaDashboardConfigLoader(Config): def __init__(self, dashboard_file_path: str): self.dashboard_file_path = os.path.abspath(dashboard_file_path) diff --git a/deployments/sequencer/src/config/native.py b/deployments/sequencer/src/config/native.py index 1443e27422c..6df4a4b954a 100644 --- a/deployments/sequencer/src/config/native.py +++ b/deployments/sequencer/src/config/native.py @@ -2,7 +2,7 @@ The legacy "preset" path fills `$$$_..._$$$` placeholders in flat dotted-key replacer JSON to produce the ConfigMap. The "native" path instead assembles the nested `SequencerNodeConfig` the -node deserializes directly (loaded with `--config_format native`). +node deserializes directly from its `--config_file`(s). Pipeline: 1. Locate the per-layer `*sequencer_config.jsonnet` override files along the SAME overlay chain the diff --git a/deployments/sequencer/src/constructs/configmap.py b/deployments/sequencer/src/constructs/configmap.py index 4b087a20bb9..971dd3175d8 100644 --- a/deployments/sequencer/src/constructs/configmap.py +++ b/deployments/sequencer/src/constructs/configmap.py @@ -1,7 +1,6 @@ import json from imports import k8s -from src.config.loaders import NodeConfigLoader from src.config.native import build_native_config from src.constructs.base import BaseConstruct @@ -16,7 +15,6 @@ def __init__( monitoring_endpoint_port, layout: str, overlays: list[str], - config_format: str, ): super().__init__( scope, @@ -28,7 +26,6 @@ def __init__( self.layout = layout self.overlays = overlays - self.config_format = config_format self.config_map = self._get_config_map() def _get_config_map(self) -> k8s.KubeConfigMap: @@ -42,11 +39,7 @@ def _get_config_map(self) -> k8s.KubeConfigMap: f"config.configList is required for service '{self.service_config.name}' but was not provided" ) - if self.config_format == "native": - node_config = self._build_native_node_config() - else: - assert self.config_format == "preset" - node_config = self._build_preset_node_config() + node_config = self._build_native_node_config() config_data = json.dumps(node_config, indent=2) @@ -60,43 +53,6 @@ def _get_config_map(self) -> k8s.KubeConfigMap: data=dict(config=config_data), ) # Key is "config" to match node/ format, mounted as /config/sequencer/presets/config - def _build_preset_node_config(self) -> dict: - """Produce the flat dotted-key preset config by filling `$$$_..._$$$` placeholders.""" - # Load JSON configs using NodeConfigLoader - node_config_loader = NodeConfigLoader( - config_list_json_path=self.service_config.config.configList, - ) - node_config = node_config_loader.load() - - # sequencerConfig is now already merged from common into service_config - merged_sequencer_config = ( - self.service_config.config.sequencerConfig - if self.service_config.config and self.service_config.config.sequencerConfig - else {} - ) - - # Apply merged overrides (includes validation for both unused keys and remaining placeholders) - if merged_sequencer_config: - node_config = NodeConfigLoader.apply_sequencer_overrides( - node_config, - merged_sequencer_config, - service_name=self.service_config.name, - config_list_path=self.service_config.config.configList, - layout=self.layout, - overlays=self.overlays, - ) - else: - # If no sequencer config overrides, still validate for remaining placeholders - NodeConfigLoader.validate_no_remaining_placeholders( - node_config, - config_list_path=self.service_config.config.configList, - layout=self.layout, - overlays=self.overlays, - service_name=self.service_config.name, - ) - - return node_config - def _build_native_node_config(self) -> dict: """Produce the nested SequencerNodeConfig for this service via jsonnet `build()`. diff --git a/deployments/sequencer/src/constructs/deployment.py b/deployments/sequencer/src/constructs/deployment.py index 11c699eb4be..10475073380 100644 --- a/deployments/sequencer/src/constructs/deployment.py +++ b/deployments/sequencer/src/constructs/deployment.py @@ -11,7 +11,6 @@ def __init__( service_config, labels, monitoring_endpoint_port, - config_format: str, ): super().__init__( scope, @@ -21,7 +20,6 @@ def __init__( monitoring_endpoint_port, ) - self.config_format = config_format self.deployment = self._create_deployment() def _create_deployment(self) -> k8s.KubeDeployment: @@ -29,7 +27,6 @@ def _create_deployment(self) -> k8s.KubeDeployment: self.service_config, self.labels, self.monitoring_endpoint_port, - config_format=self.config_format, ) selector_labels = {"service": self.labels["service"]} diff --git a/deployments/sequencer/src/constructs/helpers/pod_builder.py b/deployments/sequencer/src/constructs/helpers/pod_builder.py index a7a748f907c..b9fe374e1d1 100644 --- a/deployments/sequencer/src/constructs/helpers/pod_builder.py +++ b/deployments/sequencer/src/constructs/helpers/pod_builder.py @@ -16,12 +16,10 @@ def __init__( service_config: ServiceConfig, labels: dict[str, str], monitoring_endpoint_port: int, - config_format: str, ): self.service_config = service_config self.labels = labels self.monitoring_endpoint_port = monitoring_endpoint_port - self.config_format = config_format def build_pod_spec(self) -> k8s.PodSpec: """Build a complete PodSpec with all necessary configurations.""" @@ -83,9 +81,12 @@ def _build_container(self) -> k8s.Container: ) def _build_container_args(self) -> list[str]: - """Build container arguments, always including --config_file with fixed file paths.""" - # First argument is the config format. - args = ["--config_format", self.config_format] + """Build container arguments, always including --config_file with fixed file paths. + + The node loads the nested native config directly from its `--config_file`(s); it takes no + config-format argument (see apollo_config command parser). + """ + args: list[str] = [] # Add --config_file /config/sequencer/presets/config (ConfigMap) # Note: node version uses directory mount, so path is just the directory + "config" diff --git a/deployments/sequencer/src/constructs/secret.py b/deployments/sequencer/src/constructs/secret.py index 56f8c83c9e5..faab05bf106 100644 --- a/deployments/sequencer/src/constructs/secret.py +++ b/deployments/sequencer/src/constructs/secret.py @@ -31,7 +31,7 @@ def _load_secret_file(self) -> dict: if not self.service_config.secret.file: return {} - # Resolve file path relative to project root (same as NodeConfigLoader) + # Resolve file path relative to project root. root_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../../") file_path = os.path.join(root_dir, self.service_config.secret.file) diff --git a/deployments/sequencer/src/constructs/statefulset.py b/deployments/sequencer/src/constructs/statefulset.py index f509ff50e16..1fdc9f0d87f 100644 --- a/deployments/sequencer/src/constructs/statefulset.py +++ b/deployments/sequencer/src/constructs/statefulset.py @@ -11,7 +11,6 @@ def __init__( service_config, labels, monitoring_endpoint_port, - config_format, ): super().__init__( scope, @@ -21,7 +20,6 @@ def __init__( monitoring_endpoint_port, ) - self.config_format = config_format self.statefulset = self._create_statefulset() def _build_update_strategy(self) -> k8s.StatefulSetUpdateStrategy: @@ -69,7 +67,6 @@ def _create_statefulset(self) -> k8s.KubeStatefulSet: self.service_config, statefulset_labels, self.monitoring_endpoint_port, - config_format=self.config_format, ) selector_labels = {"service": statefulset_labels["service"]}