mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-20 17:11:31 -08:00
329 lines
14 KiB
Python
329 lines
14 KiB
Python
# Copyright 2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
|
|
"""
|
|
Migration service.
|
|
"""
|
|
|
|
import base64
|
|
import logging
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from .interfaces import MigrationStrategy
|
|
from .models import DiscoveredSecret, MigrationInput, Secret
|
|
from .secrets import SecretDiscovery, SecretDiscoveryStrategy
|
|
from .utils import (
|
|
get_nested_value,
|
|
remove_nested_value,
|
|
set_nested_value,
|
|
yaml_dump_with_pipe_for_multiline,
|
|
)
|
|
|
|
logger = logging.getLogger("migration")
|
|
|
|
|
|
@dataclass
|
|
class TransformationSpec:
|
|
"""
|
|
Specification for a configuration transformation.
|
|
|
|
Defines how to map from a source configuration path to a target ESS path.
|
|
"""
|
|
|
|
src_key: str # Source configuration path
|
|
target_key: str # Target ESS configuration path
|
|
required: bool = True # Whether this transformation is required
|
|
transformer: Callable[[Any], Any] | None = None # Optional transformation function
|
|
|
|
|
|
@dataclass
|
|
class TransformationResult:
|
|
"""
|
|
Result of a configuration transformation.
|
|
|
|
Contains the transformation specification along with the actual value.
|
|
"""
|
|
|
|
spec: TransformationSpec # Transformation specification
|
|
value: Any # Transformed value
|
|
|
|
|
|
@dataclass
|
|
class ConfigValueTransformer:
|
|
"""
|
|
Enhanced transformer for configuration values that handles the complete
|
|
transformation from source configuration to ESS format using explicit
|
|
source and target paths.
|
|
"""
|
|
|
|
results: list[TransformationResult] = field(default_factory=list) # List of transformation results
|
|
tracked_values: list[str] = field(default_factory=list) # Source paths that have been processed
|
|
ess_config: dict[str, Any] = field(default_factory=dict) # Target ESS configuration dictionary
|
|
|
|
def transform_from_config(self, source_config: dict[str, Any], transformations: list[TransformationSpec]) -> None:
|
|
"""
|
|
Transform values from a source configuration using explicit source and target path mappings.
|
|
|
|
Args:
|
|
source_config: Source configuration dictionary
|
|
transformations: List of TransformationSpec objects
|
|
"""
|
|
for transformation in transformations:
|
|
# Get the value from the source configuration
|
|
value = get_nested_value(source_config, transformation.src_key)
|
|
|
|
if value is not None:
|
|
# Apply transformer function if provided
|
|
transformed_value = value
|
|
if transformation.transformer is not None:
|
|
transformed_value = transformation.transformer(value)
|
|
|
|
# Track the source path if not already tracked
|
|
if transformation.src_key not in self.tracked_values:
|
|
self.tracked_values.append(transformation.src_key)
|
|
|
|
# Create TransformationResult using the current transformation spec
|
|
result = TransformationResult(spec=transformation, value=transformed_value)
|
|
self.results.append(result)
|
|
|
|
# Set the transformed value in the ESS config
|
|
set_nested_value(self.ess_config, transformation.target_key, transformed_value)
|
|
elif transformation.required:
|
|
# If the transformation is required but the value is missing, raise an error
|
|
raise MigrationError(
|
|
f"Required configuration value '{transformation.src_key}' is missing from the source configuration"
|
|
)
|
|
|
|
def get_component_config(self, component_key: str) -> dict[str, Any]:
|
|
"""
|
|
Get the configuration for a specific component.
|
|
|
|
Args:
|
|
component_key: The component key (e.g., "synapse", "matrixAuthenticationService")
|
|
|
|
Returns:
|
|
Dictionary with the component configuration, or empty dict if not found
|
|
"""
|
|
return self.ess_config.get(component_key, {})
|
|
|
|
def filter_config(self, config: dict[str, Any]) -> dict[str, Any]:
|
|
"""
|
|
Create a filtered copy of the config that removes tracked values.
|
|
|
|
Args:
|
|
config: Original configuration
|
|
|
|
Returns:
|
|
Filtered configuration with tracked values removed
|
|
"""
|
|
import copy
|
|
|
|
filtered_config = copy.deepcopy(config)
|
|
|
|
for source_path in self.tracked_values:
|
|
remove_nested_value(filtered_config, source_path)
|
|
|
|
return filtered_config
|
|
|
|
def add_additional_config_to_component(self, component_key: str, source_config: dict[str, Any]) -> None:
|
|
"""
|
|
Add filtered additional configurations to a specific component in the ESS config.
|
|
|
|
This method filters the source configuration to remove values that are passed to ESS,
|
|
then adds the filtered configuration to the component's 'additional' section.
|
|
|
|
Args:
|
|
component_key: The component key (e.g., "synapse", "matrixAuthenticationService")
|
|
source_config: Original configuration to filter and add
|
|
"""
|
|
# Get or create the component config in the ESS config
|
|
component_config = self.ess_config.setdefault(component_key, {})
|
|
|
|
# Filter the source config to remove tracked values
|
|
filtered_config = self.filter_config(source_config)
|
|
|
|
# Add to additional section if there's anything to add
|
|
if filtered_config:
|
|
component_config["additional"] = {
|
|
"00-imported.yaml": {"config": yaml_dump_with_pipe_for_multiline(filtered_config)}
|
|
}
|
|
|
|
def handle_secrets(
|
|
self,
|
|
secret_discovery: SecretDiscovery,
|
|
component_root_key: str,
|
|
secrets_list: list[Secret],
|
|
) -> None:
|
|
"""
|
|
Handle secrets for component using SecretDiscovery.
|
|
|
|
Args:
|
|
config: Configuration to analyze for secrets
|
|
secret_service: SecretDiscoveryService instance
|
|
component_root_key: Name of the component (synapse or mas)
|
|
secrets_list: List to append created secrets to
|
|
|
|
Returns:
|
|
ESS configuration dictionary with credential schema
|
|
"""
|
|
# Skip if secret_service is None or no secrets discovered
|
|
if secret_discovery is None or not secret_discovery.discovered_secrets:
|
|
return
|
|
|
|
# Create a Kubernetes Secret containing all discovered secrets
|
|
# Convert component_root_key to kebab-case for the secret name
|
|
# Replace camelCase with kebab-case (e.g., "matrixAuthenticationService" -> "matrix-authentication-service")
|
|
kebab_case_name = component_root_key
|
|
# Insert hyphens before uppercase letters and convert to lowercase
|
|
kebab_case_name = "".join(["-" + c.lower() if c.isupper() else c for c in kebab_case_name]).lstrip("-")
|
|
|
|
secret_name = f"imported-{kebab_case_name}"
|
|
secret_data = {}
|
|
|
|
for secret_key, discover_secret in secret_discovery.discovered_secrets.items():
|
|
# Base64 encode the secret value for Kubernetes Secret
|
|
encoded_value = base64.b64encode(discover_secret.value.encode("utf-8")).decode("utf-8")
|
|
secret_data[secret_key] = encoded_value
|
|
|
|
secret = Secret(name=secret_name, data=secret_data)
|
|
secrets_list.append(secret)
|
|
logging.info(
|
|
f"Created Kubernetes Secret with {len(secret_discovery.discovered_secrets)}"
|
|
f" secrets for {component_root_key}"
|
|
)
|
|
|
|
# Update ESS values to use credential schema instead of direct values
|
|
# Use the ess_secret_schema to map secret keys to ESS configuration paths
|
|
for secret_key, discover_secret in secret_discovery.discovered_secrets.items():
|
|
# Convert secret key to credential schema format
|
|
# Example: secret -> {"secret": "imported-synapse", "secretKey": "synapse.postgres.password"}
|
|
credential_config = {"secret": secret.name, "secretKey": secret_key}
|
|
|
|
# Get the secret configuration from the schema
|
|
secret_config = secret_discovery.strategy.ess_secret_schema.get(secret_key)
|
|
if secret_config is None:
|
|
raise RuntimeError(f"No ESS configuration mapping found for secret key: {secret_key}")
|
|
|
|
# Set the credential config in the ESS config under the component section
|
|
set_nested_value(self.ess_config, discover_secret.secret_key, credential_config)
|
|
|
|
# Track the config value that is being passed to ESS
|
|
# We need to track the original config path so it gets filtered out later
|
|
self.tracked_values.append(discover_secret.secret_key)
|
|
|
|
|
|
class MigrationError(Exception):
|
|
"""Base exception for migration-related errors."""
|
|
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class MigrationService:
|
|
"""Migration service."""
|
|
|
|
input: MigrationInput = field(init=True) # Migration input data
|
|
pretty_logger: logging.Logger = field(init=True) # Pretty logger
|
|
ess_config: dict[str, Any] = field(init=True) # Target ESS configuration
|
|
migration: MigrationStrategy = field(init=True) # Migration strategy
|
|
secret_discovery_strategy: SecretDiscoveryStrategy = field(init=True) # Secret discovery service
|
|
override_warnings: list[str] = field(default_factory=list) # Warnings about overridden configurations
|
|
init_by_ess_secrets: list[str] = field(default_factory=list) # List of secrets that will be initialized by ESS
|
|
discovered_secrets: list[DiscoveredSecret] = field(default_factory=list) # List of discovered secrets
|
|
secrets: list[Secret] = field(default_factory=list) # List of created Secrets
|
|
override_configs: set[str] = field(default_factory=set) # Set of configurations that are managed by ESS
|
|
component_root_key: str = field(init=False) # Root key for the component (e.g., 'synapse')
|
|
config_to_ess_transformer: ConfigValueTransformer = field(default_factory=ConfigValueTransformer) # Config
|
|
|
|
def __post_init__(self):
|
|
self.component_root_key = self.migration.component_root_key
|
|
self.override_configs = self.migration.override_configs
|
|
self.config_to_ess_transformer.ess_config = self.ess_config
|
|
|
|
def _check_overrides(self, config: dict[str, Any]) -> None:
|
|
"""
|
|
Check if the configuration contains any override configurations
|
|
that are managed by ESS and cannot be overridden.
|
|
|
|
Only shows overrides that are managed by ESS chart (no direct transformation).
|
|
Settings with transformations are already shown in "Successfully migrated" section.
|
|
|
|
Args:
|
|
config: Configuration to check
|
|
"""
|
|
override_warnings = []
|
|
|
|
# Build set of transformed config paths for filtering
|
|
transformed_configs = set()
|
|
for transformation in self.migration.transformations:
|
|
transformed_configs.add(transformation.src_key)
|
|
|
|
# Use the filtered configuration (with migrated values removed) for override detection
|
|
# This automatically excludes values that are tracked by the transformer,
|
|
# including both regular transformations and secrets (which are added to tracked_values during handle_secrets)
|
|
filtered_config = self.config_to_ess_transformer.filter_config(config)
|
|
|
|
for override_config in self.override_configs:
|
|
# Check if the override config path exists in the filtered configuration
|
|
# and is managed by ESS chart (no direct transformation)
|
|
if (
|
|
get_nested_value(filtered_config, override_config) is not None
|
|
and override_config not in transformed_configs
|
|
):
|
|
override_warnings.append(
|
|
f"⚠️ '{override_config}' found in {self.component_root_key}.additional[\"00-imported.yaml\"].config"
|
|
)
|
|
|
|
if override_warnings:
|
|
self.override_warnings.extend(override_warnings)
|
|
logger.warning(f"{self.component_root_key} configuration contains ESS-managed overrides:")
|
|
for warning in override_warnings:
|
|
logger.warning(warning)
|
|
|
|
def migrate(self) -> None:
|
|
"""
|
|
Perform migration using the injected strategy.
|
|
|
|
Migration steps:
|
|
1. Enable component
|
|
1. Apply component transformations
|
|
2. Check for override configurations
|
|
3. Add filtered additional configurations
|
|
"""
|
|
# Step 1: Discover secrets
|
|
secret_discovery = SecretDiscovery(
|
|
strategy=self.secret_discovery_strategy,
|
|
source_file=self.input.config_path,
|
|
pretty_logger=self.pretty_logger,
|
|
)
|
|
secret_discovery.discover_secrets(self.input.config)
|
|
# Prompt for missing secrets then validate
|
|
secret_discovery.prompt_for_missing_secrets()
|
|
secret_discovery.validate_required_secrets()
|
|
self.discovered_secrets = list(secret_discovery.discovered_secrets.values())
|
|
self.init_by_ess_secrets = secret_discovery.init_by_ess_secrets
|
|
|
|
# Step 2: Enable component
|
|
self.ess_config.setdefault(self.component_root_key, {})["enabled"] = True
|
|
|
|
# Step 3: Apply component transformations
|
|
self.config_to_ess_transformer.transform_from_config(self.input.config, self.migration.transformations)
|
|
|
|
# Step 4: Handle secrets for the component using the transformer's method
|
|
# This will update the root ESS config directly and create Kubernetes Secrets
|
|
self.config_to_ess_transformer.handle_secrets(
|
|
secret_discovery,
|
|
self.component_root_key,
|
|
self.secrets,
|
|
)
|
|
|
|
# Step 5: Check for override configurations and warn user
|
|
self._check_overrides(self.input.config)
|
|
|
|
# Step 6: Add filtered additional configurations (not from transformation)
|
|
self.config_to_ess_transformer.add_additional_config_to_component(self.component_root_key, self.input.config)
|