mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-20 09:01:32 -08:00
428 lines
19 KiB
Python
428 lines
19 KiB
Python
# Copyright 2026 Element Creations Ltd
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
|
|
"""
|
|
Migration service.
|
|
"""
|
|
|
|
import base64
|
|
import copy
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from .extra_files import ExtraFilesDiscovery
|
|
from .interfaces import ExtraFilesDiscoveryStrategy, MigrationStrategy, SecretDiscoveryStrategy
|
|
from .models import ConfigMap, DiscoveredExtraFile, DiscoveredSecret, MigrationInput, Secret, TransformationSpec
|
|
from .secrets import SecretDiscovery
|
|
from .utils import (
|
|
get_nested_value,
|
|
remove_nested_value,
|
|
set_nested_value,
|
|
to_kebab_case,
|
|
yaml_dump_with_pipe_for_multiline,
|
|
)
|
|
|
|
logger = logging.getLogger("migration")
|
|
|
|
|
|
@dataclass
|
|
class TransformationResult:
|
|
"""
|
|
Result of a configuration transformation.
|
|
|
|
Contains the transformation specification along with the actual value.
|
|
"""
|
|
|
|
spec: TransformationSpec # Transformation specification
|
|
value: Any # Transformed value
|
|
|
|
|
|
@dataclass
|
|
class ConfigValueTransformer:
|
|
"""
|
|
Enhanced transformer for configuration values that handles the complete
|
|
transformation from source configuration to ESS format using explicit
|
|
source and target paths.
|
|
"""
|
|
|
|
pretty_logger: logging.Logger = field(init=True) # Pretty logger
|
|
ess_config: dict[str, Any] = field(init=True) # Target ESS configuration dictionary
|
|
results: list[TransformationResult] = field(default_factory=list) # List of transformation results
|
|
tracked_values: list[str] = field(default_factory=list) # Source paths that have been processed
|
|
|
|
def transform_from_config(self, source_config: dict[str, Any], transformations: list[TransformationSpec]) -> None:
|
|
"""
|
|
Transform values from a source configuration using explicit source and target path mappings.
|
|
|
|
Args:
|
|
source_config: Source configuration dictionary
|
|
transformations: List of TransformationSpec objects
|
|
"""
|
|
for transformation in transformations:
|
|
# Get the value from the source configuration
|
|
value = get_nested_value(source_config, transformation.src_key)
|
|
|
|
if value is not None:
|
|
# Apply transformer function if provided
|
|
transformed_value = value
|
|
if transformation.transformer is not None:
|
|
transformed_value = transformation.transformer(self.pretty_logger, value)
|
|
|
|
# Track the source path if not already tracked
|
|
if transformation.src_key not in self.tracked_values:
|
|
self.tracked_values.append(transformation.src_key)
|
|
|
|
# Create TransformationResult using the current transformation spec
|
|
result = TransformationResult(spec=transformation, value=transformed_value)
|
|
self.results.append(result)
|
|
|
|
# Set the transformed value in the ESS config
|
|
set_nested_value(self.ess_config, transformation.target_key, transformed_value)
|
|
elif transformation.required:
|
|
# If the transformation is required but the value is missing, raise an error
|
|
raise MigrationError(
|
|
f"Required configuration value '{transformation.src_key}' is missing from the source configuration"
|
|
)
|
|
|
|
def get_component_config(self, component_key: str) -> dict[str, Any]:
|
|
"""
|
|
Get the configuration for a specific component.
|
|
|
|
Args:
|
|
component_key: The component key (e.g., "synapse", "matrixAuthenticationService")
|
|
|
|
Returns:
|
|
Dictionary with the component configuration, or empty dict if not found
|
|
"""
|
|
return self.ess_config.get(component_key, {})
|
|
|
|
def filter_config(self, config: dict[str, Any]) -> dict[str, Any]:
|
|
"""
|
|
Create a filtered copy of the config that removes tracked values.
|
|
|
|
Args:
|
|
config: Original configuration
|
|
|
|
Returns:
|
|
Filtered configuration with tracked values removed
|
|
"""
|
|
filtered_config = copy.deepcopy(config)
|
|
|
|
for source_path in self.tracked_values:
|
|
remove_nested_value(filtered_config, source_path)
|
|
|
|
return filtered_config
|
|
|
|
def add_additional_config_to_component(
|
|
self, component_key: str, source_config: dict[str, Any], extra_files_discovery: ExtraFilesDiscovery | None
|
|
) -> None:
|
|
"""
|
|
Add filtered additional configurations to a specific component in the ESS config.
|
|
|
|
This method filters the source configuration to remove values that are passed to ESS,
|
|
then adds the filtered configuration to the component's 'additional' section.
|
|
|
|
Args:
|
|
component_key: The component key (e.g., "synapse", "matrixAuthenticationService")
|
|
source_config: Original configuration to filter and add
|
|
"""
|
|
# Get or create the component config in the ESS config
|
|
component_config = self.ess_config.setdefault(component_key, {})
|
|
filtered_config = copy.deepcopy(source_config)
|
|
# Update references to extra files
|
|
if extra_files_discovery:
|
|
filtered_config = self.update_paths_in_config(filtered_config, extra_files_discovery, component_key)
|
|
|
|
# Filter the source config to remove tracked values
|
|
filtered_config = self.filter_config(filtered_config)
|
|
|
|
# Add to additional section if there's anything to add
|
|
if filtered_config:
|
|
component_config["additional"] = {
|
|
"00-imported.yaml": {"config": yaml_dump_with_pipe_for_multiline(filtered_config)}
|
|
}
|
|
|
|
def update_paths_in_config(
|
|
self,
|
|
source_config: dict[str, Any],
|
|
extra_files_discovery: ExtraFilesDiscovery,
|
|
component_root_key: str,
|
|
):
|
|
# Get the base mount path for the component
|
|
base_mount_path = f"/etc/{component_root_key}/extra"
|
|
updated_config = copy.deepcopy(source_config)
|
|
for discovered_path in extra_files_discovery.discovered_file_paths:
|
|
if discovered_path.skipped_reason:
|
|
self.tracked_values.append(discovered_path.config_key)
|
|
continue
|
|
# If it is a directory, files will be mounted as child of the directory name
|
|
# If it is a file, files will be mounted as child of the `extra` folder
|
|
mounted_path = f"{base_mount_path}/{discovered_path.source_path.name}"
|
|
original_value = get_nested_value(updated_config, discovered_path.config_key)
|
|
set_nested_value(updated_config, discovered_path.config_key, mounted_path)
|
|
logging.info(f"Updated config: {discovered_path.config_key} = {original_value} -> {mounted_path}")
|
|
return updated_config
|
|
|
|
def handle_secrets(
|
|
self,
|
|
secret_discovery: SecretDiscovery,
|
|
component_root_key: str,
|
|
secrets_list: list[Secret],
|
|
) -> None:
|
|
"""
|
|
Handle secrets for component using SecretDiscovery.
|
|
|
|
Args:
|
|
config: Configuration to analyze for secrets
|
|
secret_service: SecretDiscoveryService instance
|
|
component_root_key: Name of the component (synapse or mas)
|
|
secrets_list: List to append created secrets to
|
|
|
|
Returns:
|
|
ESS configuration dictionary with credential schema
|
|
"""
|
|
# Skip if secret_service is None or no secrets discovered
|
|
if secret_discovery is None or not secret_discovery.discovered_secrets:
|
|
return
|
|
|
|
# Create a Kubernetes Secret containing all discovered secrets
|
|
# Convert component_root_key to kebab-case for the secret name
|
|
# Replace camelCase with kebab-case (e.g., "matrixAuthenticationService" -> "matrix-authentication-service")
|
|
kebab_case_name = component_root_key
|
|
# Insert hyphens before uppercase letters and convert to lowercase
|
|
kebab_case_name = "".join(["-" + c.lower() if c.isupper() else c for c in kebab_case_name]).lstrip("-")
|
|
|
|
secret_name = f"imported-{kebab_case_name}"
|
|
secret_data = {}
|
|
|
|
for secret_key, discover_secret in secret_discovery.discovered_secrets.items():
|
|
# Base64 encode the secret value for Kubernetes Secret
|
|
encoded_value = base64.b64encode(discover_secret.value.encode("utf-8")).decode("utf-8")
|
|
secret_data[secret_key] = encoded_value
|
|
|
|
secret = Secret(name=secret_name, data=secret_data)
|
|
secrets_list.append(secret)
|
|
logging.info(
|
|
f"Created Kubernetes Secret with {len(secret_discovery.discovered_secrets)}"
|
|
f" secrets for {component_root_key}"
|
|
)
|
|
|
|
# Update ESS values to use credential schema instead of direct values
|
|
# Use the ess_secret_schema to map secret keys to ESS configuration paths
|
|
for secret_key, discover_secret in secret_discovery.discovered_secrets.items():
|
|
# Convert secret key to credential schema format
|
|
# Example: secret -> {"secret": "imported-synapse", "secretKey": "synapse.postgres.password"}
|
|
credential_config = {"secret": secret.name, "secretKey": secret_key}
|
|
|
|
# Get the secret configuration from the schema
|
|
secret_config = secret_discovery.strategy.ess_secret_schema.get(secret_key)
|
|
if secret_config is None:
|
|
raise RuntimeError(f"No ESS configuration mapping found for secret key: {secret_key}")
|
|
|
|
# Set the credential config in the ESS config under the component section
|
|
set_nested_value(self.ess_config, discover_secret.secret_key, credential_config)
|
|
|
|
# Track the config value that is being passed to ESS
|
|
# We need to track the original config path so it gets filtered out later
|
|
self.tracked_values.append(discover_secret.config_key)
|
|
|
|
def handle_extra_files_mounts(
|
|
self,
|
|
extra_files_discovery: ExtraFilesDiscovery,
|
|
component_root_key: str,
|
|
config_maps: list[ConfigMap],
|
|
):
|
|
config = self.get_component_config(component_root_key)
|
|
# Get the base mount path for the component
|
|
base_mount_path = f"/etc/{component_root_key}/extra"
|
|
|
|
configmap_name = f"imported-{to_kebab_case(component_root_key)}"
|
|
extra_volume_mounts: list[dict[str, Any]] = []
|
|
extra_volumes: list[dict[str, Any]] = []
|
|
|
|
configmap_data = {}
|
|
|
|
for discovered_extra_file in extra_files_discovery.discovered_extra_files.values():
|
|
# Encode the file content
|
|
configmap_data[discovered_extra_file.filename] = discovered_extra_file.content.decode("utf-8")
|
|
|
|
configmap = ConfigMap(name=configmap_name, data=configmap_data)
|
|
|
|
for extra_file in extra_files_discovery.discovered_extra_files.values():
|
|
for discovered_path in extra_file.discovered_source_paths:
|
|
if discovered_path.skipped_reason:
|
|
continue
|
|
|
|
# See update_paths_in_config() for the `additional` side of this behaviour
|
|
# If it is a directory, files must be mounted as child of the directory name
|
|
# If it is a file, files must be mounted as child of the `extra` folder
|
|
if discovered_path.is_dir:
|
|
mounted_path = f"{base_mount_path}/{discovered_path.source_path.name}/{extra_file.filename}"
|
|
else:
|
|
mounted_path = f"{base_mount_path}/{extra_file.filename}"
|
|
extra_volume_mounts.append(
|
|
{"name": configmap_name, "mountPath": mounted_path, "subPath": extra_file.filename}
|
|
)
|
|
if extra_volume_mounts:
|
|
extra_volumes.append(
|
|
{
|
|
"name": configmap_name,
|
|
"configMap": {"name": configmap_name},
|
|
}
|
|
)
|
|
config_maps.append(configmap)
|
|
|
|
extra_volume_mounts.sort(key=lambda x: x["mountPath"])
|
|
|
|
if extra_volumes:
|
|
config["extraVolumes"] = extra_volumes
|
|
config["extraVolumeMounts"] = extra_volume_mounts
|
|
|
|
|
|
class MigrationError(Exception):
|
|
"""Base exception for migration-related errors."""
|
|
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class MigrationService:
|
|
"""Migration service."""
|
|
|
|
input: MigrationInput = field(init=True) # Migration input data
|
|
pretty_logger: logging.Logger = field(init=True) # Pretty logger
|
|
ess_config: dict[str, Any] = field(init=True) # Target ESS configuration
|
|
migration: MigrationStrategy = field(init=True) # Migration strategy
|
|
extra_files_strategy: ExtraFilesDiscoveryStrategy = field(init=True) # Extra files discovery service
|
|
secret_discovery_strategy: SecretDiscoveryStrategy = field(init=True) # Secret discovery service
|
|
override_warnings: list[str] = field(default_factory=list) # Warnings about overridden configurations
|
|
init_by_ess_secrets: list[str] = field(default_factory=list) # List of secrets that will be initialized by ESS
|
|
discovered_secrets: list[DiscoveredSecret] = field(default_factory=list) # List of discovered secrets
|
|
discovered_extra_files: list[DiscoveredExtraFile] = field(default_factory=list) # List of discovered secrets
|
|
secrets: list[Secret] = field(default_factory=list) # List of created Secrets
|
|
configmaps: list[ConfigMap] = field(default_factory=list) # List of created ConfigMaps
|
|
override_configs: set[str] = field(default_factory=set) # Set of configurations that are managed by ESS
|
|
component_root_key: str = field(init=False) # Root key for the component (e.g., 'synapse')
|
|
results: list[TransformationResult] = field(default_factory=list) # List of transformation results
|
|
|
|
def __post_init__(self):
|
|
self.component_root_key = self.migration.component_root_key
|
|
self.override_configs = self.migration.override_configs
|
|
|
|
def _check_overrides(self, config: dict[str, Any]) -> None:
|
|
"""
|
|
Check if the configuration contains any override configurations
|
|
that are managed by ESS and cannot be overridden.
|
|
|
|
Only shows overrides that are managed by ESS chart (no direct transformation).
|
|
Settings with transformations are already shown in "Successfully migrated" section.
|
|
|
|
Args:
|
|
config: Configuration to check
|
|
"""
|
|
override_warnings = []
|
|
|
|
# Build set of transformed config paths for filtering
|
|
transformed_configs = set()
|
|
for transformation in self.migration.transformations:
|
|
transformed_configs.add(transformation.src_key)
|
|
|
|
config_to_ess_transformer = ConfigValueTransformer(self.pretty_logger, self.ess_config)
|
|
|
|
# Use the filtered configuration (with migrated values removed) for override detection
|
|
# This automatically excludes values that are tracked by the transformer,
|
|
# including both regular transformations and secrets (which are added to tracked_values during handle_secrets)
|
|
filtered_config = config_to_ess_transformer.filter_config(config)
|
|
|
|
for override_config in self.override_configs:
|
|
# Check if the override config path exists in the filtered configuration
|
|
# and is managed by ESS chart (no direct transformation)
|
|
if (
|
|
get_nested_value(filtered_config, override_config) is not None
|
|
and override_config not in transformed_configs
|
|
):
|
|
override_warnings.append(
|
|
f"⚠️ '{override_config}' found in {self.component_root_key}.additional[\"00-imported.yaml\"].config"
|
|
)
|
|
|
|
if override_warnings:
|
|
self.override_warnings.extend(override_warnings)
|
|
logger.warning(f"{self.component_root_key} configuration contains ESS-managed overrides:")
|
|
for warning in override_warnings:
|
|
logger.warning(warning)
|
|
|
|
def migrate(self) -> None:
|
|
"""
|
|
Perform migration using the injected strategy.
|
|
|
|
Migration steps:
|
|
1. Enable component
|
|
1. Apply component transformations
|
|
2. Check for override configurations
|
|
3. Add filtered additional configurations
|
|
"""
|
|
# Step 1: Discover secrets
|
|
secret_discovery = SecretDiscovery(
|
|
strategy=self.secret_discovery_strategy,
|
|
source_file=self.input.config_path,
|
|
pretty_logger=self.pretty_logger,
|
|
)
|
|
secret_discovery.discover_secrets(self.input.config)
|
|
# Prompt for missing secrets then validate
|
|
secret_discovery.prompt_for_missing_secrets()
|
|
secret_discovery.validate_required_secrets()
|
|
self.discovered_secrets = list(secret_discovery.discovered_secrets.values())
|
|
self.init_by_ess_secrets = secret_discovery.init_by_ess_secrets
|
|
|
|
# Step 2: Discover extra files
|
|
extra_files_discovery = ExtraFilesDiscovery(
|
|
source_file=self.input.config_path,
|
|
strategy=self.extra_files_strategy,
|
|
secrets_strategy=self.secret_discovery_strategy,
|
|
pretty_logger=self.pretty_logger,
|
|
)
|
|
extra_files_discovery.discover_extra_files_from_config(self.input.config)
|
|
# Prompt for missing files then validate
|
|
extra_files_discovery.prompt_for_missing_files()
|
|
extra_files_discovery.validate_extra_files()
|
|
|
|
self.discovered_file_paths = extra_files_discovery.discovered_file_paths
|
|
self.missing_file_paths = extra_files_discovery.missing_file_paths
|
|
|
|
# Step 3: Enable component
|
|
self.ess_config.setdefault(self.component_root_key, {})["enabled"] = True
|
|
|
|
config_to_ess_transformer = ConfigValueTransformer(self.pretty_logger, self.ess_config)
|
|
# Step 4: Apply component transformations
|
|
config_to_ess_transformer.transform_from_config(self.input.config, self.migration.transformations)
|
|
|
|
# Step 5: Handle secrets for the component using the transformer's method
|
|
# This will update the root ESS config directly and create Kubernetes Secrets
|
|
config_to_ess_transformer.handle_secrets(
|
|
secret_discovery,
|
|
self.component_root_key,
|
|
self.secrets,
|
|
)
|
|
|
|
# Step 6: Handle extra files mounts for the component using the transformer's method
|
|
# This will update the root ESS config directly and create Kubernetes ConfigMaps
|
|
config_to_ess_transformer.handle_extra_files_mounts(
|
|
extra_files_discovery,
|
|
self.component_root_key,
|
|
self.configmaps,
|
|
)
|
|
|
|
# Step 7: Check for override configurations and warn user
|
|
self._check_overrides(self.input.config)
|
|
|
|
# Step 8: Add filtered additional configurations
|
|
config_to_ess_transformer.add_additional_config_to_component(
|
|
self.component_root_key, self.input.config, extra_files_discovery
|
|
)
|
|
|
|
# Step 9: Store results
|
|
self.results.extend(config_to_ess_transformer.results)
|