# Copyright 2026 Element Creations Ltd # # SPDX-License-Identifier: AGPL-3.0-only """ Extra files handling module for the migration script. Handles discovery, reading, and validation of extra files from configuration files. """ import logging import re from dataclasses import dataclass, field from pathlib import Path from urllib.parse import urlparse from .interfaces import ExtraFilesDiscoveryStrategy, SecretDiscoveryStrategy from .models import DiscoveredExtraFile, DiscoveredPath, SecretConfig logger = logging.getLogger("migration") class ExtraFilesError(Exception): """Base exception for extra files-related errors.""" pass def ess_schema_config_key_secret_paths(ess_schema: dict[str, SecretConfig]): return [secret_config.config_path for secret_config in ess_schema.values() if secret_config.config_path is not None] @dataclass class ExtraFilesDiscovery: """Handles discovery and extraction of extra files from configuration files.""" pretty_logger: logging.Logger = field(init=True) strategy: ExtraFilesDiscoveryStrategy = field(init=True) # Strategy for component-specific secret discovery secrets_strategy: SecretDiscoveryStrategy = field(init=True) # Strategy for component-specific secret discovery source_file: str = field(init=True) # Source file path discovered_extra_files: dict[Path, DiscoveredExtraFile] = field( default_factory=dict ) # Discovered Extra File to mount in ESS discovered_file_paths: list[DiscoveredPath] = field(default_factory=list) # File paths discovered in configuration skipped_config_keys: list[str] = field(default_factory=list) # Configuration keys that are skipped @property def missing_file_paths(self) -> list[DiscoveredPath]: """ Get a list of missing file paths. """ return [ fp for fp in self.discovered_file_paths if not fp.skipped_reason and fp not in [ discovered_source_path for exf in self.discovered_extra_files.values() for discovered_source_path in exf.discovered_source_paths ] ] def discover_extra_files_from_config(self, config_data: dict) -> None: """ Discover extra file paths from configuration data. Args: config_data: Configuration dictionary to analyze ignored_config_keys: List of configuration keys where file paths should be ignored """ logger.info("Discovering extra files configuration") # Check for common file path patterns in the config dict self._discover_file_paths_from_list_or_dict(config_data) # Run an initial lookup of files at the paths discovered self._discover_extra_files() def _discover_file_paths_from_list_or_dict(self, config_data: dict | list, parent_key: str = "") -> None: """ Recursively discover file paths from dictionary data. Args: config_data: Configuration dictionary to analyze parent_key: Parent key path for nested configurations """ if isinstance(config_data, list): # Build the full key path if not parent_key: raise RuntimeError("ESS Migration does not handle source file with only a single list at root") for i, value in enumerate(config_data): full_key = f"{parent_key}.{i}" if isinstance(value, str) and self._is_file_path(value): # This is a regular extra file self.discovered_file_paths.append( DiscoveredPath( config_key=full_key, source_file=self.source_file, source_path=Path(value), ) ) elif isinstance(value, (dict, list)): # Recursively check nested dictionaries with updated key path self._discover_file_paths_from_list_or_dict(value, full_key) else: for key, value in config_data.items(): # Build the full key path full_key = f"{parent_key}.{key}" if parent_key else key if isinstance(value, str) and self._is_file_path(value): skipped_reason = None # Check if this key exactly matches any secret configuration paths if full_key in ess_schema_config_key_secret_paths(self.secrets_strategy.ess_secret_schema): skipped_reason = f"{key} = {value} is already managed by secret" logger.info(f"Skipping {key} = {value}") elif full_key in self.strategy.ignored_config_keys: skipped_reason = f"{key} = {value} contains file we do not want to import in a ConfigMap" logger.info(f"Skipping {key} = {value}") # This is a regular extra file self.discovered_file_paths.append( DiscoveredPath( config_key=full_key, source_file=self.source_file, source_path=Path(value), skipped_reason=skipped_reason, ) ) elif isinstance(value, (dict, list)): # Recursively check nested dictionaries with updated key path self._discover_file_paths_from_list_or_dict(value, full_key) def _is_file_path(self, value: str) -> bool: """ Check if a string value looks like a file path. Args: value: String value to check Returns: True if the value looks like a file path """ # is the value has a scheme, skip it if urlparse(value).scheme: return False # Check for common file path patterns patterns = [ r"^/[^/].*", # Absolute path starting with / r"\./.*", # Relative path starting with ./ r"\.\./.*", # Relative path starting with ../ r"~/.+", # Home directory path ] return any(re.match(pattern, value) for pattern in patterns) def _discover_extra_files(self) -> None: """ Discover extra files from discovered file paths. """ logger.info("Discovering extra files") for _discovered_path in self.discovered_file_paths: if _discovered_path.skipped_reason: continue if _discovered_path.source_path.is_dir(): files_in_dir = self._handle_directory(_discovered_path) logger.info(f"Found {len(files_in_dir)} files in directory {_discovered_path.source_path}") self.pretty_logger.info( f"šŸ“ Found {len(files_in_dir)} files in directory: {_discovered_path.source_path}" ) # Show the files being imported for file_path in files_in_dir: logger.info(f" - {file_path}") self.pretty_logger.info(f" šŸ“„ {Path(file_path).name}") if not files_in_dir: logger.warning(f"No files found in directory: {_discovered_path.source_path}") self.pretty_logger.info(f"āš ļø No files found in directory: {_discovered_path.source_path}") else: if _discovered_path.source_path not in self.discovered_extra_files: try: # If the file is not matching an existing extra file, add it to the discovered extra files self.discovered_extra_files[_discovered_path.source_path] = self._discover_extra_file( _discovered_path ) except ExtraFilesError as e: logger.error(f"Failed to process file {_discovered_path.source_path}: {e}") def _handle_directory(self, discovered_path: DiscoveredPath, override_path: Path | None = None) -> list[str]: """ Handle directory by importing files non-recursively. Args: discovered_path: Path to the directory to process """ logger.info(f"Processing directory: {discovered_path.source_path}") # Get all files in the directory (non-recursive) discovered_path.is_dir = True try: files_in_dir = [] list_path = Path(discovered_path.source_path) if override_path: list_path = override_path for item in list_path.iterdir(): if item.is_file(): files_in_dir.append(str(item)) try: extra_file = self._discover_extra_file(discovered_path, item) if extra_file: self.discovered_extra_files[item] = extra_file except ExtraFilesError as e: logger.error(f"Failed to process file {item}: {e}") return files_in_dir except Exception as e: logger.error(f"Failed to process directory {discovered_path}: {e}") raise ExtraFilesError(f"Failed to process directory {discovered_path}: {e}") from e def prompt_for_missing_files(self) -> None: """ Prompt user for alternative paths when files are missing. """ self.pretty_logger.info("\n" + "=" * 60) self.pretty_logger.info("šŸ“ EXTRA FILES DISCOVERY") self.pretty_logger.info("=" * 60) if self.missing_file_paths: self.pretty_logger.info("The following extra files were referenced in your configuration") self.pretty_logger.info("but could not be found:") for file_path in self.missing_file_paths: self.pretty_logger.info(f"šŸ“ Missing: {file_path.config_key} ({file_path.source_path})") self.pretty_logger.info("\nšŸ” Would you like to:") self.pretty_logger.info(" 1. Provide alternative path for this file") self.pretty_logger.info(" 2. Skip these files and continue") self.pretty_logger.info(" 3. Provide a directory to search for files") while True: try: choice = input(" Enter your choice (1/2/3): ").strip() if choice == "1": # Provide alternative paths self._prompt_for_individual_alternatives(file_path) break elif choice == "2": # Skip missing files self.pretty_logger.info(" āš ļø Skipping missing files...") file_path.skipped_reason = "Skipped by user" break elif choice == "3": # Provide directory to search self._prompt_for_directory_search(file_path) break else: self.pretty_logger.info(" āŒ Invalid choice. Please enter 1, 2, or 3.") except KeyboardInterrupt as err: self.pretty_logger.info("\n āŒ Operation cancelled by user") raise ExtraFilesError("User cancelled file input") from err except EOFError as err: self.pretty_logger.info("\n āŒ End of input reached") raise ExtraFilesError("End of input reached during file prompt") from err def _prompt_for_individual_alternatives(self, discovered_path: DiscoveredPath) -> None: """ Prompt user for alternative paths for each missing file. """ while True: try: new_path = input(" Please enter the correct file path (or 'skip' to ignore): ").strip() if new_path.lower() == "skip": self.pretty_logger.info(f" āš ļø Skipping file: {discovered_path.source_path}") break if not new_path: self.pretty_logger.info(" āŒ File path cannot be empty. Please try again.") continue # Validate the new path try: new_path_obj = Path(new_path) if new_path_obj.is_dir(): raise ExtraFilesError("Targetted file is a directory.") else: discovered_extra_file = self._discover_extra_file(discovered_path, new_path_obj) if discovered_extra_file: self.discovered_extra_files[new_path_obj] = discovered_extra_file self.pretty_logger.info(f" āœ… File validated: {new_path}") break except ExtraFilesError as e: self.pretty_logger.info(f" āŒ Invalid file path: {e}") self.pretty_logger.info(" Please try again or enter 'skip' to ignore.") except KeyboardInterrupt as err: self.pretty_logger.info("\n āŒ Operation cancelled by user") raise ExtraFilesError("User cancelled file input") from err except EOFError as err: self.pretty_logger.info("\n āŒ End of input reached") raise ExtraFilesError("End of input reached during file prompt") from err def _prompt_for_directory_search(self, discovered_path: DiscoveredPath) -> None: """ Prompt user for a directory to search for missing files. """ self.pretty_logger.info("\nšŸ“ Please provide a directory to search for missing files:") while True: try: search_dir = input(" Enter directory path: ").strip() if not search_dir: self.pretty_logger.info(" āŒ Directory path cannot be empty. Please try again.") continue # Validate the directory try: dir_path = Path(search_dir) if not dir_path.is_dir(): raise ExtraFilesError(f"Path is not a directory: {search_dir}") # Search for missing files in the directory found_files = self._handle_directory(discovered_path, dir_path) if found_files: self.pretty_logger.info(f" āœ… Found {len(found_files)} matching files in {search_dir}") for file_path in found_files: self.pretty_logger.info(f" šŸ“„ {Path(file_path).name}") break else: self.pretty_logger.info(f" āš ļø No matching files found in {search_dir}") self.pretty_logger.info(" Would you like to try another directory?") retry = input(" Enter 'yes' to try again or any other key to skip: ").strip().lower() if retry != "yes": self.pretty_logger.info(" āš ļø Skipping directory search...") break except Exception as e: self.pretty_logger.info(f" āŒ Error searching directory: {e}") self.pretty_logger.info(" Please try again.") except KeyboardInterrupt as err: self.pretty_logger.info("\n āŒ Operation cancelled by user") raise ExtraFilesError("User cancelled file input") from err except EOFError as err: self.pretty_logger.info("\n āŒ End of input reached") raise ExtraFilesError("End of input reached during file prompt") from err def validate_extra_files(self) -> None: """ Revalidate files after user prompting. """ # If we still have missing files after prompting, raise an error if self.missing_file_paths: missing_list = ", ".join([str(missing.source_path) for missing in self.missing_file_paths]) raise ExtraFilesError(f"Missing or invalid extra files: {missing_list}") self.pretty_logger.info("\nāœ… Extra files validation completed") self.pretty_logger.info("=" * 60) def _is_binary_file(self, path: Path) -> bool: """ Detect if a file is binary by checking for null bytes. Args: path: Path to the file to check Returns: True if the file appears to be binary, False otherwise """ try: with open(path, "rb") as f: # Read a sample of the file (first 1024 bytes) sample = f.read(1024) # Check for null bytes which are common in binary files return b"\x00" in sample except Exception as e: logger.warning(f"Could not determine if file {path} is binary: {e}") return False def _discover_extra_file( self, discovered_path: DiscoveredPath, override_path: Path | None = None ) -> DiscoveredExtraFile: """ Validate that a file path exists and is readable. Args: path: Path to validate Returns: True if file is valid, False if it's a binary file Raises: ExtraFilesError: If file doesn't exist or isn't readable """ file_path = override_path or Path(discovered_path.source_path) extra_file = DiscoveredExtraFile(filename=file_path.name, discovered_source_paths=[discovered_path]) if not file_path.exists(): raise ExtraFilesError(f"Extra file does not exist: {file_path}") if not file_path.is_file(): raise ExtraFilesError(f"Extra file path is not a file: {file_path}") # Check if file is binary if self._is_binary_file(file_path): extra_file.cleartext = False with open(file_path): extra_file.content = file_path.read_bytes() return extra_file