mirror of
https://github.com/element-hq/ess-helm.git
synced 2026-11-20 00:51:27 -08:00
Compile values schema using ref & def instead of inlining
This commit is contained in:
9 files changed
+2201
-29604
No files matched your search
+2
-2
@@ -21,8 +21,8 @@ SPDX-FileCopyrightText = ["Copyright 2026 Element Creations Ltd"]
|
||||
SPDX-License-Identifier = "AGPL-3.0-only"
|
||||
|
||||
[[annotations]]
|
||||
path = "scripts/testdata/**/*.json"
|
||||
SPDX-FileCopyrightText = ["Copyright 2024-2025 New Vector Ltd", "Copyright 2025 Element Creations Ltd"]
|
||||
path = "scripts/testdata/**/**/*.json"
|
||||
SPDX-FileCopyrightText = ["Copyright 2024-2025 New Vector Ltd", "Copyright 2025-2026 Element Creations Ltd"]
|
||||
SPDX-License-Identifier = "AGPL-3.0-only"
|
||||
|
||||
[[annotations]]
|
||||
|
||||
+1987
-29570
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
The `values.schema.json` compilation now uses `$defs` and `$ref` instead of inlining every referenced sub-schema.
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright 2024-2025 New Vector Ltd
|
||||
# Copyright 2025 Element Creations Ltd
|
||||
# Copyright 2025-2026 Element Creations Ltd
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
@@ -42,35 +42,72 @@ def schema_walker(schema_part: dict[Any], callable: Callable[[dict[any]], dict[a
|
||||
return result
|
||||
|
||||
|
||||
# Inline any sub-schemas referenced in other files
|
||||
# Move any sub-schemas referenced in other files into the $defs of the compiled schema
|
||||
#
|
||||
# When running `helm lint` (or other sub-commands that use the schema to validate the values)
|
||||
# The relative path is relative to the current working directory rather than the schema file.
|
||||
# This means that you can't use file relative refs and have `helm lint` work both for the
|
||||
# parent chart and the sub-chart. We can't use HTTP as that won't work in airgapped envs, so
|
||||
# inline the sub-schemas. Symlinks were tried but they were obnoxiously noisy due to
|
||||
# the sub-schemas are collected in the $defs of the compiled schema and referenced with
|
||||
# internal $refs. Symlinks were tried but they were obnoxiously noisy due to
|
||||
# https://helm.sh/blog/2019-10-30-helm-symlink-security-notice/. We don't want to rely on keeping
|
||||
# complex sub-schemas in-sync for multiple charts/containers/etc, so use this script to merge
|
||||
def inline_sub_schemas(source_schema: Path, schema_part: dict[Any]) -> dict[Any]:
|
||||
def move_sub_schemas_to_defs(
|
||||
source_schema: Path,
|
||||
schema_part: dict[Any],
|
||||
sub_schemas: dict[str, dict[Any]],
|
||||
sub_schema_refs: dict[str, str],
|
||||
) -> dict[Any]:
|
||||
# We're currently assuming that all $refs are file relative refs.
|
||||
# This won't always be true but lets keep this as simple as we need it for now
|
||||
if "$ref" in schema_part:
|
||||
# If we're inlining a sub-schema we're only returning things from the inlined sub-schema
|
||||
# Internal $refs point at $defs of the compiled schema, so there is nothing to move
|
||||
if not schema_part["$ref"].startswith("file://"):
|
||||
return schema_part
|
||||
|
||||
# If we're referencing a sub-schema we only return a reference to the matching $def
|
||||
assert len(schema_part.keys()) == 1
|
||||
|
||||
sub_schema_ref = schema_part["$ref"]
|
||||
sub_schema = source_schema.parent / sub_schema_ref.replace("file://", "")
|
||||
if not sub_schema.exists() or not sub_schema.is_file():
|
||||
raise Exception(f"{sub_schema} does not exist relative to {source_schema}. Please appropriately create it")
|
||||
sub_schema_path = sub_schema_ref.replace("file://", "")
|
||||
sub_schema_file = source_schema.parent / sub_schema_path
|
||||
sub_schema_name = sub_schema_path.removesuffix(".json").replace("/", "_")
|
||||
|
||||
inlined_sub_schema = json.loads(sub_schema.read_text(encoding="UTF-8"))
|
||||
# A sub-schema referenced multiple times only needs to be stored in $defs once
|
||||
if sub_schema_name in sub_schemas:
|
||||
if sub_schema_refs[sub_schema_name] != sub_schema_path:
|
||||
raise Exception(f"Multiple sub-schemas would be named {sub_schema_name}. Please rename one of them")
|
||||
|
||||
# It doesn't make sense for the root of the inlined sub-schema to immediately be referencing something else
|
||||
return {"$ref": f"#/$defs/{sub_schema_name}"}
|
||||
|
||||
if not sub_schema_file.exists() or not sub_schema_file.is_file():
|
||||
raise Exception(
|
||||
f"{sub_schema_file} does not exist relative to {source_schema}. Please appropriately create it"
|
||||
)
|
||||
|
||||
inlined_sub_schema = json.loads(sub_schema_file.read_text(encoding="UTF-8"))
|
||||
|
||||
# It doesn't make sense for the root of the sub-schema to immediately be referencing something else
|
||||
assert "$ref" not in inlined_sub_schema
|
||||
# We're only inlining objects or arrays for now to keep this easy to reason about
|
||||
|
||||
# A $def cannot have its own identity. An $id would change the base URI that
|
||||
# internal $refs inside the $def resolve against
|
||||
inlined_sub_schema.pop("$id", None)
|
||||
inlined_sub_schema.pop("$schema", None)
|
||||
|
||||
# We're only moving objects or arrays into $defs for now to keep this easy to reason about
|
||||
assert inlined_sub_schema["type"] in ["object", "array"]
|
||||
|
||||
return inlined_sub_schema
|
||||
# Reserve the name before walking so that a sub-schema can reference itself
|
||||
sub_schemas[sub_schema_name] = {}
|
||||
sub_schema_refs[sub_schema_name] = sub_schema_path
|
||||
# The sub-schema may itself reference further sub-schemas, these are moved into $defs too
|
||||
sub_schemas[sub_schema_name] = schema_walker(
|
||||
inlined_sub_schema,
|
||||
lambda part: move_sub_schemas_to_defs(source_schema, part, sub_schemas, sub_schema_refs),
|
||||
)
|
||||
|
||||
return {"$ref": f"#/$defs/{sub_schema_name}"}
|
||||
|
||||
return schema_part
|
||||
|
||||
@@ -79,6 +116,10 @@ def inline_sub_schemas(source_schema: Path, schema_part: dict[Any]) -> dict[Any]
|
||||
# https://json-schema.org/understanding-json-schema/reference/object#additionalproperties:
|
||||
# "By default any additional properties are allowed."
|
||||
def default_additionalProperties_to_off(_: Path, schema_part: dict[Any]) -> dict[Any]:
|
||||
# A schema part that only references a $def cannot have additional properties
|
||||
if "$ref" in schema_part:
|
||||
return schema_part
|
||||
|
||||
if schema_part["type"] == "object" and "additionalProperties" not in schema_part:
|
||||
schema_part["additionalProperties"] = False
|
||||
|
||||
@@ -86,13 +127,20 @@ def default_additionalProperties_to_off(_: Path, schema_part: dict[Any]) -> dict
|
||||
|
||||
|
||||
def construct_helm_schema(source_schema: Path, destination_schema: Path):
|
||||
sub_schemas: dict[str, dict[Any]] = {}
|
||||
sub_schema_refs: dict[str, str] = {}
|
||||
schema_manipulators = [
|
||||
lambda schema_part: inline_sub_schemas(source_schema, schema_part),
|
||||
lambda schema_part: move_sub_schemas_to_defs(source_schema, schema_part, sub_schemas, sub_schema_refs),
|
||||
lambda schema_part: default_additionalProperties_to_off(source_schema, schema_part),
|
||||
]
|
||||
schema_contents = json.loads(source_schema.read_text(encoding="UTF-8"))
|
||||
for schema_manipulator in schema_manipulators:
|
||||
schema_contents = schema_walker(schema_contents, schema_manipulator)
|
||||
# The collected sub-schemas are not part of the walked schema, so apply the manipulators to them too
|
||||
for sub_schema_name in sub_schemas:
|
||||
sub_schemas[sub_schema_name] = schema_walker(sub_schemas[sub_schema_name], schema_manipulator)
|
||||
if sub_schemas:
|
||||
schema_contents["$defs"] = sub_schemas
|
||||
|
||||
destination_schema.write_text(json.dumps(schema_contents, indent=2) + "\n")
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Copyright 2024 New Vector Ltd
|
||||
# Copyright 2025 Element Creations Ltd
|
||||
# Copyright 2025-2026 Element Creations Ltd
|
||||
#
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
@@ -12,7 +12,7 @@ import pytest
|
||||
from .construct_helm_schema import (
|
||||
construct_helm_schema,
|
||||
default_additionalProperties_to_off,
|
||||
inline_sub_schemas,
|
||||
move_sub_schemas_to_defs,
|
||||
schema_walker,
|
||||
)
|
||||
|
||||
@@ -109,32 +109,114 @@ def test_leaves_schema_part_for_non_object_alone():
|
||||
assert "additionalProperties" not in default_additionalProperties_to_off(None, {"type": "array"})
|
||||
|
||||
|
||||
def test_inline_sub_schema_returns_referenced_sub_schema():
|
||||
def test_leaves_schema_part_for_a_ref_alone():
|
||||
schema_part = {"$ref": "#/$defs/something"}
|
||||
assert default_additionalProperties_to_off(None, schema_part) == schema_part
|
||||
|
||||
|
||||
def test_move_sub_schemas_to_defs_returns_reference_to_the_sub_schema():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
sub_schema = inline_sub_schemas(source_root / "schema.json", {"$ref": "file://sub_schema1.json"})
|
||||
assert sub_schema["type"] == "object"
|
||||
assert sub_schema["properties"].keys() == set(["first", "second"])
|
||||
sub_schemas = {}
|
||||
sub_schema_refs = {}
|
||||
|
||||
schema_part = move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://sub_schema1.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
|
||||
assert schema_part == {"$ref": "#/$defs/sub_schema1"}
|
||||
assert sub_schemas["sub_schema1"]["type"] == "object"
|
||||
assert sub_schemas["sub_schema1"]["properties"].keys() == set(["first", "second"])
|
||||
|
||||
|
||||
def test_inline_sub_schemas_disallows_other_properties_on_attachment_point():
|
||||
def test_move_sub_schemas_to_defs_stores_each_sub_schema_once():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
sub_schemas = {}
|
||||
sub_schema_refs = {}
|
||||
|
||||
first_reference = move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://sub_schema1.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
second_reference = move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://sub_schema1.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
|
||||
assert first_reference == second_reference == {"$ref": "#/$defs/sub_schema1"}
|
||||
assert sub_schemas.keys() == {"sub_schema1"}
|
||||
|
||||
|
||||
def test_move_sub_schemas_to_defs_moves_nested_sub_schemas_too():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
sub_schemas = {}
|
||||
sub_schema_refs = {}
|
||||
|
||||
schema_part = move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://sub_schema2.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
|
||||
assert schema_part == {"$ref": "#/$defs/sub_schema2"}
|
||||
assert sub_schemas["sub_schema2"]["properties"]["nested"] == {"$ref": "#/$defs/sub_schema1"}
|
||||
assert sub_schemas["sub_schema1"]["properties"].keys() == set(["first", "second"])
|
||||
|
||||
|
||||
def test_move_sub_schemas_to_defs_strips_the_sub_schema_identity():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
sub_schemas = {}
|
||||
sub_schema_refs = {}
|
||||
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://sub_schema2.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
|
||||
# The compiled schema is a single document so the $def cannot have its own identity
|
||||
assert "$id" not in sub_schemas["sub_schema2"]
|
||||
assert "$schema" not in sub_schemas["sub_schema2"]
|
||||
|
||||
|
||||
def test_move_sub_schemas_to_defs_disallows_other_properties_on_reference_point():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
with pytest.raises(AssertionError):
|
||||
inline_sub_schemas(
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json",
|
||||
{"$ref": "file://sub_schema1.json", "other": "disallowed"},
|
||||
{},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def test_inline_sub_schemas_disallows_non_object_sub_schemas():
|
||||
def test_move_sub_schemas_to_defs_disallows_non_object_sub_schemas():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
with pytest.raises(AssertionError):
|
||||
inline_sub_schemas(source_root / "schema.json", {"$ref": "file://invalid_sub_schema1.json"})
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json",
|
||||
{"$ref": "file://invalid_sub_schema1.json"},
|
||||
{},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def test_inline_sub_schemas_disallows_immediately_referencing_another_sub_schema():
|
||||
def test_move_sub_schemas_to_defs_disallows_immediately_referencing_another_sub_schema():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
with pytest.raises(AssertionError):
|
||||
inline_sub_schemas(source_root / "schema.json", {"$ref": "file://invalid_sub_schema2.json"})
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json",
|
||||
{"$ref": "file://invalid_sub_schema2.json"},
|
||||
{},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def test_move_sub_schemas_to_defs_rejects_sub_schemas_with_the_same_name():
|
||||
source_root = Path(__file__).parent / "testdata" / "schema_construction"
|
||||
sub_schemas = {}
|
||||
sub_schema_refs = {}
|
||||
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://collision/sub.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
with pytest.raises(Exception, match="Multiple sub-schemas would be named"):
|
||||
move_sub_schemas_to_defs(
|
||||
source_root / "schema.json", {"$ref": "file://collision_sub.json"}, sub_schemas, sub_schema_refs
|
||||
)
|
||||
|
||||
|
||||
def test_sets_additionalProperties_false_for_objects_where_unspecified():
|
||||
@@ -158,11 +240,31 @@ def test_constructs_merged_schema():
|
||||
|
||||
destination_schema_contents = json.loads(destination_schema.read_text(encoding="UTF-8"))
|
||||
|
||||
# Property directly in the referenced
|
||||
assert "presentDirectly" in destination_schema_contents["properties"]
|
||||
assert destination_schema_contents["properties"]["presentDirectly"]["type"] == "object"
|
||||
# Property directly in the source schema
|
||||
assert destination_schema_contents["properties"]["presentDirectly"] == {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
assert "merged" in destination_schema_contents["properties"]
|
||||
assert "type" in destination_schema_contents["properties"]["merged"]
|
||||
assert destination_schema_contents["properties"]["merged"]["type"] == "object"
|
||||
assert "properties" in destination_schema_contents["properties"]["merged"]
|
||||
# The referenced sub-schema is referenced through an internal $ref
|
||||
assert destination_schema_contents["properties"]["merged"] == {"$ref": "#/$defs/sub_schema1"}
|
||||
assert destination_schema_contents["$defs"]["sub_schema1"]["properties"].keys() == set(["first", "second"])
|
||||
# Objects in $defs also have additionalProperties defaulted to false
|
||||
assert destination_schema_contents["$defs"]["sub_schema1"]["additionalProperties"] is False
|
||||
|
||||
|
||||
def test_constructs_schema_storing_repeated_sub_schemas_once():
|
||||
with tempfile.TemporaryDirectory() as destination_folder:
|
||||
source_schema = Path(__file__).parent / "testdata" / "schema_construction" / "schema_with_repeated_refs.json"
|
||||
destination_schema = Path(destination_folder) / "values.schema.json"
|
||||
|
||||
construct_helm_schema(source_schema, destination_schema)
|
||||
|
||||
destination_schema_contents = json.loads(destination_schema.read_text(encoding="UTF-8"))
|
||||
|
||||
assert destination_schema_contents["properties"]["first"] == {"$ref": "#/$defs/sub_schema2"}
|
||||
assert destination_schema_contents["properties"]["second"] == {"$ref": "#/$defs/sub_schema2"}
|
||||
assert destination_schema_contents["$defs"]["sub_schema2"]["properties"]["nested"] == {
|
||||
"$ref": "#/$defs/sub_schema1"
|
||||
}
|
||||
assert list(destination_schema_contents["$defs"].keys()) == ["sub_schema2", "sub_schema1"]
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"$id": "file://matrix-stack/charts/schema_with_repeated_refs",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"first": {
|
||||
"$ref": "file://sub_schema2.json"
|
||||
},
|
||||
"second": {
|
||||
"$ref": "file://sub_schema2.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"$id": "file://sub_schema2",
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"nested": {
|
||||
"$ref": "file://sub_schema1.json"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user