1#@!#!123s

: / lib / python3 / dist-packages / cloudinit / config /
Filename : schema.py
back
# This file is part of cloud-init. See LICENSE file for license information.
"""schema.py: Set of module functions for processing cloud-config schema."""
import argparse
import json
import logging
import os
import re
import shutil
import sys
from collections import defaultdict
from contextlib import suppress
from copy import deepcopy
from enum import Enum
from errno import EACCES
from functools import partial
from typing import (
    TYPE_CHECKING,
    DefaultDict,
    List,
    NamedTuple,
    Optional,
    Tuple,
    Type,
    Union,
)

import yaml

from cloudinit import features, lifecycle, performance, safeyaml
from cloudinit.cmd.devel import read_cfg_paths
from cloudinit.handlers import INCLUSION_TYPES_MAP, type_from_starts_with
from cloudinit.helpers import Paths
from cloudinit.log.log_util import error
from cloudinit.sources import DataSourceNotFoundException
from cloudinit.temp_utils import mkdtemp
from cloudinit.util import load_text_file, write_file

try:
    from jsonschema import ValidationError
except ImportError:
    ValidationError = Exception  # type: ignore

try:
    from netplan import NetplanParserException, Parser  # type: ignore

    LIBNETPLAN_AVAILABLE = True
except ImportError:
    LIBNETPLAN_AVAILABLE = False


LOG = logging.getLogger(__name__)


# Note versions.schema.json is publicly consumed by schemastore.org.
# If we change the location of versions.schema.json in github, we need
# to provide an updated PR to
# https://github.com/SchemaStore/schemastore.
# When bumping schema version due to incompatible changes:
# 1. Add a new schema-cloud-config-v#.json
# 2. change the USERDATA_SCHEMA_FILE to cloud-init-schema-v#.json
# 3. Add the new version definition to versions.schema.cloud-config.json
USERDATA_SCHEMA_FILE = "schema-cloud-config-v1.json"
NETWORK_CONFIG_V1_SCHEMA_FILE = "schema-network-config-v1.json"
NETWORK_CONFIG_V2_SCHEMA_FILE = "schema-network-config-v2.json"


DEPRECATED_KEY = "deprecated"

# user-data files typically must begin with a leading '#'
USERDATA_VALID_HEADERS = sorted(
    [t for t in INCLUSION_TYPES_MAP.keys() if t[0] == "#"]
)

# type-annotate only if type-checking.
# Consider to add `type_extensions` as a dependency when Bionic is EOL.
if TYPE_CHECKING:
    import typing

    from typing_extensions import NotRequired, TypedDict

    class MetaSchema(TypedDict):
        id: str
        distros: typing.List[str]
        frequency: str
        activate_by_schema_keys: NotRequired[List[str]]

else:
    MetaSchema = dict


class SchemaDeprecationError(ValidationError):
    def __init__(
        self,
        message: str,
        version: str,
        **kwargs,
    ):
        super().__init__(message, **kwargs)
        self.version: str = version


class SchemaProblem(NamedTuple):
    path: str
    message: str

    def format(self) -> str:
        return f"{self.path}: {self.message}"


SchemaProblems = List[SchemaProblem]


class SchemaType(Enum):
    """Supported schema types are either cloud-config or network-config.

    Vendordata and Vendordata2 format adheres to cloud-config schema type.
    Cloud Metadata is unique schema to each cloud platform and likely will not
    be represented in this enum.
    """

    CLOUD_CONFIG = "cloud-config"
    NETWORK_CONFIG = "network-config"
    NETWORK_CONFIG_V1 = "network-config-v1"
    NETWORK_CONFIG_V2 = "network-config-v2"


# Placeholders for versioned schema and schema file locations.
# The "latest" key is used in absence of a requested specific version.
SCHEMA_FILES_BY_TYPE = {
    SchemaType.CLOUD_CONFIG: {
        "latest": USERDATA_SCHEMA_FILE,
    },
    SchemaType.NETWORK_CONFIG: {
        "latest": NETWORK_CONFIG_V2_SCHEMA_FILE,
    },
    SchemaType.NETWORK_CONFIG_V1: {
        "latest": NETWORK_CONFIG_V1_SCHEMA_FILE,
    },
    SchemaType.NETWORK_CONFIG_V2: {
        "latest": NETWORK_CONFIG_V2_SCHEMA_FILE,
    },
}


class InstanceDataType(Enum):
    """Types of instance data provided to cloud-init"""

    USERDATA = "user-data"
    NETWORK_CONFIG = "network-config"
    VENDORDATA = "vendor-data"
    VENDOR2DATA = "vendor2-data"
    # METADATA = "metadata"

    def __str__(self):  # pylint: disable=invalid-str-returned
        return self.value


class InstanceDataPart(NamedTuple):
    config_type: InstanceDataType
    schema_type: SchemaType
    config_path: str


class UserDataTypeAndDecodedContent(NamedTuple):
    userdata_type: str
    content: str


def _format_schema_problems(
    schema_problems: SchemaProblems,
    *,
    prefix: Optional[str] = None,
    separator: str = ", ",
) -> str:
    formatted = separator.join(map(lambda p: p.format(), schema_problems))
    if prefix:
        formatted = f"{prefix}{formatted}"
    return formatted


class SchemaValidationError(ValueError):
    """Raised when validating a cloud-config file against a schema."""

    def __init__(
        self,
        schema_errors: Optional[SchemaProblems] = None,
        schema_deprecations: Optional[SchemaProblems] = None,
    ):
        """Init the exception an n-tuple of schema errors.

        @param schema_errors: An n-tuple of the format:
            ((flat.config.key, msg),)
        @param schema_deprecations: An n-tuple of the format:
            ((flat.config.key, msg),)
        """
        message = ""

        def handle_problems(problems, prefix):
            if not problems:
                return problems
            nonlocal message
            if message:
                message += "\n\n"
            problems = sorted(list(set(problems)))
            message += _format_schema_problems(problems, prefix=prefix)
            return problems

        self.schema_errors = handle_problems(
            schema_errors,
            prefix="Cloud config schema errors: ",
        )
        self.schema_deprecations = handle_problems(
            schema_deprecations,
            prefix="Cloud config schema deprecations: ",
        )
        super().__init__(message)

    def has_errors(self) -> bool:
        return bool(self.schema_errors)


class SchemaValidationInvalidHeaderError(SchemaValidationError):
    """Raised when no valid header is declared in the user-data file."""


def is_schema_byte_string(checker, instance):
    """TYPE_CHECKER override allowing bytes for string type

    For jsonschema v. 3.0.0+
    """
    try:
        from jsonschema import Draft4Validator
    except ImportError:
        return False
    return Draft4Validator.TYPE_CHECKER.is_type(
        instance, "string"
    ) or isinstance(instance, (bytes,))


def _add_deprecated_changed_or_new_msg(
    config: dict, annotate=False, filter_key=None
) -> str:
    """combine description with new/changed/deprecated message

    deprecated/changed/new keys require a _version key (this is verified
    in a unittest), a _description key is optional
    """

    def format_message(key: str):
        if not config.get(f"{key}"):
            return ""
        key_description = config.get(f"{key}_description", "")
        v = config.get(
            f"{key}_version",
            f"<missing {key}_version key, please file a bug report>",
        )
        msg = f"{key.capitalize()} in version {v}. {key_description}"
        if annotate:
            return f" {msg}"

        # italicised RST - no whitespace between astrisk and text
        return f"\n\n*{msg.strip()}*"

    # define print order
    filter_keys = (
        filter_key if filter_key else ["deprecated", "changed", "new"]
    )

    # build a deprecation/new/changed string
    changed_new_deprecated = "".join(map(format_message, filter_keys))
    description = config.get("description", "")
    return f"{description}{changed_new_deprecated}".rstrip()


def cloud_init_deepest_matches(errors, instance) -> List[ValidationError]:
    """Return the best_match errors based on the deepest match in the json_path

    This is useful for anyOf and oneOf subschemas where the most-specific error
    tends to be the most appropriate.
    """
    best_matches = []
    path_depth = 0
    is_type = isinstance(instance, dict) and "type" in instance
    for err in errors:
        if is_type:
            # Most appropriate subschema matches given type
            if instance["type"] in err.schema.get("properties", {}).get(
                "type", {}
            ).get("enum", []):
                return [err]

            if hasattr(err, "json_path"):
                if err.json_path[-4:] == "type":
                    # Prioritize cloud-init 'type'-related errors exclusively
                    best_matches.append(err)
            elif err.path and err.path[0] == "type":
                # Use err.paths instead of json_path on jsonschema <= 3.2
                # Prioritize cloud-init 'type'-related errors exclusively
                best_matches.append(err)
        elif len(err.path) == path_depth:
            best_matches.append(err)
        elif len(err.path) > path_depth:
            path_depth = len(err.path)
            best_matches = [err]
    return best_matches


def _validator(
    _validator,
    deprecated: bool,
    _instance,
    schema: dict,
    filter_key: str,
    error_type: Type[Exception] = SchemaDeprecationError,
):
    """Jsonschema validator for `deprecated` items.

    It yields an instance of `error_type` if deprecated that must be handled,
    otherwise the instance is consider faulty.
    """
    if deprecated:
        msg = _add_deprecated_changed_or_new_msg(
            schema, annotate=True, filter_key=[filter_key]
        )
        yield error_type(msg, schema.get("deprecated_version", "devel"))


def _anyOf(
    validator,
    anyOf,
    instance,
    _schema,
    error_type: Type[Exception] = SchemaDeprecationError,
):
    """Jsonschema validator for `anyOf`.

    It treats occurrences of `error_type` as non-errors, but yield them for
    external processing. Useful to process schema annotations, as `deprecated`.

    Cloud-init's network schema under the `config` key has a complexity of
    allowing each list dict item to declare it's type with a `type` key which
    can contain the values: bond, bridge, nameserver, physical, route, vlan.

    This schema 'flexibility' makes it hard for the default
    jsonschema.exceptions.best_match function to find the correct schema
    failure because it typically returns the failing schema error based on
    the schema of greatest match depth. Since each anyOf dict matches the
    same depth into the network schema path, `best_match` just returns the
    first set of schema errors, which is almost always incorrect.

    To find a better schema match when encountering schema validation errors,
    cloud-init network schema introduced schema $defs with the prefix
    `anyOf_type_`. If the object we are validating contains a 'type' key, and
    one of the failing schema objects in an anyOf clause has a name of the
    format anyOf_type_XXX, raise those schema errors instead of calling
    best_match.
    """
    from jsonschema.exceptions import best_match

    all_errors = []
    all_deprecations = []
    skip_best_match = False
    for index, subschema in enumerate(anyOf):
        all_errs = list(
            validator.descend(instance, subschema, schema_path=index)
        )
        errs = list(filter(lambda e: not isinstance(e, error_type), all_errs))
        deprecations = list(
            filter(lambda e: isinstance(e, error_type), all_errs)
        )
        if not errs:
            all_deprecations.extend(deprecations)
            break
        if (
            isinstance(instance, dict)
            and "type" in instance
            and "anyOf_type" in subschema.get("$ref", "")
        ):
            if f"anyOf_type_{instance['type']}" in subschema["$ref"]:
                # A matching anyOf_type_XXX $ref indicates this is likely the
                # best_match ValidationError. Skip best_match below.
                skip_best_match = True
                yield from errs
        all_errors.extend(errs)
    else:
        if not skip_best_match:
            yield best_match(all_errors)
        yield ValidationError(
            "%r is not valid under any of the given schemas" % (instance,),
            context=all_errors,
        )
    yield from all_deprecations


def _oneOf(
    validator,
    oneOf,
    instance,
    _schema,
    error_type: Type[Exception] = SchemaDeprecationError,
):
    """Jsonschema validator for `oneOf`.

    It treats occurrences of `error_type` as non-errors, but yield them for
    external processing. Useful to process schema annotations, as `deprecated`.
    """
    subschemas = enumerate(oneOf)
    all_errors = []
    all_deprecations = []
    for index, subschema in subschemas:
        all_errs = list(
            validator.descend(instance, subschema, schema_path=index)
        )
        errs = list(filter(lambda e: not isinstance(e, error_type), all_errs))
        deprecations = list(
            filter(lambda e: isinstance(e, error_type), all_errs)
        )
        if not errs:
            first_valid = subschema
            all_deprecations.extend(deprecations)
            break
        all_errors.extend(errs)
    else:
        yield from cloud_init_deepest_matches(all_errors, instance)

    more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)]
    if more_valid:
        more_valid.append(first_valid)
        reprs = ", ".join(repr(schema) for schema in more_valid)
        yield ValidationError(
            "%r is valid under each of %s" % (instance, reprs)
        )
    else:
        yield from all_deprecations


def get_jsonschema_validator():
    """Get metaschema validator and format checker

    Older versions of jsonschema require some compatibility changes.

    @returns: Tuple: (jsonschema.Validator, FormatChecker)
    @raises: ImportError when jsonschema is not present
    """
    from jsonschema import Draft4Validator, FormatChecker
    from jsonschema.validators import create

    # Allow for bytes to be presented as an acceptable valid value for string
    # type jsonschema attributes in cloud-init's schema.
    # This allows #cloud-config to provide valid yaml "content: !!binary | ..."

    meta_schema = deepcopy(Draft4Validator.META_SCHEMA)

    # This additional label allows us to specify a different name
    # than the property key when generating docs.
    # This is especially useful when using a "patternProperties" regex,
    # otherwise the property label in the generated docs will be a
    # regular expression.
    # http://json-schema.org/understanding-json-schema/reference/object.html#pattern-properties
    meta_schema["properties"]["label"] = {"type": "string"}

    validator_kwargs = {}
    meta_schema["additionalProperties"] = False
    type_checker = Draft4Validator.TYPE_CHECKER.redefine(
        "string", is_schema_byte_string
    )
    validator_kwargs = {
        "type_checker": type_checker,
    }

    # Add deprecation handling
    validators = dict(Draft4Validator.VALIDATORS)
    validators[DEPRECATED_KEY] = partial(_validator, filter_key="deprecated")
    validators["changed"] = partial(_validator, filter_key="changed")
    validators["oneOf"] = _oneOf
    validators["anyOf"] = _anyOf

    cloudinitValidator = create(
        meta_schema=meta_schema,
        validators=validators,
        version="draft4",
        **validator_kwargs,
    )

    def is_valid_pre_4_0_0(self, instance, _schema=None, **__):
        """Override version of `is_valid`.

        It does ignore instances of `SchemaDeprecationError`.
        """
        errors = filter(
            lambda e: not isinstance(  # pylint: disable=W1116
                e, SchemaDeprecationError
            ),
            self.iter_errors(instance, _schema),
        )
        return next(errors, None) is None

    def is_valid(self, instance, _schema=None, **__):
        """Override version of `is_valid`.

        It does ignore instances of `SchemaDeprecationError`.
        """
        errors = filter(
            lambda e: not isinstance(  # pylint: disable=W1116
                e, SchemaDeprecationError
            ),
            self.evolve(schema=_schema).iter_errors(instance),
        )
        return next(errors, None) is None

    # Add deprecation handling
    is_valid_fn = is_valid_pre_4_0_0
    if hasattr(cloudinitValidator, "evolve"):
        is_valid_fn = is_valid

    # this _could_ be an error, but it's not in this case
    # https://github.com/python/mypy/issues/2427#issuecomment-1419206807
    cloudinitValidator.is_valid = is_valid_fn  # type: ignore [method-assign]

    return (cloudinitValidator, FormatChecker)


def validate_cloudconfig_metaschema(validator, schema: dict, throw=True):
    """Validate provided schema meets the metaschema definition. Return strict
    Validator and FormatChecker for use in validation
    @param validator: Draft4Validator instance used to validate the schema
    @param schema: schema to validate
    @param throw: Sometimes the validator and checker are required, even if
        the schema is invalid. Toggle for whether to raise
        SchemaValidationError or log warnings.

    @raises: ImportError when jsonschema is not present
    @raises: SchemaValidationError when the schema is invalid
    """

    from jsonschema.exceptions import SchemaError

    try:
        validator.check_schema(schema)
    except SchemaError as err:
        # Raise SchemaValidationError to avoid jsonschema imports at call
        # sites
        if throw:
            raise SchemaValidationError(
                schema_errors=[
                    SchemaProblem(
                        ".".join([str(p) for p in err.path]), err.message
                    )
                ]
            ) from err
        LOG.warning(
            "Meta-schema validation failed, attempting to validate config "
            "anyway: %s",
            err,
        )


def network_schema_version(network_config: dict) -> Optional[int]:
    """Return the version of the network schema when present."""
    if "network" in network_config:
        return network_config["network"].get("version")
    return network_config.get("version")


def netplan_validate_network_schema(
    network_config: dict,
    strict: bool = False,
    annotate: bool = False,
    log_details: bool = True,
) -> bool:
    """On systems with netplan, validate network_config schema for file

    Leverage NetplanParser for error annotation line, column and detailed
    errors.

    @param network_config: Dict of network configuration settings validated
        against
    @param strict: Boolean, when True raise SchemaValidationErrors instead of
       logging warnings.
    @param annotate: Boolean, when True, print original network_config_file
        content with error annotations
    @param log_details: Boolean, when True logs details of validation errors.
       If there are concerns about logging sensitive userdata, this should
       be set to False.

    @return: True when schema validation was performed. False when not on a
        system with netplan and netplan python support.
    @raises: SchemaValidationError when netplan's parser raises
        NetplanParserExceptions.
    """
    if LIBNETPLAN_AVAILABLE:
        LOG.debug("Validating network-config with netplan API")
    else:
        LOG.debug(
            "Skipping netplan schema validation. No netplan API available"
        )
        return False
    # netplan Parser looks at all *.yaml files in the target directory underA
    # /etc/netplan. cloud-init should only validate schema of the
    # network-config it generates, so create a <tmp_dir>/etc/netplan
    # to validate only our network-config.
    parse_dir = mkdtemp()
    netplan_file = os.path.join(parse_dir, "etc/netplan/network-config.yaml")

    # Datasource network config can optionally exclude top-level network key
    net_cfg = deepcopy(network_config)
    if "network" not in net_cfg:
        net_cfg = {"network": net_cfg}

    src_content = safeyaml.dumps(net_cfg)
    write_file(netplan_file, src_content, mode=0o600)

    parser = Parser()
    errors = []
    try:
        # Parse all netplan *.yaml files.load_yaml_hierarchy looks for nested
        # etc/netplan subdir under "/".
        parser.load_yaml_hierarchy(parse_dir)
    except NetplanParserException as e:
        errors.append(
            SchemaProblem(
                "format-l{line}.c{col}".format(line=e.line, col=e.column),
                f"Invalid netplan schema. {e.message}",
            )
        )
    if os.path.exists(parse_dir):
        shutil.rmtree(parse_dir)
    if errors:
        if strict:
            if annotate:
                # Load YAML marks for annotation
                _, marks = safeyaml.load_with_marks(src_content)
                print(
                    annotated_cloudconfig_file(
                        src_content,
                        marks,
                        schema_errors=errors,
                    )
                )
            raise SchemaValidationError(errors)
        if log_details:
            message = _format_schema_problems(
                errors,
                prefix=(
                    f"{SchemaType.NETWORK_CONFIG.value} failed "
                    "schema validation!\n"
                ),
                separator="\n",
            )
        else:
            message = (
                f"{SchemaType.NETWORK_CONFIG.value} failed schema validation! "
                "You may run 'sudo cloud-init schema --system' to "
                "check the details."
            )
        LOG.warning(message)
    return True


@performance.timed("Validating schema")
def validate_cloudconfig_schema(
    config: dict,
    schema: Optional[dict] = None,
    schema_type: SchemaType = SchemaType.CLOUD_CONFIG,
    strict: bool = False,
    strict_metaschema: bool = False,
    log_details: bool = True,
    log_deprecations: bool = False,
) -> bool:
    """Validate provided config meets the schema definition.

    @param config: Dict of cloud configuration settings validated against
        schema. Ignored if strict_metaschema=True
    @param schema: jsonschema dict describing the supported schema definition
       for the cloud config module (config.cc_*). If None, validate against
       global schema.
    @param schema_type: Optional SchemaType.
       One of: SchemaType.CLOUD_CONFIG or SchemaType.NETWORK_CONFIG_V1 or
            SchemaType.NETWORK_CONFIG_V2
       Default: SchemaType.CLOUD_CONFIG
    @param strict: Boolean, when True raise SchemaValidationErrors instead of
       logging warnings.
    @param strict_metaschema: Boolean, when True validates schema using strict
       metaschema definition at runtime (currently unused)
    @param log_details: Boolean, when True logs details of validation errors.
       If there are concerns about logging sensitive userdata, this should
       be set to False.
    @param log_deprecations: Controls whether to log deprecations or not.

    @raises: SchemaValidationError when provided config does not validate
        against the provided schema.
    @raises: RuntimeError when provided config sourced from YAML is not a dict.
    @raises: ValueError on invalid schema_type not in CLOUD_CONFIG or
        NETWORK_CONFIG_V1 or NETWORK_CONFIG_V2
    """
    from cloudinit.net.netplan import available as netplan_available

    if schema_type == SchemaType.NETWORK_CONFIG:
        network_version = network_schema_version(config)
        if network_version == 2:
            schema_type = SchemaType.NETWORK_CONFIG_V2
        elif network_version == 1:
            schema_type = SchemaType.NETWORK_CONFIG_V1
        schema = get_schema(schema_type)

    if schema_type == SchemaType.NETWORK_CONFIG_V2:
        if netplan_validate_network_schema(
            network_config=config, strict=strict, log_details=log_details
        ):
            # Schema was validated by netplan
            return True
        elif netplan_available():
            # We found no netplan API on netplan system, do not perform schema
            # validation against cloud-init's network v2 schema because netplan
            # supports more config keys than in cloud-init's netv2 schema.
            # This may result in schema warnings for valid netplan config
            # which would be successfully rendered by netplan but doesn't
            # adhere to cloud-init's network v2.
            return False

    if schema is None:
        schema = get_schema(schema_type)
    try:
        (cloudinitValidator, FormatChecker) = get_jsonschema_validator()
        if strict_metaschema:
            validate_cloudconfig_metaschema(
                cloudinitValidator, schema, throw=False
            )
    except ImportError:
        LOG.debug("Ignoring schema validation. jsonschema is not present")
        return False

    validator = cloudinitValidator(schema, format_checker=FormatChecker())

    errors: SchemaProblems = []
    deprecations: SchemaProblems = []
    info_deprecations: SchemaProblems = []
    for schema_error in sorted(
        validator.iter_errors(config), key=lambda e: e.path
    ):
        path = ".".join([str(p) for p in schema_error.path])
        if (
            not path
            and schema_error.validator == "additionalProperties"
            and schema_error.schema == schema
        ):
            # an issue with invalid top-level property
            prop_match = re.match(
                r".*\('(?P<name>.*)' was unexpected\)", schema_error.message
            )
            if prop_match:
                path = prop_match["name"]
        if isinstance(
            schema_error, SchemaDeprecationError
        ):  # pylint: disable=W1116
            if (
                schema_error.version == "devel"
                or lifecycle.should_log_deprecation(
                    schema_error.version, features.DEPRECATION_INFO_BOUNDARY
                )
            ):
                deprecations.append(SchemaProblem(path, schema_error.message))
            else:
                info_deprecations.append(
                    SchemaProblem(path, schema_error.message)
                )
        else:
            errors.append(SchemaProblem(path, schema_error.message))

    if log_deprecations:
        if info_deprecations:
            message = _format_schema_problems(
                info_deprecations,
                prefix="Deprecated cloud-config provided: ",
            )
            LOG.info(message)
        if deprecations:
            message = _format_schema_problems(
                deprecations,
                prefix="Deprecated cloud-config provided: ",
            )
            # This warning doesn't fit the standardized lifecycle.deprecated()
            # utility format, but it is a deprecation log, so log it directly.
            LOG.deprecated(message)  # type: ignore
    if strict and (errors or deprecations or info_deprecations):
        raise SchemaValidationError(errors, deprecations + info_deprecations)
    if errors:
        if log_details:
            details = _format_schema_problems(
                errors,
                prefix=f"{schema_type.value} failed schema validation!\n",
                separator="\n",
            )
        else:
            details = (
                f"{schema_type.value} failed schema validation! "
                "You may run 'sudo cloud-init schema --system' to "
                "check the details."
            )
        LOG.warning(details)
    return True


class _Annotator:
    def __init__(
        self,
        original_content: str,
        schemamarks: dict,
    ):
        self._original_content = original_content
        self._schemamarks = schemamarks

    @staticmethod
    def _build_footer(title: str, content: List[str]) -> str:
        body = "\n".join(content)
        return f"# {title}: -------------\n{body}\n\n"

    def _build_errors_by_line(self, schema_problems: SchemaProblems):
        errors_by_line: DefaultDict[Union[str, int], List] = defaultdict(list)
        for path, msg in schema_problems:
            match = re.match(r"format-l(?P<line>\d+)\.c(?P<col>\d+).*", path)
            if match:
                line, col = match.groups()
                errors_by_line[int(line)].append(msg)
            else:
                col = None
                errors_by_line[self._schemamarks[path]].append(msg)
            if col is not None:
                msg = "Line {line} column {col}: {msg}".format(
                    line=line, col=col, msg=msg
                )
        return errors_by_line

    @staticmethod
    def _add_problems(
        problems: List[str],
        labels: List[str],
        footer: List[str],
        index: int,
        label_prefix: str = "",
    ) -> int:
        for problem in problems:
            label = f"{label_prefix}{index}"
            labels.append(label)
            footer.append(f"# {label}: {problem}")
            index += 1
        return index

    def _annotate_content(
        self,
        lines: List[str],
        errors_by_line: dict,
        deprecations_by_line: dict,
    ) -> List[str]:
        annotated_content = []
        error_footer: List[str] = []
        deprecation_footer: List[str] = []
        error_index = 1
        deprecation_index = 1
        for line_number, line in enumerate(lines, 1):
            errors = errors_by_line[line_number]
            deprecations = deprecations_by_line[line_number]
            if errors or deprecations:
                labels: List[str] = []
                error_index = self._add_problems(
                    errors, labels, error_footer, error_index, label_prefix="E"
                )
                deprecation_index = self._add_problems(
                    deprecations,
                    labels,
                    deprecation_footer,
                    deprecation_index,
                    label_prefix="D",
                )
                annotated_content.append(line + "\t\t# " + ",".join(labels))
            else:
                annotated_content.append(line)

        annotated_content.extend(
            map(
                lambda seq: self._build_footer(*seq),
                filter(
                    lambda seq: bool(seq[1]),
                    (
                        ("Errors", error_footer),
                        ("Deprecations", deprecation_footer),
                    ),
                ),
            )
        )
        return annotated_content

    def annotate(
        self,
        schema_errors: SchemaProblems,
        schema_deprecations: SchemaProblems,
    ) -> str:
        if not schema_errors and not schema_deprecations:
            return self._original_content
        lines = self._original_content.split("\n")
        errors_by_line = self._build_errors_by_line(schema_errors)
        deprecations_by_line = self._build_errors_by_line(schema_deprecations)
        annotated_content = self._annotate_content(
            lines, errors_by_line, deprecations_by_line
        )
        return "\n".join(annotated_content)


def annotated_cloudconfig_file(
    original_content: str,
    schemamarks: dict,
    *,
    schema_errors: Optional[SchemaProblems] = None,
    schema_deprecations: Optional[SchemaProblems] = None,
) -> Union[str, bytes]:
    """Return contents of the cloud-config file annotated with schema errors.

    @param cloudconfig: YAML-loaded dict from the original_content or empty
        dict if unparsable.
    @param original_content: The contents of a cloud-config file
    @param schemamarks: Dict with schema marks.
    @param schema_errors: Instance of `SchemaProblems`.
    @param schema_deprecations: Instance of `SchemaProblems`.

    @return Annotated schema
    """
    return _Annotator(original_content, schemamarks).annotate(
        schema_errors or [], schema_deprecations or []
    )


def process_merged_cloud_config_part_problems(
    content: str,
) -> List[SchemaProblem]:
    """Annotate and return schema validation errors in merged cloud-config.txt

    When merging multiple cloud-config parts cloud-init logs an error and
    ignores any user-data parts which are declared as #cloud-config but
    cannot be processed. the handler.cloud_config module also leaves comments
    in the final merged config for every invalid part file which begin with
    MERGED_CONFIG_SCHEMA_ERROR_PREFIX to aid in triage.
    """
    from cloudinit.handlers.cloud_config import MERGED_PART_SCHEMA_ERROR_PREFIX

    if MERGED_PART_SCHEMA_ERROR_PREFIX not in content:
        return []
    errors: List[SchemaProblem] = []
    for line_num, line in enumerate(content.splitlines(), 1):
        if line.startswith(MERGED_PART_SCHEMA_ERROR_PREFIX):
            errors.append(
                SchemaProblem(
                    f"format-l{line_num}.c1",
                    line.replace(
                        MERGED_PART_SCHEMA_ERROR_PREFIX,
                        "Ignored invalid user-data: ",
                    ),
                )
            )
    return errors


def _get_config_type_and_rendered_userdata(
    config_path: str,
    content: str,
    instance_data_path: Optional[str] = None,
) -> UserDataTypeAndDecodedContent:
    """
    Return tuple of user-data-type and rendered content.

    When encountering jinja user-data, render said content.

    :return: UserDataTypeAndDecodedContent
    :raises: SchemaValidationError when non-jinja content found but
        header declared ## template: jinja.
    :raises JinjaSyntaxParsingException when jinja syntax error found.
    :raises JinjaLoadError when jinja template fails to load.
    """
    from cloudinit.handlers.jinja_template import (
        JinjaLoadError,
        JinjaSyntaxParsingException,
        NotJinjaError,
        render_jinja_payload_from_file,
    )

    user_data_type = type_from_starts_with(content)
    schema_position = "format-l1.c1"
    if user_data_type == "text/jinja2":
        try:
            content = render_jinja_payload_from_file(
                content, config_path, instance_data_path
            )
        except NotJinjaError as e:
            raise SchemaValidationError(
                [
                    SchemaProblem(
                        schema_position,
                        "Detected type '{user_data_type}' from header. "
                        "But, content is not a jinja template",
                    )
                ]
            ) from e
        except JinjaSyntaxParsingException as e:
            error(
                "Failed to render templated user-data. " + str(e),
                sys_exit=True,
            )
        except JinjaLoadError as e:
            error(str(e), sys_exit=True)
        schema_position = "format-l2.c1"
        user_data_type = type_from_starts_with(content)
    if not user_data_type:  # Neither jinja2 nor #cloud-config
        header_line, _, _ = content.partition("\n")
        raise SchemaValidationInvalidHeaderError(
            [
                SchemaProblem(
                    schema_position,
                    f"Unrecognized user-data header in {config_path}:"
                    f' "{header_line}".\nExpected first line'
                    f" to be one of: {', '.join(USERDATA_VALID_HEADERS)}",
                )
            ]
        )
    elif user_data_type != "text/cloud-config":
        print(
            f"User-data type '{user_data_type}' not currently evaluated"
            " by cloud-init schema"
        )
    return UserDataTypeAndDecodedContent(user_data_type, content)


def validate_cloudconfig_file(
    config_path: str,
    schema: dict,
    schema_type: SchemaType = SchemaType.CLOUD_CONFIG,
    annotate: bool = False,
    instance_data_path: Optional[str] = None,
) -> bool:
    """Validate cloudconfig file adheres to a specific jsonschema.

    @param config_path: Path to the yaml cloud-config file to parse, or None
        to default to system userdata from Paths object.
    @param schema: Dict describing a valid jsonschema to validate against.
    @param schema_type: One of SchemaType.NETWORK_CONFIG or CLOUD_CONFIG
    @param annotate: Boolean set True to print original config file with error
        annotations on the offending lines.
    @param instance_data_path: Path to instance_data JSON, used for text/jinja
        rendering.

    :return: True when validation was performed successfully
    :raises SchemaValidationError containing any of schema_errors encountered.
    :raises RuntimeError when config_path does not exist.
    """
    from cloudinit.net.netplan import available as netplan_available

    decoded_content = load_text_file(config_path)
    if not decoded_content:
        print(
            "Empty '%s' found at %s. Nothing to validate."
            % (schema_type.value, config_path)
        )
        return False

    if schema_type in (SchemaType.NETWORK_CONFIG,):
        decoded_config = UserDataTypeAndDecodedContent(
            schema_type.value, decoded_content
        )
    else:
        decoded_config = _get_config_type_and_rendered_userdata(
            config_path, decoded_content, instance_data_path
        )
    if decoded_config.userdata_type not in (
        "network-config",
        "text/cloud-config",
    ):
        return False
    content = decoded_config.content
    errors = process_merged_cloud_config_part_problems(content)
    try:
        if annotate:
            cloudconfig, marks = safeyaml.load_with_marks(content)
        else:
            cloudconfig = yaml.safe_load(content)
            marks = {}
    except yaml.YAMLError as e:
        line = column = 1
        mark = None
        if hasattr(e, "context_mark") and getattr(e, "context_mark"):
            mark = getattr(e, "context_mark")
        elif hasattr(e, "problem_mark") and getattr(e, "problem_mark"):
            mark = getattr(e, "problem_mark")
        if mark:
            line = mark.line + 1
            column = mark.column + 1
        errors.append(
            SchemaProblem(
                "format-l{line}.c{col}".format(line=line, col=column),
                "File {0} is not valid YAML. {1}".format(config_path, str(e)),
            ),
        )
        schema_error = SchemaValidationError(errors)
        if annotate:
            print(
                annotated_cloudconfig_file(
                    content, {}, schema_errors=schema_error.schema_errors
                )
            )
        raise schema_error from e
    if not isinstance(cloudconfig, dict):
        # Return a meaningful message on empty cloud-config
        if not annotate:
            raise RuntimeError(
                f"{schema_type.value} {config_path} is not a YAML dict."
            )
    if schema_type == SchemaType.NETWORK_CONFIG:
        if not cloudconfig.get("network", cloudconfig):
            print("Skipping network-config schema validation on empty config.")
            return False
        network_version = network_schema_version(cloudconfig)
        if network_version == 2:
            schema_type = SchemaType.NETWORK_CONFIG_V2
            if netplan_validate_network_schema(
                network_config=cloudconfig, strict=True, annotate=annotate
            ):
                return True  # schema validation performed by netplan
            elif netplan_available():
                print(
                    "Skipping network-config schema validation for version: 2."
                    " No netplan API available."
                )
                return False
        elif network_version == 1:
            schema_type = SchemaType.NETWORK_CONFIG_V1
            # refresh schema since NETWORK_CONFIG defaults to V2
            schema = get_schema(schema_type)
    try:
        if not validate_cloudconfig_schema(
            cloudconfig,
            schema=schema,
            schema_type=schema_type,
            strict=True,
            log_deprecations=False,
        ):
            print(
                f"Skipping {schema_type.value} schema validation."
                " Jsonschema dependency missing."
            )
            return False
    except SchemaValidationError as e:
        if e.has_errors():
            errors += e.schema_errors
        if annotate:
            print(
                annotated_cloudconfig_file(
                    content,
                    marks,
                    schema_errors=errors,
                    schema_deprecations=e.schema_deprecations,
                )
            )
        elif e.schema_deprecations:
            message = _format_schema_problems(
                e.schema_deprecations,
                prefix="Cloud config schema deprecations: ",
                separator=", ",
            )
            print(message)
        if errors:
            raise SchemaValidationError(schema_errors=errors) from e
    return True


def get_schema_dir() -> str:
    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "schemas")


def get_schema(schema_type: SchemaType = SchemaType.CLOUD_CONFIG) -> dict:
    """Return jsonschema for a specific type.

    Return empty schema when no specific schema file exists.
    """
    schema_file = os.path.join(
        get_schema_dir(), SCHEMA_FILES_BY_TYPE[schema_type]["latest"]
    )
    full_schema = None
    try:
        full_schema = json.loads(load_text_file(schema_file))
    except (IOError, OSError):
        LOG.warning(
            "Skipping %s schema validation. No JSON schema file found %s.",
            schema_type.value,
            schema_file,
        )
        return {}
    return full_schema


def get_parser(parser=None):
    """Return a parser for supported cmdline arguments."""
    if not parser:
        parser = argparse.ArgumentParser(
            prog="cloudconfig-schema",
            description=(
                "Schema validation and documentation of instance-data"
                " configuration provided to cloud-init. This includes:"
                " user-data, vendor-data and network-config"
            ),
        )
    parser.add_argument(
        "-c",
        "--config-file",
        help=(
            "Path of the cloud-config or network-config YAML file to validate"
        ),
    )
    parser.add_argument(
        "-t",
        "--schema-type",
        type=str,
        choices=[
            SchemaType.CLOUD_CONFIG.value,
            SchemaType.NETWORK_CONFIG.value,
        ],
        help=(
            "When providing --config-file, the schema type to validate config"
            f" against. Default: {SchemaType.CLOUD_CONFIG}"
        ),
    )
    parser.add_argument(
        "-i",
        "--instance-data",
        type=str,
        help=(
            "Path to instance-data.json file for variable expansion "
            "of '##template: jinja' user-data. Default: "
            f"{read_cfg_paths().get_runpath('instance_data')}"
        ),
    )
    parser.add_argument(
        "--system",
        action="store_true",
        default=False,
        help=(
            "Validate the system instance-data provided as vendor-data"
            " user-data and network-config"
        ),
    )
    parser.add_argument(
        "--annotate",
        action="store_true",
        default=False,
        help="Annotate existing instance-data files any discovered errors",
    )
    return parser


def _assert_exclusive_args(args):
    """Error or warn on invalid exclusive parameter combinations."""
    exclusive_args = [args.config_file, args.system]
    if len([arg for arg in exclusive_args if arg]) != 1:
        error(
            "Expected one of --config-file or --system arguments",
            sys_exit=True,
        )
    if args.system and args.schema_type:
        print(
            "WARNING: The --schema-type parameter is inapplicable when "
            "--system is present"
        )


def get_config_paths_from_args(
    args,
) -> Tuple[str, List[InstanceDataPart]]:
    """Return appropriate instance-data.json and instance data parts

    Based on command line args, and user permissions, determine the
    appropriate instance-data.json to source for jinja templates and
    a list of applicable InstanceDataParts such as user-data, vendor-data
    and network-config for which to validate schema. Avoid returning any
    InstanceDataParts when the expected config_path does not exist.

    :return: A tuple of the instance-data.json path and a list of
        viable InstanceDataParts present on the system.
    """

    def get_processed_or_fallback_path(
        paths: Paths,
        primary_path_key: str,
        raw_fallback_path_key: str,
    ) -> str:
        """Get processed data path when non-empty of fallback to raw data path.

        - When primary path and raw path exist and are empty, prefer primary
          path.
        - When primary path is empty but the raw fallback path is non-empty,
          this indicates an invalid and ignored raw user-data was provided and
          cloud-init emitted a warning and did not process unknown raw
          user-data.
          In the case of invalid raw user-data header, prefer
          raw_fallback_path_key so actionable sensible warnings can be
          reported to the user about the raw unparsable user-data.
        """
        primary_datapath = paths.get_ipath(primary_path_key) or ""
        with suppress(FileNotFoundError):
            if not os.stat(primary_datapath).st_size:
                raw_path = paths.get_ipath(raw_fallback_path_key) or ""
                if os.stat(raw_path).st_size:
                    return raw_path
        return primary_datapath

    try:
        paths = read_cfg_paths(fetch_existing_datasource="trust")
    except (IOError, OSError) as e:
        if e.errno == EACCES:
            LOG.debug(
                "Using default instance-data/user-data paths for non-root user"
            )
            paths = read_cfg_paths()
        else:
            raise
    except DataSourceNotFoundException:
        paths = read_cfg_paths()
        LOG.warning(
            "datasource not detected, using default"
            " instance-data/user-data paths."
        )
    if args.instance_data:
        instance_data_path = args.instance_data
    elif os.getuid() != 0:
        instance_data_path = paths.get_runpath("instance_data")
    else:
        instance_data_path = paths.get_runpath("instance_data_sensitive")
    config_files: List[InstanceDataPart] = []
    if args.config_file:
        if args.schema_type:
            schema_type = SchemaType(args.schema_type)
        else:
            schema_type = SchemaType.CLOUD_CONFIG
        if schema_type == SchemaType.NETWORK_CONFIG:
            instancedata_type = InstanceDataType.NETWORK_CONFIG
        else:
            instancedata_type = InstanceDataType.USERDATA
        config_files.append(
            InstanceDataPart(instancedata_type, schema_type, args.config_file)
        )
    else:
        if os.getuid() != 0:
            error(
                "Unable to read system userdata or vendordata as non-root"
                " user. Try using sudo.",
                sys_exit=True,
            )
        userdata_file = get_processed_or_fallback_path(
            paths, "cloud_config", "userdata_raw"
        )
        config_files.append(
            InstanceDataPart(
                InstanceDataType.USERDATA,
                SchemaType.CLOUD_CONFIG,
                userdata_file,
            )
        )
        supplemental_config_files: List[InstanceDataPart] = [
            InstanceDataPart(
                InstanceDataType.VENDORDATA,
                SchemaType.CLOUD_CONFIG,
                get_processed_or_fallback_path(
                    paths, "vendor_cloud_config", "vendordata_raw"
                ),
            ),
            InstanceDataPart(
                InstanceDataType.VENDOR2DATA,
                SchemaType.CLOUD_CONFIG,
                get_processed_or_fallback_path(
                    paths, "vendor2_cloud_config", "vendordata2_raw"
                ),
            ),
            InstanceDataPart(
                InstanceDataType.NETWORK_CONFIG,
                SchemaType.NETWORK_CONFIG,
                paths.get_ipath("network_config") or "",
            ),
        ]
        for data_part in supplemental_config_files:
            if data_part.config_path and os.path.exists(data_part.config_path):
                config_files.append(data_part)
    if not os.path.exists(config_files[0].config_path):
        error(
            f"Config file {config_files[0].config_path} does not exist",
            fmt="Error: {}",
            sys_exit=True,
        )
    return instance_data_path, config_files


def handle_schema_args(name, args):
    """Handle provided schema args and perform the appropriate actions."""
    _assert_exclusive_args(args)
    full_schema = get_schema()
    instance_data_path, config_files = get_config_paths_from_args(args)

    nested_output_prefix = ""
    multi_config_output = bool(len(config_files) > 1)
    if multi_config_output:
        print(
            "Found cloud-config data types: %s"
            % ", ".join(str(cfg_part.config_type) for cfg_part in config_files)
        )
        nested_output_prefix = "  "

    error_types = []
    for idx, cfg_part in enumerate(config_files, 1):
        performed_schema_validation = False
        if multi_config_output:
            print(
                f"\n{idx}. {cfg_part.config_type} at {cfg_part.config_path}:"
            )
        if cfg_part.schema_type == SchemaType.NETWORK_CONFIG:
            cfg_schema = get_schema(cfg_part.schema_type)
        else:
            cfg_schema = full_schema
        try:
            performed_schema_validation = validate_cloudconfig_file(
                cfg_part.config_path,
                cfg_schema,
                cfg_part.schema_type,
                args.annotate,
                instance_data_path,
            )
        except SchemaValidationError as e:
            if not args.annotate:
                print(
                    f"{nested_output_prefix}Invalid"
                    f" {cfg_part.config_type} {cfg_part.config_path}"
                )
                error(
                    str(e),
                    fmt=nested_output_prefix + "Error: {}\n",
                )
            error_types.append(cfg_part.config_type)
        except RuntimeError as e:
            print(f"{nested_output_prefix}Invalid {cfg_part.config_type!s}")
            error(str(e), fmt=nested_output_prefix + "Error: {}\n")
            error_types.append(cfg_part.config_type)
        else:
            if performed_schema_validation:
                if args.config_file:
                    cfg = cfg_part.config_path
                else:
                    cfg = str(cfg_part.config_type)
                print(f"{nested_output_prefix}Valid schema {cfg!s}")
    if error_types:
        error(
            ", ".join(str(error_type) for error_type in error_types),
            fmt="Error: Invalid schema: {}\n",
            sys_exit=True,
        )


def get_meta_doc(*_args, **_kwargs) -> str:
    """Provide a stub for backwards compatibility.

    This function is no longer used, but earlier versions of modules
    required this function for documentation purposes. This is a stub so
    that custom modules do not break on upgrade.
    """
    lifecycle.log_with_downgradable_level(
        logger=LOG,
        version="24.4",
        requested_level=logging.WARNING,
        msg=(
            "The 'get_meta_doc()' function is deprecated and will be removed "
            "in a future version of cloud-init."
        ),
        args=(),
    )
    return ""


def main():
    """Tool to validate schema of a cloud-config file or print schema docs."""
    parser = get_parser()
    handle_schema_args("cloudconfig-schema", parser.parse_args())
    return 0


if __name__ == "__main__":
    sys.exit(main())