Doc module render fixes

doc/rtd/conf.py

@@ -256,26 +256,81 @@ def render_property_template(prop_name, prop_cfg, prefix=""):
 
 def render_nested_properties(prop_cfg, defs, prefix):
     prop_str = ""
+    prop_types = set(["properties", "patternProperties"])
     flatten_schema_refs(prop_cfg, defs)
     if "items" in prop_cfg:
         prop_str += render_nested_properties(prop_cfg["items"], defs, prefix)
-    if not set(["properties", "patternProperties"]).intersection(prop_cfg):
-        return prop_str
-    for prop_name, nested_cfg in prop_cfg.get("properties", {}).items():
-        flatten_schema_all_of(nested_cfg)
-        flatten_schema_refs(nested_cfg, defs)
-        prop_str += render_property_template(prop_name, nested_cfg, prefix)
-        prop_str += render_nested_properties(nested_cfg, defs, prefix + "  ")
-    for prop_name, nested_cfg in prop_cfg.get("patternProperties", {}).items():
-        flatten_schema_all_of(nested_cfg)
-        flatten_schema_refs(nested_cfg, defs)
-        if nested_cfg.get("label"):
-            prop_name = nested_cfg.get("label")
-        prop_str += render_property_template(prop_name, nested_cfg, prefix)
-        prop_str += render_nested_properties(nested_cfg, defs, prefix + "  ")
+        for alt_schema in prop_cfg["items"].get("oneOf", []):
+            if prop_types.intersection(alt_schema):
+                prop_str += render_nested_properties(alt_schema, defs, prefix)
+
+    for hidden_key in prop_cfg.get("hidden", []):
+        prop_cfg.pop(hidden_key, None)
+
+    # Render visible property types
+    for prop_type in prop_types.intersection(prop_cfg):
+        for prop_name, nested_cfg in prop_cfg.get(prop_type, {}).items():
+            flatten_schema_all_of(nested_cfg)
+            flatten_schema_refs(nested_cfg, defs)
+            if nested_cfg.get("label"):
+                prop_name = nested_cfg.get("label")
+            prop_str += render_property_template(prop_name, nested_cfg, prefix)
+            prop_str += render_nested_properties(
+                nested_cfg, defs, prefix + "  "
+            )
     return prop_str
 
 
+def debug_module_docs(
+    module_id: str, mod_docs: dict, debug_file_path: str = None
+):
+    """Print rendered RST module docs during build.
+
+    The intent is to make rendered RST inconsistencies easier to see when
+    modifying jinja template files or JSON schema as white-space and format
+    inconsistencies can lead to significant sphinx rendering issues in RTD.
+
+    To trigger this inline print of rendered docs, set the environment
+    variable CLOUD_INIT_DEBUG_MODULE_DOC.
+
+    :param module_id: A specific 'cc_*' module name to print rendered RST for,
+        or provide 'all' to print out all rendered module docs.
+    :param mod_docs: A dict represnting doc metadata for each config module.
+        The dict is keyed on config module id (cc_*) and each value is a dict
+        with values such as: title, name, examples, schema_doc.
+    :param debug_file_path: A specific file to write the rendered RST content.
+        When unset,
+    """
+    from cloudinit.util import load_text_file, load_yaml
+
+    if not module_id:
+        return
+    if module_id == "all":
+        module_ids = mod_docs.keys()
+    else:
+        module_ids = [module_id]
+    rendered_content = ""
+    for mod_id in module_ids:
+        try:
+            data = load_yaml(
+                load_text_file(f"../module-docs/{mod_id}/data.yaml")
+            )
+        except FileNotFoundError:
+            continue
+        with open("templates/modules.tmpl", "r") as stream:
+            tmpl_content = "## template: jinja\n" + stream.read()
+            params = {"data": data, "config": {"html_context": mod_docs}}
+            rendered_content += render_jinja_payload(
+                tmpl_content, "changed_modules_page", params
+            )
+    if debug_file_path:
+        print(f"--- Writing rendered module docs: {debug_file_path} ---")
+        with open(debug_file_path, "w") as stream:
+            stream.write(rendered_content)
+    else:
+        print(rendered_content)
+
+
 def render_module_schemas():
     from cloudinit.importer import import_module
 
@@ -298,6 +353,11 @@ def render_module_schemas():
             mod_docs[cc_key][
                 "schema_doc"
             ] = "No schema definitions for this module"
+    debug_module_docs(
+        os.environ.get("CLOUD_INIT_DEBUG_MODULE_DOC"),
+        mod_docs,
+        debug_file_path=os.environ.get("CLOUD_INIT_DEBUG_MODULE_DOC_FILE"),
+    )
     return mod_docs
 
 

doc/rtd/templates/module_property.tmpl

@@ -6,7 +6,17 @@
 {{prefix ~ '  '}}{{ line }}
 {% endfor -%}
 {%- endmacro -%}
-{% if prop_cfg.get('items', {}).get('type') == 'object' %}
-{% set description = description ~ " Each object in **" ~ name ~ "** list supports the following keys:" %}
-{% endif %}
-{{ print_prop(name, types, description, prefix ) }}
+{% set ns = namespace(is_obj_type=false) -%}
+{% if ('properties' in prop_cfg or 'patternProperties' in prop_cfg) %}{% set ns.is_obj_type = true -%}{% endif -%}
+{% for key, val in prop_cfg.get('items', {}).items() -%}
+  {% if key in ('properties', 'patternProperties') -%}{% set ns.is_obj_type = true -%}{% endif -%}
+  {% if key == 'oneOf' -%}
+    {% for oneOf in val -%}
+      {% if ('properties' in oneOf or 'patternProperties' in oneOf ) -%}{% set ns.is_obj_type = true -%}{% endif -%}
+    {% endfor -%}
+  {% endif -%}
+{% endfor -%}
+{% if ns.is_obj_type -%}
+{% set description = description ~ " Each object in **" ~ name ~ "** list supports the following keys:" -%}
+{% endif -%}
+{{ print_prop(name, types, description, prefix ) -}}

doc/rtd/templates/property_deprecation.tmpl

@@ -1,2 +1,2 @@
 
-{{ '*Deprecated in version ' ~ deprecated_version ~ '.' ~ deprecated_description ~ '*' -}}
+{{ '*Deprecated in version ' ~ deprecated_version ~ ':* ' ~ deprecated_description -}}

tests/unittests/config/test_cc_ca_certs.py

@@ -390,10 +390,12 @@ class TestCACertsSchema:
             # Valid, yet deprecated schemas
             (
                 {"ca-certs": {"remove-defaults": True}},
-                "Cloud config schema deprecations: ca-certs:  "
-                "Deprecated in version 22.3. Use ``ca_certs`` instead.,"
-                " ca-certs.remove-defaults:  Deprecated in version 22.3"
-                ". Use ``remove_defaults`` instead.",
+                re.escape(
+                    "Cloud config schema deprecations: ca-certs:  "
+                    "Deprecated in version 22.3. Use **ca_certs** instead.,"
+                    " ca-certs.remove-defaults:  Deprecated in version 22.3"
+                    ". Use **remove_defaults** instead."
+                ),
             ),
             # Invalid schemas
             (

tests/unittests/config/test_cc_growpart.py

@@ -774,11 +774,11 @@ class TestGrowpartSchema:
                 {"growpart": {"mode": False}},
                 pytest.raises(
                     SchemaValidationError,
-                    match=(
+                    match=re.escape(
                         "Cloud config schema deprecations: "
                         "growpart.mode:  Changed in version 22.3. "
                         "Specifying a boolean ``false`` value for "
-                        "``mode`` is deprecated. Use the string ``'off'`` "
+                        "**mode** is deprecated. Use the string ``'off'`` "
                         "instead."
                     ),
                 ),

tests/unittests/config/test_cc_grub_dpkg.py

@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import re
 from unittest import mock
 
 import pytest
@@ -299,10 +300,10 @@ class TestGrubDpkgSchema:
                 {"grub-dpkg": {"grub-pc/install_devices_empty": False}},
                 pytest.raises(
                     SchemaValidationError,
-                    match=(
+                    match=re.escape(
                         "Cloud config schema deprecations: grub-dpkg:"
                         "  Deprecated in version 22.2. Use "
-                        "``grub_dpkg`` instead."
+                        "**grub_dpkg** instead."
                     ),
                 ),
                 False,

tests/unittests/config/test_cc_package_update_upgrade_install.py

@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 import logging
+import re
 from unittest import mock
 
 import pytest
@@ -299,26 +300,26 @@ class TestPackageUpdateUpgradeSchema:
             ({"packages": []}, SCHEMA_EMPTY_ERROR),
             (
                 {"apt_update": False},
-                (
+                re.escape(
                     "Cloud config schema deprecations: apt_update:  "
                     "Deprecated in version 22.2. "
-                    "Use ``package_update`` instead."
+                    "Use **package_update** instead."
                 ),
             ),
             (
                 {"apt_upgrade": False},
-                (
+                re.escape(
                     "Cloud config schema deprecations: apt_upgrade:  "
                     "Deprecated in version 22.2. "
-                    "Use ``package_upgrade`` instead."
+                    "Use **package_upgrade** instead."
                 ),
             ),
             (
                 {"apt_reboot_if_required": False},
-                (
+                re.escape(
                     "Cloud config schema deprecations: "
                     "apt_reboot_if_required:  Deprecated in version 22.2. Use "
-                    "``package_reboot_if_required`` instead."
+                    "**package_reboot_if_required** instead."
                 ),
             ),
         ],

tests/unittests/config/test_cc_ubuntu_pro.py

@@ -444,7 +444,7 @@ class TestUbuntuProSchema:
                     SchemaValidationError,
                     match=re.escape(
                         "ubuntu_advantage:  Deprecated in version 24.1."
-                        " Use ``ubuntu_pro`` instead"
+                        " Use **ubuntu_pro** instead"
                     ),
                 ),
                 # If __version__ no longer exists on jsonschema, that means

tests/unittests/config/test_cc_update_etc_hosts.py

@@ -87,10 +87,10 @@ class TestUpdateEtcHosts:
                 {"manage_etc_hosts": "template"},
                 pytest.raises(
                     SchemaValidationError,
-                    match=(
+                    match=re.escape(
                         "Cloud config schema deprecations: "
                         "manage_etc_hosts:  Changed in version 22.3. "
-                        "Use of ``template`` is deprecated, use "
+                        "Use of **template** is deprecated, use "
                         "``true`` instead."
                     ),
                 ),

tests/unittests/config/test_cc_users_groups.py

@@ -372,9 +372,11 @@ class TestUsersGroupsSchema:
                 pytest.raises(
                     SchemaValidationError,
                     match=(
-                        "Cloud config schema deprecations: "
-                        "users.0.lock-passwd:  Deprecated in version 22.3."
-                        " Use ``lock_passwd`` instead."
+                        re.escape(
+                            "Cloud config schema deprecations: "
+                            "users.0.lock-passwd:  Deprecated in version 22.3."
+                            " Use **lock_passwd** instead."
+                        )
                     ),
                 ),
                 False,
@@ -384,9 +386,11 @@ class TestUsersGroupsSchema:
                 pytest.raises(
                     SchemaValidationError,
                     match=(
-                        "Cloud config schema deprecations: "
-                        "users.0.no-create-home:  Deprecated in version 24.2."
-                        " Use ``no_create_home`` instead."
+                        re.escape(
+                            "Cloud config schema deprecations: "
+                            "users.0.no-create-home:  Deprecated in version"
+                            " 24.2. Use **no_create_home** instead."
+                        )
                     ),
                 ),
                 False,
@@ -527,15 +531,17 @@ class TestUsersGroupsSchema:
                 pytest.raises(
                     SchemaValidationError,
                     match=(
-                        "Cloud config schema deprecations: "
-                        "users.0.ssh-authorized-keys: "
-                        " Deprecated in version 18.3."
-                        " Use ``ssh_authorized_keys`` instead."
-                        ", "
-                        "users.0.uid: "
-                        " Changed in version 22.3."
-                        " The use of ``string`` type is deprecated."
-                        " Use an ``integer`` instead."
+                        re.escape(
+                            "Cloud config schema deprecations: "
+                            "users.0.ssh-authorized-keys: "
+                            " Deprecated in version 18.3."
+                            " Use **ssh_authorized_keys** instead."
+                            ", "
+                            "users.0.uid: "
+                            " Changed in version 22.3."
+                            " The use of ``string`` type is deprecated."
+                            " Use an ``integer`` instead."
+                        )
                     ),
                 ),
                 False,

tests/unittests/config/test_schema.py

@@ -2751,9 +2751,9 @@ def test_handle_schema_unable_to_read_cfg_paths(
                     apt_reboot_if_required: true            # D3
 
                     # Deprecations: -------------
-                    # D1: Deprecated in version 22.2. Use ``package_update`` instead.
-                    # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead.
-                    # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead.
+                    # D1: Deprecated in version 22.2. Use **package_update** instead.
+                    # D2: Deprecated in version 22.2. Use **package_upgrade** instead.
+                    # D3: Deprecated in version 22.2. Use **package_reboot_if_required** instead.
 
                     Valid schema {cfg_file}
                     """  # noqa: E501
@@ -2773,9 +2773,9 @@ def test_handle_schema_unable_to_read_cfg_paths(
                     apt_reboot_if_required: true            # D3
 
                     # Deprecations: -------------
-                    # D1: Deprecated in version 22.2. Use ``package_update`` instead.
-                    # D2: Deprecated in version 22.2. Use ``package_upgrade`` instead.
-                    # D3: Deprecated in version 22.2. Use ``package_reboot_if_required`` instead.
+                    # D1: Deprecated in version 22.2. Use **package_update** instead.
+                    # D2: Deprecated in version 22.2. Use **package_upgrade** instead.
+                    # D3: Deprecated in version 22.2. Use **package_reboot_if_required** instead.
 
                     Valid schema {cfg_file}
                     """  # noqa: E501
@@ -2789,9 +2789,9 @@ def test_handle_schema_unable_to_read_cfg_paths(
                     """\
                     Cloud config schema deprecations: \
 apt_reboot_if_required: Deprecated in version 22.2. Use\
- ``package_reboot_if_required`` instead., apt_update: Deprecated in version\
- 22.2. Use ``package_update`` instead., apt_upgrade: Deprecated in version\
- 22.2. Use ``package_upgrade`` instead.\
+ **package_reboot_if_required** instead., apt_update: Deprecated in version\
+ 22.2. Use **package_update** instead., apt_upgrade: Deprecated in version\
+ 22.2. Use **package_upgrade** instead.\
                     Valid schema {cfg_file}
                     """  # noqa: E501
                 ),

tox.ini

@@ -184,6 +184,8 @@ deps = -r{toxinidir}/doc-requirements.txt
 commands =
     {envpython} -m sphinx {posargs:-W doc/rtd doc/rtd_html}
     {envpython} -m doc8 doc/rtd
+passenv =
+    CLOUD_INIT_*
 
 [testenv:doc-spelling]
 deps = -r{toxinidir}/doc-requirements.txt