From db7acb3cac97937f07ac4af09f0d65085a67bafd Mon Sep 17 00:00:00 2001 From: Alberto Contreras Date: Thu, 25 Jul 2024 14:55:06 +0200 Subject: [PATCH] doc: restructure existent drop-in functionality (#5548) --- doc/rtd/explanation/format.rst | 41 +----------- doc/rtd/reference/cli.rst | 26 +------- doc/rtd/reference/custom_modules.rst | 17 +++++ .../custom_modules/custom_cleaners.rst | 28 ++++++++ .../custom_configuration_module.rst | 4 ++ .../custom_modules/custom_datasource.rst | 4 ++ .../custom_modules/custom_formats.rst | 51 ++++++++++++++ .../custom_modules/custom_mergers.rst | 66 +++++++++++++++++++ doc/rtd/reference/index.rst | 1 + doc/rtd/reference/merging.rst | 60 +---------------- 10 files changed, 177 insertions(+), 121 deletions(-) create mode 100644 doc/rtd/reference/custom_modules.rst create mode 100644 doc/rtd/reference/custom_modules/custom_cleaners.rst create mode 100644 doc/rtd/reference/custom_modules/custom_configuration_module.rst create mode 100644 doc/rtd/reference/custom_modules/custom_datasource.rst create mode 100644 doc/rtd/reference/custom_modules/custom_formats.rst create mode 100644 doc/rtd/reference/custom_modules/custom_mergers.rst diff --git a/doc/rtd/explanation/format.rst b/doc/rtd/explanation/format.rst index c1eda9006d97..bb5f74ff6292 100644 --- a/doc/rtd/explanation/format.rst +++ b/doc/rtd/explanation/format.rst @@ -172,44 +172,8 @@ using a MIME archive. Part-handler ============ -This is a `part-handler`: It contains custom code for either supporting new -mime-types in multi-part user data, or overriding the existing handlers for -supported mime-types. It will be written to a file in -:file:`/var/lib/cloud/data` based on its filename (which is generated). - -This must be Python code that contains a ``list_types`` function and a -``handle_part`` function. Once the section is read the ``list_types`` method -will be called. It must return a list of mime-types that this `part-handler` -handles. Since MIME parts are processed in order, a `part-handler` part -must precede any parts with mime-types it is expected to handle in the same -user data. - -The ``handle_part`` function must be defined like: - -.. code-block:: python - - def handle_part(data, ctype, filename, payload): - # data = the cloudinit object - # ctype = "__begin__", "__end__", or the mime-type of the part that is being handled. - # filename = the filename of the part (or a generated filename if none is present in mime data) - # payload = the parts' content - -``Cloud-init`` will then call the ``handle_part`` function once before it -handles any parts, once per part received, and once after all parts have been -handled. The ``'__begin__'`` and ``'__end__'`` sentinels allow the part -handler to do initialisation or teardown before or after receiving any parts. - -Begins with: ``#part-handler`` or ``Content-Type: text/part-handler`` when -using a MIME archive. - -Example -------- - -.. literalinclude:: ../../examples/part-handler.txt - :language: python - :linenos: - -Also, `this blog post`_ offers another example for more advanced usage. +This is a `part-handler`. It serves for implementing custom formats, +see :ref:`custom_formats`. Disabling user data =================== @@ -222,4 +186,3 @@ appliances. Setting ``allow_userdata: false`` in the configuration will disable .. _make-mime: https://github.com/canonical/cloud-init/blob/main/cloudinit/cmd/devel/make_mime.py .. _YAML version 1.1: https://yaml.org/spec/1.1/current.html .. [#] See your cloud provider for applicable user-data size limitations... -.. _this blog post: http://foss-boss.blogspot.com/2011/01/advanced-cloud-init-custom-handlers.html diff --git a/doc/rtd/reference/cli.rst b/doc/rtd/reference/cli.rst index 9c0bbe9c3ee9..4bb602802e9a 100644 --- a/doc/rtd/reference/cli.rst +++ b/doc/rtd/reference/cli.rst @@ -83,30 +83,8 @@ re-run all stages as it did on first boot. .. note:: - Cloud-init provides the directory :file:`/etc/cloud/clean.d/` for third party - applications which need additional configuration artifact cleanup from - the filesystem when the `clean` command is invoked. - - The :command:`clean` operation is typically performed by image creators - when preparing a golden image for clone and redeployment. The clean command - removes any cloud-init semaphores, allowing cloud-init to treat the next - boot of this image as the "first boot". When the image is next booted - cloud-init will performing all initial configuration based on any valid - datasource meta-data and user-data. - - Any executable scripts in this subdirectory will be invoked in lexicographical - order with run-parts when running the :command:`clean` command. - - Typical format of such scripts would be a ##- like the following: - :file:`/etc/cloud/clean.d/99-live-installer` - - An example of a script is: - - .. code-block:: bash - - sudo rm -rf /var/lib/installer_imgs/ - sudo rm -rf /var/log/installer/ - + What `clean` cleans can be supplemented / customized. See: + :ref:`custom_cleaners`. .. _cli_collect_logs: diff --git a/doc/rtd/reference/custom_modules.rst b/doc/rtd/reference/custom_modules.rst new file mode 100644 index 000000000000..687190d6cce1 --- /dev/null +++ b/doc/rtd/reference/custom_modules.rst @@ -0,0 +1,17 @@ +Custom Modules +************** + +Our reference section contains support information for ``cloud-init``. +This includes details on the network requirements, API definitions, support +matrices and so on. + +----- + +.. toctree:: + :maxdepth: 1 + + custom_modules/custom_cleaners.rst + custom_modules/custom_configuration_module.rst + custom_modules/custom_datasource.rst + custom_modules/custom_formats.rst + custom_modules/custom_mergers.rst diff --git a/doc/rtd/reference/custom_modules/custom_cleaners.rst b/doc/rtd/reference/custom_modules/custom_cleaners.rst new file mode 100644 index 000000000000..d7aec7836932 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_cleaners.rst @@ -0,0 +1,28 @@ +.. _custom_cleaners: + +Custom Cleaners +*************** + +Cloud-init provides the directory :file:`/etc/cloud/clean.d/` for third party +applications which need additional configuration artifact cleanup from +the filesystem when the :ref:`cloud-init clean` command is invoked. + +The :command:`clean` operation is typically performed by image creators +when preparing a golden image for clone and redeployment. The clean command +removes any cloud-init semaphores, allowing cloud-init to treat the next +boot of this image as the "first boot". When the image is next booted +cloud-init will performing all initial configuration based on any valid +datasource meta-data and user-data. + +Any executable scripts in this subdirectory will be invoked in lexicographical +order with run-parts when running the :command:`clean` command. + +Typical format of such scripts would be a ##- like the following: +:file:`/etc/cloud/clean.d/99-live-installer` + +An example of a script is: + +.. code-block:: bash + + sudo rm -rf /var/lib/installer_imgs/ + sudo rm -rf /var/log/installer/ diff --git a/doc/rtd/reference/custom_modules/custom_configuration_module.rst b/doc/rtd/reference/custom_modules/custom_configuration_module.rst new file mode 100644 index 000000000000..c12b6dd1b1a5 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_configuration_module.rst @@ -0,0 +1,4 @@ +.. _custom_configuration_module: + +Custom Configuration Module +*************************** diff --git a/doc/rtd/reference/custom_modules/custom_datasource.rst b/doc/rtd/reference/custom_modules/custom_datasource.rst new file mode 100644 index 000000000000..7b300781cc70 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_datasource.rst @@ -0,0 +1,4 @@ +.. _custom_datasource: + +Custom DataSource +***************** diff --git a/doc/rtd/reference/custom_modules/custom_formats.rst b/doc/rtd/reference/custom_modules/custom_formats.rst new file mode 100644 index 000000000000..63d48481af68 --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_formats.rst @@ -0,0 +1,51 @@ +.. _custom_formats: + +Custom Formats +************** + +One can define custom data formats by presenting a `#part-handler` +:ref:`user-data format` +config via user-data or vendor-data with the contents described in this page. + +It contains custom code for either supporting new +mime-types in multi-part user data, or overriding the existing handlers for +supported mime-types. It will be written to a file in +:file:`/var/lib/cloud/data` based on its filename (which is generated). + +This must be Python code that contains a ``list_types`` function and a +``handle_part`` function. Once the section is read the ``list_types`` method +will be called. It must return a list of mime-types that this `part-handler` +handles. Since MIME parts are processed in order, a `part-handler` part +must precede any parts with mime-types it is expected to handle in the same +user data. + +The ``handle_part`` function must be defined like: + +.. code-block:: python + + #part-handler + + def handle_part(data, ctype, filename, payload): + # data = the cloudinit object + # ctype = "__begin__", "__end__", or the mime-type of the part that is being handled. + # filename = the filename of the part (or a generated filename if none is present in mime data) + # payload = the parts' content + +``Cloud-init`` will then call the ``handle_part`` function once before it +handles any parts, once per part received, and once after all parts have been +handled. The ``'__begin__'`` and ``'__end__'`` sentinels allow the part +handler to do initialisation or teardown before or after receiving any parts. + +Begins with: ``#part-handler`` or ``Content-Type: text/part-handler`` when +using a MIME archive. + +Example +------- + +.. literalinclude:: ../../../examples/part-handler.txt + :language: python + :linenos: + +Also, `this blog post`_ offers another example for more advanced usage. + +.. _this blog post: http://foss-boss.blogspot.com/2011/01/advanced-cloud-init-custom-handlers.html diff --git a/doc/rtd/reference/custom_modules/custom_mergers.rst b/doc/rtd/reference/custom_modules/custom_mergers.rst new file mode 100644 index 000000000000..c277983b140d --- /dev/null +++ b/doc/rtd/reference/custom_modules/custom_mergers.rst @@ -0,0 +1,66 @@ +.. _custom_mergers: + +Custom Mergers +************** + +TODO adapt and backreference + +Because the default +:ref:`merging` +algorithms and stragies may not always be desired, +the concept of customised merging was introduced through `merge classes`. + +A `merge class` is a class definition providing functions that can be used +to merge a given type with another given type. + +An example of one of these `merging classes` is the following: + +.. code-block:: python + + class Merger: + def __init__(self, merger, opts): + self._merger = merger + self._overwrite = 'overwrite' in opts + + # This merging algorithm will attempt to merge with + # another dictionary, on encountering any other type of object + # it will not merge with said object, but will instead return + # the original value + # + # On encountering a dictionary, it will create a new dictionary + # composed of the original and the one to merge with, if 'overwrite' + # is enabled then keys that exist in the original will be overwritten + # by keys in the one to merge with (and associated values). Otherwise + # if not in overwrite mode the 2 conflicting keys themselves will + # be merged. + def _on_dict(self, value, merge_with): + if not isinstance(merge_with, (dict)): + return value + merged = dict(value) + for (k, v) in merge_with.items(): + if k in merged: + if not self._overwrite: + merged[k] = self._merger.merge(merged[k], v) + else: + merged[k] = v + else: + merged[k] = v + return merged + +As you can see, there is an ``_on_dict`` method here that will be given a +source value, and a value to merge with. The result will be the merged object. + +This code itself is called by another merging class which "directs" the +merging to happen by analysing the object types to merge, and attempting to +find a known object that will merge that type. An example of this can be found +in the :file:`mergers/__init__.py` file (see ``LookupMerger`` and +``UnknownMerger``). + +So, following the typical ``cloud-init`` approach of allowing source code to +be downloaded and used dynamically, it is possible for users to inject their +own merging files to handle specific types of merging as they choose (the +basic ones included will handle lists, dicts, and strings). Note how each +merge can have options associated with it, which affect how the merging is +performed. For example, a dictionary merger can be told to overwrite instead +of attempting to merge, or a string merger can be told to append strings +instead of discarding other strings to merge with. diff --git a/doc/rtd/reference/index.rst b/doc/rtd/reference/index.rst index 14e754b295f7..d1791fa96314 100644 --- a/doc/rtd/reference/index.rst +++ b/doc/rtd/reference/index.rst @@ -25,3 +25,4 @@ matrices and so on. ubuntu_stable_release_updates.rst breaking_changes.rst user_files.rst + custom_modules.rst diff --git a/doc/rtd/reference/merging.rst b/doc/rtd/reference/merging.rst index 7f1fc022f178..097892e25360 100644 --- a/doc/rtd/reference/merging.rst +++ b/doc/rtd/reference/merging.rst @@ -94,64 +94,8 @@ merging is done on other types. Customisation ============= -Because the above merging algorithm may not always be desired (just as the -previous merging algorithm was not always the preferred one), the concept of -customised merging was introduced through `merge classes`. - -A `merge class` is a class definition providing functions that can be used -to merge a given type with another given type. - -An example of one of these `merging classes` is the following: - -.. code-block:: python - - class Merger: - def __init__(self, merger, opts): - self._merger = merger - self._overwrite = 'overwrite' in opts - - # This merging algorithm will attempt to merge with - # another dictionary, on encountering any other type of object - # it will not merge with said object, but will instead return - # the original value - # - # On encountering a dictionary, it will create a new dictionary - # composed of the original and the one to merge with, if 'overwrite' - # is enabled then keys that exist in the original will be overwritten - # by keys in the one to merge with (and associated values). Otherwise - # if not in overwrite mode the 2 conflicting keys themselves will - # be merged. - def _on_dict(self, value, merge_with): - if not isinstance(merge_with, (dict)): - return value - merged = dict(value) - for (k, v) in merge_with.items(): - if k in merged: - if not self._overwrite: - merged[k] = self._merger.merge(merged[k], v) - else: - merged[k] = v - else: - merged[k] = v - return merged - -As you can see, there is an ``_on_dict`` method here that will be given a -source value, and a value to merge with. The result will be the merged object. - -This code itself is called by another merging class which "directs" the -merging to happen by analysing the object types to merge, and attempting to -find a known object that will merge that type. An example of this can be found -in the :file:`mergers/__init__.py` file (see ``LookupMerger`` and -``UnknownMerger``). - -So, following the typical ``cloud-init`` approach of allowing source code to -be downloaded and used dynamically, it is possible for users to inject their -own merging files to handle specific types of merging as they choose (the -basic ones included will handle lists, dicts, and strings). Note how each -merge can have options associated with it, which affect how the merging is -performed. For example, a dictionary merger can be told to overwrite instead -of attempting to merge, or a string merger can be told to append strings -instead of discarding other strings to merge with. +Custom 3rd party mergers can be defined, for more info visit +:ref:`custom_mergers`. How to activate ===============