Merge pull request #326 from larsewi/cfbs-render-input

ENT-11346: Added the 'cfbs render-input' command

Author: larsewi

JSON.md

@@ -477,6 +477,9 @@ Some modules allow for users to add module input by responding to questions expr
 User input can be added using the `cfbs input <module-name>` command, which stores responses in `./<module-name>/input.json`.
 These responses are translated into augments which will be added to `./out/masterfiles/def.json` during `cfbs build`.
 
+The `cfbs render-input` command performs the same translation, but writes the augments to a file of your choosing or stdout.
+Augments rendered with `cfbs render-input` will not be added to `./out/masterfiles/def.json` automatically during a build.
+
 ### Create single file example
 
 The `"input"` attribute takes a list of input definitions as illustrated below.

README.md

@@ -150,6 +150,43 @@ Here is an example of an `input.json` file with responses:
 
 The `input.json` file is converted and merged into the main `def.json` during the build, using the `input` build step.
 
+### Render module input as an augments file
+
+```
+cfbs render-input <module> <infile (or - for stdin)> <outfile (or - for stdout)>
+```
+
+Converts input data for a module, as shown above, into the augments (`def.json`) format, and writes it to the outfile.
+This is the same conversion the `input` build step performs during `cfbs build`, except that the input data is read from the infile instead of `<module-name>/input.json`, and nothing is stored in the project.
+
+Use it to render an augment for input data which is kept outside of the project, for example input entered per host group in Mission Portal:
+
+```
+$ cfbs render-input create-single-file - - <<EOF
+[
+  {
+    "type": "string",
+    "variable": "filename",
+    "label": "Filename",
+    "question": "What file should this module create?",
+    "response": "/tmp/create-single-file.txt"
+  }
+]
+EOF
+{
+  "variables": {
+    "cfbs:create_single_file.filename": {
+      "value": "/tmp/create-single-file.txt",
+      "comment": "Added by 'cfbs input'"
+    }
+  }
+}
+```
+
+The input data must conform with the module's input definition, just like for `cfbs set-input`.
+A variable is only added to the augment if the input data has a `response` for it.
+Thus, if none of the questions have been answered, the rendered augment contains no variables at all.
+
 ### Deploy your policy set to a remote hub
 
 ```
@@ -288,6 +325,9 @@ These commands are intended to be run as part of build systems / deployment pipe
   Empty list `[]` is returned if the module was found, but it does not accept any input.
 - `cfbs install`: Run this on a hub as root to install the policy set (copy the files from `out/masterfiles` to `/var/cfengine/masterfiles`).
 - `cfbs pretty`: Run on a JSON file to pretty-format it. (May be expanded to other formats in the future).
+- `cfbs render-input`: Convert input data for a module into an augments file (`def.json`) and print it.
+  Takes the same input data as `cfbs set-input`, validates it the same way, but stores nothing - the augment is written to the given outfile instead.
+  Useful for rendering the augment for input data which is not stored in the project, for example input entered per host group in Mission Portal.
 - `cfbs set-input`: Set input data for a module.
   Non-interactive version of `cfbs input`, takes the input as a JSON, validates it and stores it.
   `cfbs set-input` and `cfbs get-input` can be thought of as ways to save and load the input file.

cfbs/augments.py

@@ -0,0 +1,44 @@
+"""
+Functions for generating CFEngine augments (def.json)
+"""
+
+from collections import OrderedDict
+
+from cfbs.utils import canonify
+
+
+def generate_augment(module_name, input_data):
+    """
+    Generate augment from input data.
+
+    :param module_name: name of module
+    :param input_data: input data
+    :return: generated augment or None if input data is incomplete
+    """
+    if not isinstance(input_data, list):
+        return None
+
+    # OrderedDict, so that the keys are in the same order regardless of
+    # the Python version. Dictionaries don't preserve the insertion order
+    # before Python 3.7:
+    augment = OrderedDict()
+    augment["variables"] = OrderedDict()
+
+    for variable in input_data:
+        if not isinstance(variable, dict) or any(
+            key not in variable for key in ("variable", "response")
+        ):
+            continue
+
+        name = variable["variable"]
+        namespace = variable.get("namespace", "cfbs")
+        bundle = variable.get("bundle", canonify(module_name))
+        value = variable["response"]
+        comment = variable.get("comment", "Added by 'cfbs input'")
+
+        augment_variable = OrderedDict()
+        augment_variable["value"] = value
+        augment_variable["comment"] = comment
+        augment["variables"]["%s:%s.%s" % (namespace, bundle, name)] = augment_variable
+
+    return augment

cfbs/build.py

@@ -15,10 +15,10 @@
 import logging as log
 import shutil
 import subprocess
+from cfbs.augments import generate_augment
 from cfbs.cfbs_config import CFBSConfig
 from cfbs.utils import (
     CFBSUserError,
-    canonify,
     cli_tool_present,
     cp,
     cp_dry_overwrites,
@@ -54,39 +54,6 @@ def init_out_folder():
     mkdir("out/steps")
 
 
-def _generate_augment(module_name, input_data):
-    """
-    Generate augment from input data.
-
-    :param module_name: name of module
-    :param input_data: input data
-    :return: generated augment or None if input data is incomplete
-    """
-    if not isinstance(input_data, list):
-        return None
-
-    augment = {"variables": {}}
-
-    for variable in input_data:
-        if not isinstance(variable, dict) or any(
-            key not in variable for key in ("variable", "response")
-        ):
-            continue
-
-        name = variable["variable"]
-        namespace = variable.get("namespace", "cfbs")
-        bundle = variable.get("bundle", canonify(module_name))
-        value = variable["response"]
-        comment = variable.get("comment", "Added by 'cfbs input'")
-
-        augment["variables"]["%s:%s.%s" % (namespace, bundle, name)] = {
-            "value": value,
-            "comment": comment,
-        }
-
-    return augment
-
-
 def _perform_replacement(n, a, b, filename):
     assert n and a and b and filename
     assert a not in b
@@ -297,7 +264,7 @@ def _perform_input_step(args, name, destination, prefix):
         )
         return
     extras, original = read_json(src), read_json(dst)
-    extras = _generate_augment(name, extras)
+    extras = generate_augment(name, extras)
     log.debug("Generated augment: %s", pretty(extras))
     if not extras:
         raise CFBSExitError(

cfbs/cfbs_config.py

@@ -449,7 +449,8 @@ def _add_modules(
         ]
         modules_already_added = self["build"]
 
-        assert not any(m for m in modules_to_add if "name" not in m)
+        # check_existence() above ensures all of them are in the index:
+        assert not any(m for m in modules_to_add if m is None or "name" not in m)
         assert not any(m for m in modules_already_added if "name" not in m)
 
         # Find all unmet dependencies:

cfbs/cfbs_json.py

@@ -215,7 +215,7 @@ def _module_is_in_build(self, module):
         return "build" in self and module["name"] in (m["name"] for m in self["build"])
 
     def get_module_from_build(self, module):
-        for m in self["build"]:
+        for m in self.get("build") or []:
             if m["name"] == module:
                 return m
         return None

cfbs/commands.py

@@ -92,12 +92,14 @@ def search_command(terms: List[str]):
     pretty_file,
     CFBS_DEFAULT_SORTING_RULES,
 )
+from cfbs.augments import generate_augment
 from cfbs.build import (
     init_out_folder,
     perform_build,
 )
 from cfbs.cfbs_config import CFBSConfig, CFBSReturnWithoutCommit
 from cfbs.validate import (
+    input_data_matches_spec,
     validate_config,
     validate_config_raise_exceptions,
     validate_module_name_content,
@@ -703,6 +705,9 @@ def update_command(to_update):
             new_module = provides[module_name]
         elif is_module_absolute(old_module["name"]):
             new_module = index.get_module_object(update.name)
+            # Module objects for absolute modules are generated, not looked up
+            # in the index, so this is never None:
+            assert new_module is not None
             new_module["commit"] = head_commit_hash(old_module["name"])
         else:
 
@@ -1268,7 +1273,10 @@ def cfbs_convert_git_commit(
         raise
 
     current_index = CFBSConfig.get_instance().index
-    default_version = current_index.get_module_object("masterfiles")["version"]
+    masterfiles = current_index.get_module_object("masterfiles")
+    if masterfiles is None:
+        raise CFBSExitError("Could not find the 'masterfiles' module in the index")
+    default_version = masterfiles["version"]
 
     reference_version = analyzed_files.reference_version
     if reference_version is None:
@@ -1603,48 +1611,11 @@ def set_input_command(name, infile):
         return CFBSCommandGitResult(1)
     log.debug("Input data for module '%s': %s" % (name, pretty(data)))
 
-    def _compare_dict(a, b, ignore=None):
-        assert isinstance(a, dict) and isinstance(b, dict)
-        ignore = ignore or set()
-        if set(a.keys()) != set(b.keys()) - ignore:
-            return False
-        # Avoid code duplication by converting the values of the two dicts
-        # into two lists in the same order and compare the lists instead
-        keys = a.keys()
-        return _compare_list([a[key] for key in keys], [b[key] for key in keys])
-
-    def _compare_list(a, b):
-        assert isinstance(a, list) and isinstance(b, list)
-        if len(a) != len(b):
-            return False
-        for x, y in zip(a, b):
-            if type(x) is not type(y):
-                return False
-            if isinstance(x, dict):
-                if not _compare_dict(x, y):
-                    return False
-            elif isinstance(x, list):
-                if not _compare_list(x, y):
-                    return False
-            else:
-                assert x is None or isinstance(
-                    x, (int, float, str, bool)
-                ), "Illegal value type"
-                if x != y:
-                    return False
-        return True
-
-    for a, b in zip(spec, data):
-        if (
-            not isinstance(a, dict)
-            or not isinstance(b, dict)
-            or not _compare_dict(a, b, ignore=set({"response"}))
-        ):
-            log.error(
-                "Input data for module '%s' does not conform with input definition"
-                % name
-            )
-            return CFBSCommandGitResult(1)
+    if not input_data_matches_spec(spec, data):
+        log.error(
+            "Input data for module '%s' does not conform with input definition" % name
+        )
+        return CFBSCommandGitResult(1)
 
     path = os.path.join(name, "input.json")
 
@@ -1692,3 +1663,44 @@ def get_input_command(name, outfile):
         log.error("Failed to write json: %s" % e)
         return 1
     return 0
+
+
+@cfbs_command("render-input")
+def render_input_command(name, infile, outfile):
+    config = CFBSConfig.get_instance()
+    config.warn_about_unknown_keys()
+    module = config.get_module_from_build(name)
+    if module is None:
+        module = config.index.get_module_object(name)
+    if module is None:
+        log.error("Module '%s' not found" % name)
+        return 1
+
+    spec = module.get("input")
+    if spec is None:
+        log.error("Module '%s' does not accept input" % name)
+        return 1
+    log.debug("Input spec for module '%s': %s" % (name, pretty(spec)))
+
+    try:
+        data = json.load(infile, object_pairs_hook=OrderedDict)
+    except json.decoder.JSONDecodeError as e:
+        log.error("Error reading input data for module '%s': %s" % (name, e))
+        return 1
+    log.debug("Input data for module '%s': %s" % (name, pretty(data)))
+
+    if not input_data_matches_spec(spec, data):
+        log.error(
+            "Input data for module '%s' does not conform with input definition" % name
+        )
+        return 1
+
+    augment = generate_augment(name, data)
+    log.debug("Generated augment: %s" % pretty(augment))
+
+    try:
+        outfile.write(pretty(augment) + "\n")
+    except OSError as e:
+        log.error("Failed to write json: %s" % e)
+        return 1
+    return 0

cfbs/index.py

@@ -214,7 +214,23 @@ def get_module_object(
         module,
         added_by: Optional[str] = None,
         explicit_build_steps: Optional[List[str]] = None,
+        default=None,
     ):
+        """Get the module object for a module in the index.
+
+        Local ("./name") and absolute ("/path/") modules are not in the index,
+        their module objects are generated with default build steps instead.
+
+        :param module: module name, "name@version", or a Module object
+        :param added_by: what to put in the module's "added_by" field, omitted
+                         from the module object if not specified
+        :param explicit_build_steps: build steps to use for a local subdirectory
+                                     module, instead of the default "directory"
+                                     build step
+        :param default: what to return if the module, or the requested version
+                        of it, is not in the index
+        :return: the module object, or default if it was not found
+        """
         if isinstance(module, str):
             module = Module(module)
         name = module.name
@@ -231,6 +247,8 @@ def get_module_object(
             # due to that, this hack is used to prevent creating the "version" field
             module = Module(name).to_dict()
         else:
+            if name not in self:
+                return default
             object = self[name]
             if version:
                 try:
@@ -239,6 +257,8 @@ def get_module_object(
                     raise CFBSExitError(
                         "Downloading CFEngine Build Module Index failed - check your Wi-Fi / network settings."
                     )
+                if name not in versions or version not in versions[name]:
+                    return default
                 new_values = versions[name][version]
                 specifics = {
                     k: v for (k, v) in new_values.items() if k in Module.attributes()