refactor(map): compound matchers like parsing with `libmatchers`

openssh/libmatchers.jinja

@@ -0,0 +1,208 @@
+{#- -*- coding: utf-8 -*- #}
+{#- vim: ft=jinja #}
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split("/")[0] %}
+{%- from tplroot ~ "/libsaltcli.jinja" import cli %}
+
+{%- set query_map = {
+      "C": "config.get",
+      "G": "grains.get",
+      "I": "pillar.get",
+    } %}
+
+{#- When no part before `@` is provided: #}
+{#-   - define a filename path, noted `F` #}
+{#-   - use `salt["config.get"]`, noted `C` #}
+{%- set _defaults = {
+      "type": "F",
+      "query_type": "C",
+    } %}
+
+{%- macro parse_matchers(
+      matchers,
+      config_get_strategy=None,
+      log_prefix="libmatchers: "
+    ) %}
+{#-   matcher format is `[<TYPE>[:<OPTION>]@]<KEY>` #}
+{#-   each matcher has a type: #}
+{#-   - `F` to build a file name (the default when no type is set) #}
+{#-   - `C` to lookup values with `config.get` #}
+{#-   - `G` to lookup values with `grains.get` #}
+{#-   - `I` to lookup values with `pillar.get` #}
+{#-   The `FILE` type option can define query type to build the file name: #}
+{#-   - `C` for query with `config.get` (the default when to query type is set) #}
+{#-   - `G` for query with `grains.get` #}
+{#-   - `I` for query with `pillar.get` #}
+{%-   set parsed_matchers = [] %}
+{%-   for matcher in matchers %}
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "process matcher: '"
+          ~ matcher
+          ~ "'"
+        ) %}
+
+{%-     set parsed = {} %}
+{%-     set matcher_parts = matcher.split('@') %}
+{%-     if matcher_parts | length == 1 %}
+{#-       By default we load YAML files for config looked up by `config.get` #}
+{%-       do parsed.update(
+            {
+              "type": _defaults["type"],
+              "option": None,
+              "query_method": query_map[_defaults["query_type"]],
+              "query": matcher
+            }
+          ) %}
+{%-       do salt["log.debug"](
+            log_prefix
+            ~ "use built-in defaults for matcher:\n"
+            ~ parsed
+            | yaml(False)
+            | indent(4, True)
+          ) %}
+{%-     else %}
+{%-       do salt["log.debug"](
+            log_prefix
+            ~ "parse matcher: '"
+            ~ matcher
+            ~ "'"
+          ) %}
+{%-       set metadatas = matcher_parts[0].split(":") %}
+{%-       do parsed.update(
+            {
+              "query": matcher_parts[1]
+            }
+          ) %}
+{%-       if metadatas | length == 1 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": "C",
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 1 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 2 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 2 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 3 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 3 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       elif metadatas | length == 4 %}
+{%-         do parsed.update(
+              {
+                "type": metadatas[0],
+                "option": metadatas[1],
+              }
+            ) %}
+{%-         do salt["log.debug"](
+              log_prefix
+              ~ "parse as 4 metadata matcher:\n"
+              ~ parsed
+              | yaml(False)
+              | indent(4, True)
+            ) %}
+{%-       endif %}
+{%-     endif %}
+
+{#-     The `<OPTION>` has different meaning based on type #}
+{%-     if query_map.get(parsed.type, False) %}
+{%-       do parsed.update(
+            {
+              "query_method": query_map[parsed.type]
+            }
+          ) %}
+{%-     else %}
+{%-       do parsed.update(
+            {
+              "query_method": query_map[
+                parsed.option
+                | default("C", boolean=True)
+              ]
+            }
+          ) %}
+{%-     endif %}
+
+{#-     Add `merge:` option to `salt["config.get"]` if configured #}
+{%-     if cli in ["minion", "local"] and parsed.query_method == "config.get" and config_get_strategy %}
+{%-       set merge_opt = {"merge": config_get_strategy} %}
+{%-       set merge_msg = (
+            ", merge: strategy='"
+            ~ config_get_strategy
+            ~ "'"
+          ) %}
+{%-     else %}
+{%-       if cli not in ["minion", "local"] %}
+{%-         do salt["log.error"](
+              log_prefix
+              ~ "the 'merge' option of 'config.get' is skipped when the salt command type is '"
+              ~ cli
+              ~ "'"
+            ) %}
+{%-       endif %}
+{%-       set merge_opt = {} %}
+{%-       set merge_msg = "" %}
+{%-     endif %}
+
+{%-     do salt["log.debug"](
+          log_prefix
+          ~ "lookup '"
+          ~ parsed.query
+          ~ "' with '"
+          ~ parsed.query_method
+          ~ "'"
+          ~ merge_msg
+        ) %}
+{%-     set values = salt[parsed.query_method](
+          parsed.query,
+          default=[],
+          **merge_opt
+        ) %}
+{%-     do parsed.update(
+          {
+            "value": values
+          }
+        ) %}
+
+{%-     do parsed_matchers.append(parsed) %}
+
+{%-   endfor %}
+{%-   do salt["log.debug"](
+        log_prefix
+        ~ "parsed matchers:\n"
+        ~ parsed_matchers
+        | yaml(False)
+        | indent(4, True)
+      ) %}
+
+{{ parsed_matchers | yaml }}
+{%- endmacro %}

openssh/map.jinja

@@ -3,7 +3,7 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split("/")[0] %}
-{%- from tplroot ~ "/libsaltcli.jinja" import cli with context %}
+{%- from tplroot ~ "/libmatchers.jinja" import parse_matchers, query_map %}
 
 {#- Where to lookup parameters source files #}
 {%- set map_sources_dir = tplroot ~ "/parameters" %}
@@ -114,93 +114,24 @@
       "merge_lists": salt["config.get"](tplroot ~ ":merge_lists", False),
     } %}
 
-{#- the `config.get` merge option only works for `minion` or `local` salt command types #}
-{%- if cli in ["minion", "local"] %}
-{%-   do _config.update(
-        {
-          "merge_opt": {"merge": _config["merge_strategy"]},
-          "merge_msg": ", merge: strategy='" ~ _config["merge_strategy"] ~ "'",
-        }
-      ) %}
-{#- the `config.get` merge option is not available for `ssh` or `unknown` salt command types #}
-{%- else %}
-{%-   if _config["merge_strategy"] %}
-{%-     do salt["log.error"](
-          "map.jinja: the 'merge' option of 'config.get' is skipped when the salt command type is '"
-          ~ cli
-          ~ "'"
-        ) %}
-{%-   endif %}
-{%-   do _config.update(
-        {
-          "merge_opt": {},
-          "merge_msg": "",
-        }
-      ) %}
-{%- endif %}
-
-{%- set query_map = {
-      "C": "config.get",
-      "G": "grains.get",
-      "I": "pillar.get",
-    } %}
-
-{#- Process each `map.jinja` source #}
-{#- each source has a type: #}
-{#- - `Y` to load values from YAML files (the default when no type is set) #}
-{#- - `C` to lookup values with `config.get` #}
-{#- - `G` to lookup values with `grains.get` #}
-{#- - `I` to lookup values with `pillar.get` #}
-{#- The YAML type option can define query type to build the file name: #}
-{#- - `C` for query with `config.get` (the default when to query type is set) #}
-{#- - `G` for query with `grains.get` #}
-{#- - `I` for query with `pillar.get` #}
-{#- The `C`, `G` or `I` types can define the `SUB` option #}
-{#- to merge values in the sub key `mapdata.<key>` instead of directly in `mapdata` #}
-{%- for map_source in map_sources %}
-{%-   set source_parts = map_source.split('@') %}
-{%-   if source_parts|length == 1 %}
-{#-     By default we load YAML files for config looked up by `config.get` #}
-{%-     set source_type = "Y" %}
-{%-     set query_type = "C" %}
-{%-     set source_key = map_source %}
-{%-   elif source_parts[0][0] == "Y" %}
-{%-     set source_type = "Y" %}
-{%-     set query_type = source_parts[0].split(':')[1] | default("C") %}
-{%-     set source_key = source_parts[1] %}
-{%-   elif source_parts[0][0] in query_map.keys() %}
-{%-     set source_type = source_parts[0].split(':') | first %}
-{%-     set query_type = source_type %}
-{%-     set is_sub_key = source_parts[0].split(':')[1] | default(False) == "SUB" %}
-{%-     set source_key = source_parts[1] %}
-{%-   endif %}
-
-{%-   set query_method = query_map[query_type] %}
-
-{%-   if source_type in query_map.keys() %}
-{#-     Lookup source `<QUERY_METHOD>@key:to:query` #}
-{%-     if source_type == "C" %}
-{%-       set merge_opts = _config["merge_opt"] %}
-{%-       set merge_msg = _config["merge_msg"] %}
-{%-     else %}
-{#-       No merging strategy supported for `grains.get` and `pillar.get` #}
-{%-       set merge_opts = {} %}
-{%-       set merge_msg = "" %}
-{%-     endif %}
-
-{%-     do salt["log.debug"](
-          "map.jinja: retrieve '"
-          ~ source_key
-          ~ "' with '"
-          ~ query_method
-          ~ "'"
-          ~ merge_msg
-        ) %}
-{%-     set _config_get = salt[query_method](
-          source_key,
-          default={},
-          **merge_opts
-        ) %}
+{#- `parse_matchers` returns a `list` of `dict` with #}
+{#-  - type: `F` to load file, `C`, `G`, `I` for lookup #}
+{#-  - option: specific to the type #}
+{#-  - query: which key is requested #}
+{#-  - query_method: the salt method doing the query `config.get`, `pillar.get` and `grains.get` #}
+{#-  - value: the result of the `salt[<query_method>](<query>)` #}
+{%- set map_matchers = parse_matchers(
+      matchers=map_sources,
+      config_get_strategy=_config["merge_strategy"],
+      log_prefix="map.jinja: "
+    )
+    | load_yaml %}
+
+{%- for matcher in map_matchers %}
+
+{%-   if matcher.type in query_map.keys() %}
+{#-     Merge in `mapdata.<query>` instead of directcly in `mapdata` #}
+{%-     set is_sub_key = matcher.option | default(False) == "SUB" %}
 
 {#-     `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
 {%-     set _strategy = _config["merge_strategy"] | default("smart", boolean=True) %}
@@ -208,9 +139,9 @@
           "map.jinja: merge "
           ~ "sub key " * is_sub_key
           ~ "'"
-          ~ source_key
+          ~ matcher.query
           ~ "' retrieved with '"
-          ~ query_method
+          ~ matcher.query_method
           ~ "', merge: strategy='"
           ~ _strategy
           ~ "', lists='"
@@ -218,15 +149,17 @@
           ~ "'"
         ) %}
 
+{#-     empty value is an empty list, it must be an empty dict for `.update` #}
+{%-     set value = matcher.value | default({}, boolean=True) %}
 {%-     if is_sub_key %}
 {#-       Merge values with `mapdata.<key>`, `<key>` and `<key>:lookup` are merged together #}
-{%-       set _config_get = { source_key.rstrip(':lookup'): _config_get } %}
+{%-       set value = { matcher.query | regex_replace("(:lookup)?$", ""): value } %}
 {%-     endif %}
 {%-     do _config.update(
           {
             "stack": salt["slsutil.merge"](
               _config["stack"],
-              _config_get,
+              value,
               strategy=_strategy,
               merge_lists=_config["merge_lists"],
             )
@@ -235,20 +168,14 @@
 {%-   else %}
 {#-     Load YAML file matching the grain/pillar/... #}
 {#-     Fallback to use the source name as a direct filename #}
-{%-     do salt["log.debug"](
-          "map.jinja: lookup '"
-          ~ source_key
-          ~ "' with '"
-          ~ query_method
-          ~ "'"
-        ) %}
-{%-     set map_values = salt[query_method](source_key, []) %}
-
 {#-     Mangle `source_key` to use it as literal path #}
-{%-     if map_values | length == 0 %}
-{%-       set map_source_parts = source_key.split("/") %}
-{%-       set source_key = map_source_parts[0:-1] | join("/") %}
-{%-       set map_values = map_source_parts[-1].rstrip(".yaml") %}
+{%-     if matcher.value | length == 0 %}
+{%-       set query_parts = matcher.query.split("/") %}
+{%-       set map_dirname = query_parts[0:-1] | join("/") %}
+{%-       set map_values = query_parts[-1] | regex_replace("(\.yaml)?$", ".yaml") %}
+{%-     else %}
+{%-       set map_dirname = matcher.query %}
+{%-       set map_values = matcher.value %}
 {%-     endif %}
 
 {#-     Some configuration return list #}
@@ -256,10 +183,10 @@
 {%-       set map_values = [map_values] %}
 {%-     endif %}
 
-{#-     `source_key` can be an empty string with literal path like `myconf.yaml` #}
+{#-     `map_dirname` can be an empty string with literal path like `myconf.yaml` #}
 {%-     set yaml_dir = [
           map_sources_dir,
-          source_key
+          map_dirname
         ]
         | select
         | join("/") %}