Orchestration file generation and metadata

salt/files/orchestrate.sls

@@ -1,43 +1,113 @@
 {%- from "salt/map.jinja" import master with context %}
-{%- if master.enabled %}
-
-{{ formula_dict }}
 
-{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+{%- if master.enabled %}
+  {%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+    {%- if master.base_environment == environment_name %}
+      {%- set sorted_priorities = priorities|dictsort(false, 'value') %}
+      {#- Set debug = True to log simple state result - Fail/True - to /var/log/salt/ on both master and respective minions #}
+      {%- set debug = False %}
+      {#- Uncomment to print parsed metadata from formula_name/meta/salt.yml to output file %}
 
-{%- if master.base_environment == environment_name %}
+PARSED METADATA: 
 
-{%- set formula_dict = environment.get('formula', {}) %}
-{%- set new_formula_dict = {} %}
+---------------
 
-{%- for formula_name, formula in formula_dict.iteritems() %}
+Passed from salt/master/orchestrate.sls
+---
 
-{%- set _tmp = new_formula_dict.update({formula_name: formula.get('orchestrate_order', 100)}) %}
+priorities: {{ priorities }}
 
-{%- endfor %}
+sorted_priorities: {{ sorted_priorities }}
 
-{%- set sorted_formula_list = new_formula_dict|dictsort(false, 'value') %}
-	
-{%- for formula in sorted_formula_list %}
+args: {{ args }}
 
-{%- if salt['file.file_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate.sls') %}
+---------------
 
-{{ salt['cmd.run']('cat /srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate.sls') }}
+      #}
 
-{%- else %}
+      {%- for state in sorted_priorities %}
+        {%- set formula = state.0.split('.') %}
 
-{{ formula.0 }}:
-  salt.state:
-    - tgt: 'services:{{ formula.0 }}'
-    - tgt_type: grain
-    - sls: {{ formula.0 }}
+        {%- if salt['file.directory_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate') and formula|length > 1 and salt['file.file_exists']('/srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate/'+formula.1+'.sls') %}
 
-{%- endif %}
+{{ salt['cmd.run']('cat /srv/salt/env/'+environment_name+'/'+formula.0+'/orchestrate/'+formula.1+'.sls') }}
 
-{%- endfor %}
+        {%- else %}
+          {%- if args[ state.0 ] is defined %}
 
+{{ state.0 }}:
+  salt.state:
+    - tgt: '{{ state.0|replace(".", ":") }}'
+    - tgt_type: pillar
+    - queue: True
+    - sls: {{ state.0 }}
+    {{ args[ state.0 ]|yaml(false)|indent(4) }}
+
+            {%- if debug %}
+
+{{ state.0 }}.logok:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: OK" >> /var/log/salt/orchestrate_runner'
+    - require:
+      - salt: {{ state.0 }}
+
+{{ state.0 }}.logfail:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: FAIL" >> /var/log/salt/orchestrate_runner'
+    - onfail:
+      - salt: {{ state.0 }}
+
+            {%- endif %}
+          {%- else %}
+
+{{ state.0 }}:
+  salt.state:
+    - tgt: '{{ state.0|replace(".", ":") }}{%- if "." not in state.0 %}:*{%- endif %}'
+    - tgt_type: pillar
+    - queue: True
+    - sls: {{ state.0 }}
+
+            {%- if debug %}
+
+{{ state.0 }}.logok:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: OK" >> /var/log/salt/orchestrate_runner'
+    - require:
+      - salt: {{ state.0 }}
+
+{{ state.0 }}.logfail:
+  salt.function:
+    - tgt: 'I@salt:master or I@{{ state.0|replace(".", ":") }}'
+    - tgt_type: compound
+    - queue: True
+    - name: cmd.run
+    - arg:
+      - 'echo "$(date +"%d %h %Y %H:%M:%S") | state: {{ state.0}} - status: FAIL" >> /var/log/salt/orchestrate_runner'
+    - onfail:
+      - salt: {{ state.0 }}
+
+            {%- endif %}
+          {%- endif %}
+        {%- endif %}
+
+      {%- endfor %}
+
+    {%- endif %}
+  {%- endfor %}
 {%- endif %}
 
-{%- endfor %}
-
-{%- endif %}

salt/master/orchestrate.sls

@@ -1,32 +1,54 @@
 {%- from "salt/map.jinja" import master with context %}
-{%- if master.enabled %}
-
-{%- for environment_name, environment in master.get('environment', {}).iteritems() %}
-
-{%- if master.base_environment == environment_name %}
-
-{%- set formula_dict = {} %}
-{%- for formula_name, formula in formula_dict.iteritems() %}
 
-{%- if salt['file.file_exists']('salt://'+formula_name+'/meta/salt.yml') %}
-{%- set grains_fragment_file = formula_name+'/meta/salt.yml' %}
-{%- macro load_grains_file() %}{% include grains_fragment_file %}{% endmacro %}
-{%- set grains_yaml = load_grains_file()|load_yaml %}
-{% _dummy = formula_dict.update{formula_name: grains_yaml.orchestrate }}
-{%- else %}
-{%- endif %}
-{%- endfor %}
-
-/srv/salt/env/{{ environment_name}}/orchestrate.sls:
+{%- if master.enabled %}
+  {%- for environment_name, environment in master.get('environment', {}).iteritems() %}
+    {%- if master.base_environment == environment_name %}
+      {%- set priorities = {} %}
+      {%- set args = {} %}
+      {%- set formulas = environment.get('formula', {}) %}
+
+      {%- for formula_name, formula in formulas.iteritems() %}
+        {%- if salt['file.file_exists'](master.dir.files+'/'+environment_name+'/'+formula_name+'/meta/salt.yml') %}
+          {%- set grains_fragment_file = formula_name+'/meta/salt.yml' %}
+          {%- macro load_grains_file() %}{% include grains_fragment_file %}{% endmacro %}
+          {%- set grains_yaml = load_grains_file()|load_yaml %}
+
+          {%- for state, priority in grains_yaml['orchestrate'].iteritems() %}
+            {%- do priorities.update({ formula_name+'.'+state: grains_yaml['orchestrate'][state]['priority'] }) %}
+            {%- set arguments = [] %}
+
+            {%- for arg_name, arg_value in grains_yaml['orchestrate'][state].iteritems() %}
+              {%- if 'priority' not in arg_name %}
+                {%- do arguments.append({arg_name: arg_value}) %}
+              {%- endif %}
+            {%- endfor %}
+
+            {%- if arguments %}
+              {%- do args.update({ formula_name+'.'+state: arguments }) %}
+            {%- endif %}
+          {%- endfor %}
+        {%- else %}
+          {%- do priorities.update({ formula_name: 10000 }) %}
+        {%- endif %}
+      {%- endfor %}
+
+{{ master.dir.files }}/{{ environment_name }}/orchestrate:
+  file.directory:
+    - user: root
+    - group: root
+    - mode: 755
+    - makedirs: True
+
+{{ master.dir.files }}/{{ environment_name }}/orchestrate/init.sls:
   file.managed:
   - source: salt://salt/files/orchestrate.sls
   - user: root
   - template: jinja
   - defaults:
-      formula_dict: {{ formula_dict|yaml }}
+      priorities: {{ priorities }}
+      args: {{ args }}
 
+    {%- endif %}
+  {%- endfor %}
 {%- endif %}
 
-{%- endfor %}
-
-{%- endif %}

salt/meta/salt.yml

@@ -1,9 +1,10 @@
 orchestrate:
   master:
-    priority: 10
-  minion:
-    priority: 20
-  syndic:
-    priority: 200
-  control:
+    priority: 60
+  minion: 
+    priority: 70
+  control: 
     priority: 400
+    require:
+    - salt: salt.master
+