Merge pull request #75 from salt-formulas/pr_proxy_advance2

Add system.env, system.profile, system.proxy and configure proxy for repo

README.rst

@@ -445,6 +445,69 @@ Also pin it's packages with priority 900.
                priority: 900
                package: '*'
 
+
+Package manager proxy setup globally:
+
+.. code-block:: yaml
+
+    linux:
+      system:
+        ...
+        repo:
+          apt-mk:
+            source: "deb http://apt-mk.mirantis.com/ stable main salt"
+        ...
+        proxy:
+          pkg:
+            enabled: true
+            ftp:   ftp://ftp-proxy-for-apt.host.local:2121
+          ...
+          # NOTE: Global defaults for any other componet that configure proxy on the system.
+          #       If your environment has just one simple proxy, set it on linux:system:proxy.
+          #
+          # fall back system defaults if linux:system:proxy:pkg has no protocol specific entries
+          # as for https and http
+          ftp:   ftp://proxy.host.local:2121
+          http:  http://proxy.host.local:3142
+          https: https://proxy.host.local:3143
+
+Package manager proxy setup per repository:
+
+.. code-block:: yaml
+
+    linux:
+      system:
+        ...
+        repo:
+          debian:
+            source: "deb http://apt-mk.mirantis.com/ stable main salt"
+        ...
+          apt-mk:
+            source: "deb http://apt-mk.mirantis.com/ stable main salt"
+            # per repository proxy
+            proxy:
+              enabled: true
+              http:  http://maas-01:8080
+              https: http://maas-01:8080
+        ...
+        proxy:
+          # package manager fallback defaults 
+          # used if linux:system:repo:apt-mk:proxy has no protocol specific entries
+          pkg:
+            enabled: true
+            ftp:   ftp://proxy.host.local:2121
+            #http:  http://proxy.host.local:3142
+            #https: https://proxy.host.local:3143
+          ... 
+          # global system fallback system defaults
+          ftp:   ftp://proxy.host.local:2121
+          http:  http://proxy.host.local:3142
+          https: https://proxy.host.local:3143
+
+
+RC
+~~
+
 rc.local example
 
 .. code-block:: yaml
@@ -467,6 +530,7 @@ rc.local example
            # By default this script does nothing.
            exit 0
 
+
 Prompt
 ~~~~~~
 
@@ -708,18 +772,70 @@ OpenVswitch Bridges
             use_interfaces:
             - eth1
 
-Linux with proxy
+Configure global environment variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Linux /etc/environment:
+``/etc/environment`` is for static system wide variable assignment after boot. Variable expansion is frequently not supported.
 
 .. code-block:: yaml
 
     linux:
-      network:
+      system:
+        env:
+          BOB_VARIABLE: Alice
+          ...
+          BOB_PATH:
+            - /srv/alice/bin
+            - /srv/bob/bin
+          ...
+          ftp_proxy:   none
+          http_proxy:  http://global-http-proxy.host.local:8080
+          https_proxy: ${linux:system:proxy:https}
+          no_proxy:
+            - 192.168.0.80
+            - 192.168.1.80
+            - .domain.com
+            - .local
         ...
+        # NOTE: global defaults proxy configuration.
         proxy:
-          host: proxy.domain.com
-          port: 3128
+          ftp:   ftp://proxy.host.local:2121
+          http:  http://proxy.host.local:3142
+          https: https://proxy.host.local:3143
+          noproxy:
+            - .domain.com
+            - .local
+
+Configure profile.d scripts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Linux /etc/profile.d:
+The profile.d scripts are being sourced during .sh execution and support variable expansion in opposite to /etc/environment
+global settings in ``/etc/environment``.
+
+.. code-block:: yaml
+
+    linux:
+      system:
+        profile:
+          locales: |
+            export LANG=C
+            export LC_ALL=C
+          ...
+          vi_flavors.sh: |
+            export PAGER=view
+            export EDITOR=vim
+            alias vi=vim
+          shell_locales.sh: |
+            export LANG=en_US
+            export LC_ALL=en_US.UTF-8
+          shell_proxies.sh: |
+            export FTP_PROXY=ftp://127.0.3.3:2121
+            export NO_PROXY='.local'
 
 Linux with hosts
+~~~~~~~~~~~~~~~~
 
 Parameter purge_hosts will enforce whole /etc/hosts file, removing entries
 that are not defined in model except defaults for both IPv4 and IPv6 localhost
@@ -753,6 +869,7 @@ clean state however it's not enabled by default for safety.
 
 
 Setup resolv.conf, nameservers, domain and search domains
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. code-block:: yaml
 

linux/files/apt.conf.d_proxies

@@ -0,0 +1,9 @@
+{%- if ftp and ftp.lower() != 'none' %}
+Acquire::ftp::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ ftp }}";
+{%- endif %}
+{%- if http and http.lower() != 'none' %}
+Acquire::http::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ http }}";
+{%- endif %}
+{%- if https and https.lower() != 'none' %}
+Acquire::https::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ https }}";
+{%- endif -%}

linux/files/etc_environment

@@ -0,0 +1,32 @@
+
+{%- for name,value in variables.iteritems() if not name.lower().endswith('_proxy') %}
+
+{%- if value is sequence and value is not string %}
+{{ name }}="{{ value|join(':') }}"
+
+{%- else %}
+{{ name }}="{{ value }}"
+
+{%- endif %}
+{%- endfor %}
+
+{%- if ftp_proxy and ftp_proxy.lower() != 'none' %}
+ftp_proxy="{{ ftp_proxy }}";
+FTP_PROXY="{{ ftp_proxy }}";
+{%- endif %}
+
+{%- if http_proxy and http_proxy.lower() != 'none' %}
+http_proxy="{{ http_proxy }}";
+HTTP_PROXY="{{ http_proxy }}";
+{%- endif %}
+
+{%- if https_proxy and https_proxy.lower() != 'none' %}
+https_proxy="{{ https_proxy }}";
+HTTPS_PROXY="{{ https_proxy }}";
+{%- endif %}
+
+{%- if no_proxy %}
+no_proxy="{{ no_proxy|join(',') }}";
+NO_PROXY="{{ no_proxy|join(',') }}";
+{%- endif %}
+

linux/files/etc_profile

@@ -0,0 +1 @@
+{{ script }}

linux/files/etc_profile_vi_flavors.sh

@@ -0,0 +1,3 @@
+set -o vi
+export EDITOR=vim
+{{ script }}

linux/map.jinja

@@ -8,6 +8,9 @@
         'limit': {},
         'locale': {},
         'motd': {},
+        'env': {},
+        'profile': {},
+        'proxy': {},
         'repo': {},
         'package': {},
         'autoupdates': {
@@ -26,6 +29,9 @@
         'limit': {},
         'locale': {},
         'motd': {},
+        'env': {},
+        'profile': {},
+        'proxy': {},
         'repo': {},
         'package': {},
         'autoupdates': {
@@ -44,6 +50,9 @@
         'limit': {},
         'locale': {},
         'motd': {},
+        'env': {},
+        'profile': {},
+        'proxy': {},
         'repo': {},
         'package': {},
         'autoupdates': {
@@ -112,7 +121,7 @@
         'interface_params': interface_params,
         'bridge': 'none',
         'proxy': {
-           'host': 'none',
+           'host': 'none'
         },
         'host': {},
     },
@@ -125,7 +134,7 @@
         'interface_params': interface_params,
         'bridge': 'none',
         'proxy': {
-           'host': 'none',
+           'host': 'none'
         },
         'host': {},
     },

linux/system/env.sls

@@ -0,0 +1,36 @@
+{%- from "linux/map.jinja" import system with context %}
+{%- if system.enabled %}
+
+{%- if system.env|length > 0 %}
+
+linux_system_environment_proxies:
+  file.blockreplace:
+  - name: /etc/environment
+  - marker_start: '# START - SALT MANAGED VARIABLES, DO NOT EDIT'
+  - marker_end:   '# END - SALT MANAGED VARIABLES'
+  - template: jinja
+  - source: salt://linux/files/etc_environment
+  - append_if_not_found: True
+  - backup: '.bak'
+  - show_changes: True
+  - defaults:
+      variables: {{ system.env | yaml }}
+      no_proxy: {{ system.env.get('no_proxy', None) }}
+      https_proxy: {{ system.env.get('https_proxy', None) }}
+      http_proxy: {{ system.env.get('http_proxy', None) }}
+      ftp_proxy: {{ system.env.get('ftp_proxy', None) }}
+
+{%- else %}
+
+linux_system_environment_proxies:
+  file.blockreplace:
+  - name: /etc/environment
+  - marker_start: '# SALT MANAGED VARIABLES - DO NOT EDIT - START'
+  - content:      '# '
+  - marker_end:   '# SALT MANAGED VARIABLES - END'
+  - append_if_not_found: True
+  - backup: '.bak'
+  - show_changes: True
+
+{%- endif %}
+{%- endif %}

linux/system/init.sls

@@ -1,5 +1,7 @@
 {%- from "linux/map.jinja" import system with context %}
 include:
+- linux.system.env
+- linux.system.profile
 {%- if system.repo|length > 0 %}
 - linux.system.repo
 {%- endif %}

linux/system/profile.sls

@@ -0,0 +1,35 @@
+{%- from "linux/map.jinja" import system with context %}
+{%- if system.enabled %}
+
+/etc/profile.d:
+  file.directory:
+  - user: root
+  - mode: 750
+  - makedirs: true
+
+profile.d_clean:
+  file.directory:
+  - name: /etc/profile.d
+  - clean: true
+  - exclude_pat: 'E@^((?!salt_profile*).)*$'
+
+{%- if system.profile|length > 0 %}
+
+{%- for name, script in system.profile.iteritems() %}
+profile.d_script_{{ name  }}:
+    file.managed:
+    - name: /etc/profile.d/salt_profile_{{ name }}{%if name.split('.')|length == 1 %}.sh{% endif %}
+    - mode: 755
+    - source:
+      - salt://linux/files/etc_profile_{{ name }}
+      - salt://linux/files/etc_profile
+    - template: jinja
+    - defaults:
+          script: {{ script|yaml }}
+    - require_in:
+      - service: profile.d_clean
+{% endfor %}
+
+{%- endif %}
+{%- endif %}
+

linux/system/repo.sls

@@ -1,12 +1,51 @@
 {%- from "linux/map.jinja" import system with context %}
 {%- if system.enabled %}
 
+# global proxy setup
+{%- if system.proxy.get('pkg', {}).get('enabled', False) %}
+{%- if grains.os_family == 'Debian' %}
+
+/etc/apt/apt.conf.d/99proxies-salt:
+  file.managed:
+  - template: jinja
+  - source: salt://linux/files/apt.conf.d_proxies
+  - defaults:
+      external_host: False
+      https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }}
+      http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }}
+      ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }}
+
+{%- else %}
+
+/etc/apt/apt.conf.d/99proxies-salt:
+  file.absent
+
+{%- endif %}
+{%- endif %}
+
 {% set default_repos = {} %}
 
 {%- for name, repo in system.repo.iteritems() %}
 
 {%- if grains.os_family == 'Debian' %}
 
+# per repository proxy setup
+{%- if repo.get('proxy', {}).get('enabled', False) %}
+{%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %}
+/etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+  file.managed:
+  - template: jinja
+  - source: salt://linux/files/apt.conf.d_proxies
+  - defaults:
+      external_host: {{ external_host }}
+      https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }}
+      http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }}
+      ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }}
+{%- else %}
+/etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+  file.absent
+{%- endif %}
+
 {%- if repo.pin is defined %}
 
 linux_repo_{{ name }}_pin:
@@ -63,6 +102,12 @@ linux_repo_{{ name }}:
   - refresh_db: {{ repo.get('refresh_db', True) }}
   - require:
     - pkg: linux_packages
+  {%- if repo.get('proxy', {}).get('enabled', False) %}
+    - file: /etc/apt/apt.conf.d/99proxies-salt-{{ name }}
+  {%- endif %}
+  {%- if system.proxy.get('pkg', {}).get('enabled', False) %}
+    - file: /etc/apt/apt.conf.d/99proxies-salt
+  {%- endif %}
 
 {%- endif %}
 
@@ -70,6 +115,11 @@ linux_repo_{{ name }}:
 
 {%- if grains.os_family == "RedHat" %}
 
+{%- if repo.get('proxy', {}).get('enabled', False) %}
+# PLACEHOLDER
+# TODO, implement per proxy configuration for Yum
+{%- endif %}
+
 {%- if not repo.get('default', False) %}
 
 linux_repo_{{ name }}:

tests/integration/system/env_spec.rb

@@ -0,0 +1,12 @@
+
+## PROXIES
+#
+describe file('/etc/environment') do
+    it('should exist')
+    its('content') { should_not match /HTTPS_PROXY"/ }
+    its('content') { should match /HTTP_PROXY="http:\/\/127.0.4.2:80"/ }
+    its('content') { should match /BOB_PATH=/}
+    its('content') { should match /LC_ALL="C"/ }
+    its('content') { should match /ftp_proxy=.*127.0.4.3:2121/ }
+    its('content') { should match /NO_PROXY=.*dummy.net,.local/ }
+end

tests/integration/system/profile_spec.rb

@@ -0,0 +1,17 @@
+
+describe file('/etc/profile.d/salt_profile_vi_flavors.sh') do
+    it('should exist')
+    its('content') { should match /EDITOR=vim/ }
+    its('content') { should match /PAGER=view/ }
+    its('content') { should match /alias vi=vim/ }
+end
+
+describe file('/etc/profile.d/salt_profile_locales.sh') do
+    it('should exist')
+    its('content') { should match /LANG=en_US/ }
+end
+
+describe file('/etc/profile.d/prompt.sh') do
+    it('should exist')
+end
+

tests/integration/system/repo_spec.rb

@@ -0,0 +1,17 @@
+
+# PROXIES
+#
+# globally
+describe file('/etc/apt/apt.conf.d/99proxies-salt') do
+    it('should exist')
+    its('content') { should_not match /ftp/ }
+    its('content') { should match /proxy "https.*127.0.2.1:4443"/ }
+end
+
+# per repo
+describe file('/etc/apt/apt.conf.d/99proxies-salt-opencontrail') do
+    it('should exist')
+    its('content') { should_not match /ftp/ }
+    its('content') { should match /Acquire::https::proxy::ppa.launchpad.net/ }
+end
+

tests/pillar/system.sls

@@ -90,7 +90,29 @@ linux:
         version: latest
     repo:
       opencontrail:
-        source: "deb http://ppa.launchpad.net/tcpcloud/contrail-2.20/ubuntu trusty main"
+        source: "deb http://ppa.launchpad.net/tcpcloud/contrail-3.0/ubuntu xenial main"
+        keyid: E79EE90C
+        keyserver: keyserver.ubuntu.com
+        architectures: amd64
+        proxy:
+          enabled: true
+          https: https://127.0.5.1:443
+          #http: http://127.0.5.2:8080
+      apt-mk-salt:
+        source: "deb http://apt-mk.mirantis.com/xenial stable salt"
+        key_url: http://apt-mk.mirantis.com/public.gpg
+        architectures: amd64
+        proxy:
+          enabled: true
+      apt-mk-salt-nightly:
+        source: "deb http://apt-mk.mirantis.com/xenial nightly salt"
+        key_url: http://apt-mk.mirantis.com/public.gpg
+        architectures: amd64
+        proxy:
+          enabled: false
+      apt-mk-extra-nightly:
+        source: "deb http://apt-mk.mirantis.com/xenial nightly extra"
+        key_url: http://apt-mk.mirantis.com/public.gpg
         architectures: amd64
     locale:
       en_US.UTF-8:
@@ -200,3 +222,46 @@ linux:
         sudogroup3:
           commands:
             - ALL
+    env:
+      BOB_VARIABLE: Alice
+      BOB_PATH:
+        - /srv/alice/bin
+        - /srv/bob/bin
+      HTTPS_PROXY: https://127.0.4.1:443
+      http_proxy: http://127.0.4.2:80
+      ftp_proxy: ftp://127.0.4.3:2121
+      no_proxy:
+        - 192.168.0.1
+        - 192.168.0.2
+        - .saltstack.com
+        - .ubuntu.com
+        - .mirantis.com
+        - .launchpad.net
+        - .dummy.net
+        - .local
+      LANG: C
+      LC_ALL: C
+    profile:
+      vi_flavors.sh: |
+        export PAGER=view
+        alias vi=vim
+      locales: |
+        export LANG=en_US
+        export LC_ALL=en_US.UTF-8
+
+    # pillar for proxy configuration
+    proxy:
+      # for package managers
+      pkg:
+        enabled: true
+        https: https://127.0.2.1:4443
+        #http: http://127.0.2.2
+        ftp: none
+      # fallback, system defaults
+      https: https://127.0.1.1:443
+      #http: http://127.0.1.2
+      ftp: ftp://127.0.1.3
+      noproxy:
+        - host1
+        - host2
+        - .local