Browse Source

Merge pull request #221 from bradthurber/new-gitfs-functionality

add new gitfs functionality
master
Forrest 8 years ago
parent
commit
8ed58c6528
6 changed files with 355 additions and 7 deletions
  1. +5
    -0
      README.rst
  2. +10
    -0
      pillar.example
  3. +1
    -0
      salt/files/gitfs_key.jinja
  4. +90
    -4
      salt/files/master.d/f_defaults.conf
  5. +229
    -3
      salt/files/minion.d/f_defaults.conf
  6. +20
    -0
      salt/gitfs/keys.sls

+ 5
- 0
README.rst View File

@@ -63,6 +63,11 @@ Install gitfs backend dulwich dependencies. Set ``salt:master:gitfs_provider: du

Install gitfs backend GitPython dependenciess. Set ``salt:master:gitfs_provider: gitpython`` in your pillar.

``salt.gitfs.keys``
----------------------

Install ssh keys to be used by gitfs

``salt.gitfs.pygit2``
----------------------


+ 10
- 0
pillar.example View File

@@ -115,6 +115,16 @@ salt:
user: ubuntu
sudo: True
priv: /etc/salt/ssh_keys/sshkey.pem
gitfs:
keys:
global:
# key and pub end up being the extension used on the key file. values other than key and pub are possible
key: |
-----BEGIN RSA PRIVATE KEY-----
...........
-----END RSA PRIVATE KEY-----
pub: |
...........

salt_cloud_certs:
aws:

+ 1
- 0
salt/files/gitfs_key.jinja View File

@@ -0,0 +1 @@
{{ pillar['salt']['gitfs']['keys'][key][type] }}

+ 90
- 4
salt/files/master.d/f_defaults.conf View File

@@ -699,9 +699,36 @@ fileserver_backend:

# Git File Server Backend Configuration
#
# Gitfs can be provided by one of two python modules: GitPython or pygit2. If
# using pygit2, both libgit2 and git must also be installed.
{{ get_config('gitfs_provider', 'gitpython') }}
# Optional parameter used to specify the provider to be used for gitfs. Must
# be one of the following: pygit2, gitpython, or dulwich. If unset, then each
# will be tried in that same order, and the first one with a compatible
# version installed will be the provider that is used.
{{ get_config('gitfs_provider', 'pygit2') }}

# Along with gitfs_password, is used to authenticate to HTTPS remotes.
{{ get_config('gitfs_user', 'git') }}

# Along with gitfs_user, is used to authenticate to HTTPS remotes.
# This parameter is not required if the repository does not use authentication.
{{ get_config('gitfs_password', '') }}

# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
# This parameter enables authentication over HTTP. Enable this at your own risk.
{{ get_config('gitfs_insecure_auth', 'False') }}

# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to
# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
# is required for SSH remotes.
{{ get_config('gitfs_pubkey', '') }}

# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to
# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
# is required for SSH remotes.
{{ get_config('gitfs_privkey', '') }}

# This parameter is optional, required only when the SSH key being used to
# authenticate is protected by a passphrase.
{{ get_config('gitfs_passphrase', '') }}

# When using the git fileserver backend at least one git remote needs to be
# defined. The user running the salt master will need read access to the repo.
@@ -865,8 +892,67 @@ ext_pillar:
{{ get_config('pillar_source_merging_strategy', 'smart') }}

# Recursively merge lists by aggregating them instead of replacing them.
{{ get_config('pillar_merge_lists', 'False') }}
{{ get_config('pillar_merge_lists', False) }}

# Git External Pillar (git_pillar) Configuration Options
#
# Specify the provider to be used for git_pillar. Must be either pygit2 or
# gitpython. If unset, then both will be tried in that same order, and the
# first one with a compatible version installed will be the provider that
# is used.
{{ get_config('git_pillar_provider', 'pygit2') }}

# If the desired branch matches this value, and the environment is omitted
# from the git_pillar configuration, then the environment for that git_pillar
# remote will be base.
{{ get_config('git_pillar_base', 'master') }}

# If the branch is omitted from a git_pillar remote, then this branch will
# be used instead.
{{ get_config('git_pillar_branch', 'master') }}

# Environment to use for git_pillar remotes. This is normally derived from
# the branch/tag (or from a per-remote env parameter), but if set this will
# override the process of deriving the env from the branch/tag name.
{{ get_config('git_pillar_env', '') }}

# Path relative to the root of the repository where the git_pillar top file
# and SLS files are located.
{{ get_config('git_pillar_root', 'pillar') }}

# Specifies whether or not to ignore SSL certificate errors when contacting
# the remote repository.
{{ get_config('git_pillar_ssl_verify', True) }}

# When set to False, if there is an update/checkout lock for a git_pillar
# remote and the pid written to it is not running on the master, the lock
# file will be automatically cleared and a new lock will be obtained.
{{ get_config('git_pillar_global_lock', False) }}

# Git External Pillar Authentication Options
#
# Along with git_pillar_password, is used to authenticate to HTTPS remotes.
{{ get_config('git_pillar_user', '') }}

# Along with git_pillar_user, is used to authenticate to HTTPS remotes.
# This parameter is not required if the repository does not use authentication.
{{ get_config('git_pillar_password', '') }}

# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
# This parameter enables authentication over HTTP.
{{ get_config('git_pillar_insecure_auth', False) }}

# Along with git_pillar_privkey (and optionally git_pillar_passphrase),
# is used to authenticate to SSH remotes.
{{ get_config('git_pillar_pubkey', '') }}

# Along with git_pillar_pubkey (and optionally git_pillar_passphrase),
# is used to authenticate to SSH remotes.
{{ get_config('git_pillar_privkey', '') }}

# This parameter is optional, required only when the SSH key being used
# to authenticate is protected by a passphrase.
{{ get_config('git_pillar_passphrase', '') }}

##### Syndic settings #####
##########################################

+ 229
- 3
salt/files/minion.d/f_defaults.conf View File

@@ -488,6 +488,33 @@ file_client: local
{{ file_roots(cfg_salt['file_roots']) }}
{%- elif formulas|length -%}
{{ file_roots({'base': ['/srv/salt']}) }}
{%- else -%}
#file_roots:
# base:
# - /srv/salt
{%- endif %}


# File Server Backend
#
# Salt supports a modular fileserver backend system, this system allows
# the salt minion to link directly to third party systems to gather and
# manage the files available to minions. Multiple backends can be
# configured and will be searched for the requested file in the order in which
# they are defined here. The default setting only enables the standard backend
# "roots" which uses the "file_roots" option.
#fileserver_backend:
# - roots
#
# To use multiple backends list them in the order they are searched:
#fileserver_backend:
# - git
# - roots
{% if 'fileserver_backend' in cfg_minion -%}
fileserver_backend:
{%- for backend in cfg_minion['fileserver_backend'] %}
- {{ backend }}
{%- endfor -%}
{%- endif %}

# By default, the Salt fileserver recurses fully into all defined environments
@@ -508,7 +535,40 @@ file_client: local

# gitfs provider
{{ get_config('gitfs_provider', 'pygit2') }}
# gitfs remotes

# Along with gitfs_password, is used to authenticate to HTTPS remotes.
{{ get_config('gitfs_user', 'git') }}

# Along with gitfs_user, is used to authenticate to HTTPS remotes.
# This parameter is not required if the repository does not use authentication.
{{ get_config('gitfs_password', '') }}

# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
# This parameter enables authentication over HTTP. Enable this at your own risk.
{{ get_config('gitfs_insecure_auth', 'False') }}

# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to
# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
# is required for SSH remotes.
{{ get_config('gitfs_pubkey', '') }}

# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to
# authenticate to SSH remotes. This parameter (or its per-remote counterpart)
# is required for SSH remotes.
{{ get_config('gitfs_privkey', '') }}

# This parameter is optional, required only when the SSH key being used to
# authenticate is protected by a passphrase.
{{ get_config('gitfs_passphrase', '') }}
# When using the git fileserver backend at least one git remote needs to be
# defined. The user running the salt master will need read access to the repo.
#
# The repos will be searched in order to find the file requested by a client
# and the first repo to have the file will return it.
# When using the git backend branches and tags are translated into salt
# environments.
# Note: file:// repos will be treated as a remote, so refs you want used must
# exist in that repo as *local* refs.
{% if 'gitfs_remotes' in cfg_minion -%}
gitfs_remotes:
{%- for remote in cfg_minion['gitfs_remotes'] %}
@@ -526,11 +586,45 @@ gitfs_remotes:
{%- endif -%}
{%- endfor -%}
{%- endif %}
# verify git ssl errors

# The gitfs_ssl_verify option specifies whether to ignore ssl certificate
# errors when contacting the gitfs backend. You might want to set this to
# false if you're using a git backend that uses a self-signed certificate but
# keep in mind that setting this flag to anything other than the default of True
# is a security concern, you may want to try using the ssh transport.
{{ get_config('gitfs_ssl_verify', 'True') }}
# gitfs root dir

# The gitfs_root option gives the ability to serve files from a subdirectory
# within the repository. The path is defined relative to the root of the
# repository and defaults to the repository root.
{{ get_config('gitfs_root', 'somefolder/otherfolder') }}

# The gitfs_env_whitelist and gitfs_env_blacklist parameters allow for greater
# control over which branches/tags are exposed as fileserver environments.
{% if 'gitfs_env_whitelist' in cfg_minion -%}
gitfs_env_whitelist:
{%- for git_env in cfg_minion['gitfs_env_whitelist'] %}
- {{ git_env }}
{%- endfor -%}
{% else -%}
# gitfs_env_whitelist:
# - base
# - v1.*
{% endif %}

{% if 'gitfs_env_blacklist' in cfg_minion -%}
gitfs_env_blacklist:
{%- for git_env in cfg_minion['gitfs_env_blacklist'] %}
- {{ git_env }}
{%- endfor -%}
{% else -%}
# gitfs_env_blacklist:
# - bug/*
# - feature/*
{% endif %}

##### Pillar settings #####
##########################################
# The Salt pillar is searched for locally if file_client is set to local. If
# this is the case, and pillar data is defined, then the pillar_roots need to
# also be configured on the minion:
@@ -550,8 +644,140 @@ pillar_roots:
- {{ dir }}
{%- endfor -%}
{%- endfor -%}
{%- else -%}
#pillar_roots:
# base:
# - /srv/pillar
{%- endif %}

{% if 'ext_pillar' in cfg_minion %}
ext_pillar:
{%- for pillar in cfg_minion['ext_pillar'] -%}
{%- for key in pillar -%}
{%- if pillar[key] is string %}
- {{ key }}: {{ pillar[key] }}
{%- elif pillar[key] is iterable and pillar[key] is not mapping %}
- {{ key }}:
{%- for parameter in pillar[key] %}
- {{ parameter }}
{%- endfor -%}
{%- elif pillar[key] is mapping and pillar[key] is not string %}
- {{ key }}:
{%- for parameter in pillar[key] %}
{{ parameter }}: {{pillar[key][parameter]}}
{%- endfor %}
{%- else %}
# Error in rendering {{ key }}, please read https://docs.saltstack.com/en/latest/topics/development/external_pillars.html#configuration
{% endif %}
{%- endfor -%}
{%- endfor %}
{% elif 'ext_pillar' in cfg_salt %}
ext_pillar:
{% for pillar in cfg_salt['ext_pillar'] %}
- {{ pillar.items()[0][0] }}: {{ pillar.items()[0][1] }}
{% endfor %}
{% else %}
#ext_pillar:
# - hiera: /etc/hiera.yaml
# - cmd_yaml: cat /etc/salt/yaml
{% endif %}

# The ext_pillar_first option allows for external pillar sources to populate
# before file system pillar. This allows for targeting file system pillar from
# ext_pillar.
{{ get_config('ext_pillar_first', 'False') }}

# The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate
# errors when contacting the pillar gitfs backend. You might want to set this to
# false if you're using a git backend that uses a self-signed certificate but
# keep in mind that setting this flag to anything other than the default of True
# is a security concern, you may want to try using the ssh transport.
{{ get_config('pillar_gitfs_ssl_verify', 'True') }}

# The pillar_opts option adds the master configuration file data to a dict in
# the pillar called "master". This is used to set simple configurations in the
# master config file that can then be used on minions.
{{ get_config('pillar_opts', 'True') }}

# The pillar_safe_render_error option prevents the master from passing pillar
# render errors to the minion. This is set on by default because the error could
# contain templating data which would give that minion information it shouldn't
# have, like a password! When set true the error message will only show:
# Rendering SLS 'my.sls' failed. Please see master log for details.
{{ get_config('pillar_safe_render_error', 'True') }}

# The pillar_source_merging_strategy option allows you to configure merging strategy
# between different sources. It accepts four values: recurse, aggregate, overwrite,
# or smart. Recurse will merge recursively mapping of data. Aggregate instructs
# aggregation of elements between sources that use the #!yamlex renderer. Overwrite
# will verwrite elements according the order in which they are processed. This is
# behavior of the 2014.1 branch and earlier. Smart guesses the best strategy based
# on the "renderer" setting and is the default value.
{{ get_config('pillar_source_merging_strategy', 'smart') }}

# Recursively merge lists by aggregating them instead of replacing them.
{{ get_config('pillar_merge_lists', False) }}

# Git External Pillar (git_pillar) Configuration Options
#
# Specify the provider to be used for git_pillar. Must be either pygit2 or
# gitpython. If unset, then both will be tried in that same order, and the
# first one with a compatible version installed will be the provider that
# is used.
{{ get_config('git_pillar_provider', 'pygit2') }}

# If the desired branch matches this value, and the environment is omitted
# from the git_pillar configuration, then the environment for that git_pillar
# remote will be base.
{{ get_config('git_pillar_base', 'master') }}

# If the branch is omitted from a git_pillar remote, then this branch will
# be used instead.
{{ get_config('git_pillar_branch', 'master') }}

# Environment to use for git_pillar remotes. This is normally derived from
# the branch/tag (or from a per-remote env parameter), but if set this will
# override the process of deriving the env from the branch/tag name.
{{ get_config('git_pillar_env', '') }}

# Path relative to the root of the repository where the git_pillar top file
# and SLS files are located.
{{ get_config('git_pillar_root', 'pillar') }}

# Specifies whether or not to ignore SSL certificate errors when contacting
# the remote repository.
{{ get_config('git_pillar_ssl_verify', True) }}

# When set to False, if there is an update/checkout lock for a git_pillar
# remote and the pid written to it is not running on the master, the lock
# file will be automatically cleared and a new lock will be obtained.
{{ get_config('git_pillar_global_lock', False) }}

# Git External Pillar Authentication Options
#
# Along with git_pillar_password, is used to authenticate to HTTPS remotes.
{{ get_config('git_pillar_user', '') }}

# Along with git_pillar_user, is used to authenticate to HTTPS remotes.
# This parameter is not required if the repository does not use authentication.
{{ get_config('git_pillar_password', '') }}

# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
# This parameter enables authentication over HTTP.
{{ get_config('git_pillar_insecure_auth', False) }}

# Along with git_pillar_privkey (and optionally git_pillar_passphrase),
# is used to authenticate to SSH remotes.
{{ get_config('git_pillar_pubkey', '') }}

# Along with git_pillar_pubkey (and optionally git_pillar_passphrase),
# is used to authenticate to SSH remotes.
{{ get_config('git_pillar_privkey', '') }}

# This parameter is optional, required only when the SSH key being used
# to authenticate is protected by a passphrase.
{{ get_config('git_pillar_passphrase', '') }}


###### Security settings #####
###########################################

+ 20
- 0
salt/gitfs/keys.sls View File

@@ -0,0 +1,20 @@
{%- from "salt/map.jinja" import salt_settings with context %}

{%- set gitfs_keys=salt['pillar.get']('salt:gitfs:keys') %}

{%- for key, keyvalues in gitfs_keys.items() %}
{%- for type, keydata in keyvalues.items() %}
gitfs-key-{{ key }}-{{ type }}:
file.managed:
- name: {{ salt_settings.config_path }}/pki/gitfs/{{ key }}.{{ type }}
- source: salt://salt/files/gitfs_key.jinja
- template: jinja
- user: root
- group: root
- mode: 600
- makedirs: True
- defaults:
key: {{ key }}
type: {{ type }}
{%- endfor %}
{%- endfor %}

Loading…
Cancel
Save