# -*- coding: utf-8 -*-
'''
The ``file_tree`` external pillar allows values from all files in a directory
tree to be imported as Pillar data.

.. note::

    This is an external pillar and is subject to the :ref:`rules and
    constraints <external-pillars>` governing external pillars.

.. versionadded:: 2015.5.0

In this pillar, data is organized by either Minion ID or Nodegroup name.  To
setup pillar data for a specific Minion, place it in
``<root_dir>/hosts/<minion_id>``.  To setup pillar data for an entire
Nodegroup, place it in ``<root_dir>/nodegroups/<node_group>`` where
``<node_group>`` is the Nodegroup's name.

Example ``file_tree`` Pillar
============================

Master Configuration
--------------------

.. code-block:: yaml

    ext_pillar:
      - file_tree:
          root_dir: /srv/ext_pillar
          follow_dir_links: False
          keep_newline: True

The ``root_dir`` parameter is required and points to the directory where files
for each host are stored. The ``follow_dir_links`` parameter is optional and
defaults to False. If ``follow_dir_links`` is set to True, this external pillar
will follow symbolic links to other directories.

.. warning::
    Be careful when using ``follow_dir_links``, as a recursive symlink chain
    will result in unexpected results.

.. versionchanged:: 2018.3.0
    If ``root_dir`` is a relative path, it will be treated as relative to the
    :conf_master:`pillar_roots` of the environment specified by
    :conf_minion:`pillarenv`. If an environment specifies multiple
    roots, this module will search for files relative to all of them, in order,
    merging the results.

If ``keep_newline`` is set to ``True``, then the pillar values for files ending
in newlines will keep that newline. The default behavior is to remove the
end-of-file newline. ``keep_newline`` should be turned on if the pillar data is
intended to be used to deploy a file using ``contents_pillar`` with a
:py:func:`file.managed <salt.states.file.managed>` state.

.. versionchanged:: 2015.8.4
    The ``raw_data`` parameter has been renamed to ``keep_newline``. In earlier
    releases, ``raw_data`` must be used. Also, this parameter can now be a list
    of globs, allowing for more granular control over which pillar values keep
    their end-of-file newline. The globs match paths relative to the
    directories named for minion IDs and nodegroups underneath the ``root_dir``
    (see the layout examples in the below sections).

    .. code-block:: yaml

        ext_pillar:
          - file_tree:
              root_dir: /path/to/root/directory
              keep_newline:
                - files/testdir/*

.. note::
    In earlier releases, this documentation incorrectly stated that binary
    files would not affected by the ``keep_newline`` configuration.  However,
    this module does not actually distinguish between binary and text files.

.. versionchanged:: 2017.7.0
    Templating/rendering has been added. You can now specify a default render
    pipeline and a black- and whitelist of (dis)allowed renderers.

    ``template`` must be set to ``True`` for templating to happen.

    .. code-block:: yaml

        ext_pillar:
          - file_tree:
            root_dir: /path/to/root/directory
            render_default: jinja|yaml
            renderer_blacklist:
              - gpg
            renderer_whitelist:
              - jinja
              - yaml
            template: True

Assigning Pillar Data to Individual Hosts
-----------------------------------------

To configure pillar data for each host, this external pillar will recursively
iterate over ``root_dir``/hosts/``id`` (where ``id`` is a minion ID), and
compile pillar data with each subdirectory as a dictionary key and each file
as a value.

For example, the following ``root_dir`` tree:

.. code-block:: text

    ./hosts/
    ./hosts/test-host/
    ./hosts/test-host/files/
    ./hosts/test-host/files/testdir/
    ./hosts/test-host/files/testdir/file1.txt
    ./hosts/test-host/files/testdir/file2.txt
    ./hosts/test-host/files/another-testdir/
    ./hosts/test-host/files/another-testdir/symlink-to-file1.txt

will result in the following pillar tree for minion with ID ``test-host``:

.. code-block:: text

    test-host:
        ----------
        apache:
            ----------
            config.d:
                ----------
                00_important.conf:
                    <important_config important_setting="yes" />
                20_bob_extra.conf:
                    <bob_specific_cfg has_freeze_ray="yes" />
        corporate_app:
            ----------
            settings:
                ----------
                common_settings:
                    // This is the main settings file for the corporate
                    // internal web app
                    main_setting: probably
                bob_settings:
                    role: bob

.. note::

    The leaf data in the example shown is the contents of the pillar files.
'''
from __future__ import absolute_import, print_function, unicode_literals

# Import python libs
import fnmatch
import logging
import os

# Import salt libs
import salt.loader
import salt.utils.dictupdate
import salt.utils.files
import salt.utils.minions
import salt.utils.path
import salt.utils.stringio
import salt.utils.stringutils
import salt.template
from salt.ext import six

# Set up logging
log = logging.getLogger(__name__)


def _on_walk_error(err):
    '''
    Log salt.utils.path.os_walk() error.
    '''
    log.error('%s: %s', err.filename, err.strerror)


def _check_newline(prefix, file_name, keep_newline):
    '''
    Return a boolean stating whether or not a file's trailing newline should be
    removed. To figure this out, first check if keep_newline is a boolean and
    if so, return its opposite. Otherwise, iterate over keep_newline and check
    if any of the patterns match the file path. If a match is found, return
    False, otherwise return True.
    '''
    if isinstance(keep_newline, bool):
        return not keep_newline
    full_path = os.path.join(prefix, file_name)
    for pattern in keep_newline:
        try:
            if fnmatch.fnmatch(full_path, pattern):
                return False
        except TypeError:
            if fnmatch.fnmatch(full_path, six.text_type(pattern)):
                return False
    return True


def _construct_pillar(top_dir,
                      follow_dir_links,
                      keep_newline=False,
                      render_default=None,
                      renderer_blacklist=None,
                      renderer_whitelist=None,
                      template=False):
    '''
    Construct pillar from file tree.
    '''
    pillar = {}
    renderers = salt.loader.render(__opts__, __salt__)

    norm_top_dir = os.path.normpath(top_dir)
    for dir_path, dir_names, file_names in salt.utils.path.os_walk(
            top_dir, topdown=True, onerror=_on_walk_error,
            followlinks=follow_dir_links):
        # Find current path in pillar tree
        pillar_node = pillar
        norm_dir_path = os.path.normpath(dir_path)
        prefix = os.path.relpath(norm_dir_path, norm_top_dir)
        if norm_dir_path != norm_top_dir:
            path_parts = []
            head = prefix
            while head:
                head, tail = os.path.split(head)
                path_parts.insert(0, tail)
            while path_parts:
                pillar_node = pillar_node[path_parts.pop(0)]

        # Create dicts for subdirectories
        for dir_name in dir_names:
            pillar_node[dir_name] = {}

        # Add files
        for file_name in file_names:
            file_path = os.path.join(dir_path, file_name)
            if not os.path.isfile(file_path):
                log.error('file_tree: %s: not a regular file', file_path)
                continue

            contents = b''
            try:
                with salt.utils.files.fopen(file_path, 'rb') as fhr:
                    buf = fhr.read(__opts__['file_buffer_size'])
                    while buf:
                        contents += buf
                        buf = fhr.read(__opts__['file_buffer_size'])
                    if contents.endswith(b'\n') \
                            and _check_newline(prefix,
                                               file_name,
                                               keep_newline):
                        contents = contents[:-1]
            except (IOError, OSError) as exc:
                log.error('file_tree: Error reading %s: %s',
                          file_path,
                          exc.strerror)
            else:
                data = contents
                if template is True:
                    data = salt.template.compile_template_str(template=salt.utils.stringutils.to_unicode(contents),
                                                              renderers=renderers,
                                                              default=render_default,
                                                              blacklist=renderer_blacklist,
                                                              whitelist=renderer_whitelist)
                if salt.utils.stringio.is_readable(data):
                    pillar_node[file_name] = data.getvalue()
                else:
                    pillar_node[file_name] = data

    return pillar


def ext_pillar(minion_id,
               pillar,
               root_dir=None,
               follow_dir_links=False,
               debug=False,
               keep_newline=False,
               render_default=None,
               renderer_blacklist=None,
               renderer_whitelist=None,
               template=False):
    '''
    Compile pillar data from the given ``root_dir`` specific to Nodegroup names
    and Minion IDs.

    If a Minion's ID is not found at ``<root_dir>/host/<minion_id>`` or if it
    is not included in any Nodegroups named at
    ``<root_dir>/nodegroups/<node_group>``, no pillar data provided by this
    pillar module will be available for that Minion.

    .. versionchanged:: 2017.7.0
        Templating/rendering has been added. You can now specify a default
        render pipeline and a black- and whitelist of (dis)allowed renderers.

        ``template`` must be set to ``True`` for templating to happen.

        .. code-block:: yaml

            ext_pillar:
              - file_tree:
                root_dir: /path/to/root/directory
                render_default: jinja|yaml
                renderer_blacklist:
                  - gpg
                renderer_whitelist:
                  - jinja
                  - yaml
                template: True

    :param minion_id:
        The ID of the Minion whose pillar data is to be collected

    :param pillar:
        Unused by the ``file_tree`` pillar module

    :param root_dir:
        Filesystem directory used as the root for pillar data (e.g.
        ``/srv/ext_pillar``)

        .. versionchanged:: 2018.3.0
            If ``root_dir`` is a relative path, it will be treated as relative to the
            :conf_master:`pillar_roots` of the environment specified by
            :conf_minion:`pillarenv`. If an environment specifies multiple
            roots, this module will search for files relative to all of them, in order,
            merging the results.

    :param follow_dir_links:
        Follow symbolic links to directories while collecting pillar files.
        Defaults to ``False``.

        .. warning::

            Care should be exercised when enabling this option as it will
            follow links that point outside of ``root_dir``.

        .. warning::

            Symbolic links that lead to infinite recursion are not filtered.

    :param debug:
        Enable debug information at log level ``debug``.  Defaults to
        ``False``.  This option may be useful to help debug errors when setting
        up the ``file_tree`` pillar module.

    :param keep_newline:
        Preserve the end-of-file newline in files.  Defaults to ``False``.
        This option may either be a boolean or a list of file globs (as defined
        by the `Python fnmatch package
        <https://docs.python.org/library/fnmatch.html>`_) for which end-of-file
        newlines are to be kept.

        ``keep_newline`` should be turned on if the pillar data is intended to
        be used to deploy a file using ``contents_pillar`` with a
        :py:func:`file.managed <salt.states.file.managed>` state.

        .. versionchanged:: 2015.8.4
            The ``raw_data`` parameter has been renamed to ``keep_newline``. In
            earlier releases, ``raw_data`` must be used. Also, this parameter
            can now be a list of globs, allowing for more granular control over
            which pillar values keep their end-of-file newline. The globs match
            paths relative to the directories named for Minion IDs and
            Nodegroup namess underneath the ``root_dir``.

            .. code-block:: yaml

                ext_pillar:
                  - file_tree:
                      root_dir: /srv/ext_pillar
                      keep_newline:
                        - apache/config.d/*
                        - corporate_app/settings/*

        .. note::
            In earlier releases, this documentation incorrectly stated that
            binary files would not affected by the ``keep_newline``.  However,
            this module does not actually distinguish between binary and text
            files.


    :param render_default:
        Override Salt's :conf_master:`default global renderer <renderer>` for
        the ``file_tree`` pillar.

        .. code-block:: yaml

            render_default: jinja

    :param renderer_blacklist:
        Disallow renderers for pillar files.

        .. code-block:: yaml

            renderer_blacklist:
              - json

    :param renderer_whitelist:
        Allow renderers for pillar files.

        .. code-block:: yaml

            renderer_whitelist:
              - yaml
              - jinja

    :param template:
        Enable templating of pillar files.  Defaults to ``False``.
    '''
    # Not used
    del pillar

    if not root_dir:
        log.error('file_tree: no root_dir specified')
        return {}

    if not os.path.isabs(root_dir):
        pillarenv = __opts__['pillarenv']
        if pillarenv is None:
            log.error('file_tree: root_dir is relative but pillarenv is not set')
            return {}
        log.debug('file_tree: pillarenv = %s', pillarenv)

        env_roots = __opts__['pillar_roots'].get(pillarenv, None)
        if env_roots is None:
            log.error('file_tree: root_dir is relative but no pillar_roots are specified '
                      ' for pillarenv %s', pillarenv)
            return {}

        env_dirs = []
        for env_root in env_roots:
            env_dir = os.path.normpath(os.path.join(env_root, root_dir))
            # don't redundantly load consecutively, but preserve any expected precedence
            if env_dir not in env_dirs or env_dir != env_dirs[-1]:
                env_dirs.append(env_dir)
        dirs = env_dirs
    else:
        dirs = [root_dir]

    result_pillar = {}
    for root in dirs:
        dir_pillar = _ext_pillar(minion_id,
                                 root,
                                 follow_dir_links,
                                 debug,
                                 keep_newline,
                                 render_default,
                                 renderer_blacklist,
                                 renderer_whitelist,
                                 template)
        result_pillar = salt.utils.dictupdate.merge(result_pillar,
                                                    dir_pillar,
                                                    strategy='recurse')
    return result_pillar


def _ext_pillar(minion_id,
                root_dir,
                follow_dir_links,
                debug,
                keep_newline,
                render_default,
                renderer_blacklist,
                renderer_whitelist,
                template):
    '''
    Compile pillar data for a single root_dir for the specified minion ID
    '''
    log.debug('file_tree: reading %s', root_dir)

    if not os.path.isdir(root_dir):
        log.error(
            'file_tree: root_dir %s does not exist or is not a directory',
            root_dir
        )
        return {}

    if not isinstance(keep_newline, (bool, list)):
        log.error(
            'file_tree: keep_newline must be either True/False or a list '
            'of file globs. Skipping this ext_pillar for root_dir %s',
            root_dir
        )
        return {}

    ngroup_pillar = {}
    nodegroups_dir = os.path.join(root_dir, 'nodegroups')
    if os.path.exists(nodegroups_dir) and len(__opts__.get('nodegroups', ())) > 0:
        master_ngroups = __opts__['nodegroups']
        ext_pillar_dirs = os.listdir(nodegroups_dir)
        if len(ext_pillar_dirs) > 0:
            for nodegroup in ext_pillar_dirs:
                if (os.path.isdir(nodegroups_dir) and
                        nodegroup in master_ngroups):
                    ckminions = salt.utils.minions.CkMinions(__opts__)
                    _res = ckminions.check_minions(
                        master_ngroups[nodegroup],
                        'compound')
                    match = _res['minions']
                    if minion_id in match:
                        ngroup_dir = os.path.join(
                            nodegroups_dir, six.text_type(nodegroup))
                        ngroup_pillar = salt.utils.dictupdate.merge(ngroup_pillar,
                            _construct_pillar(ngroup_dir,
                                              follow_dir_links,
                                              keep_newline,
                                              render_default,
                                              renderer_blacklist,
                                              renderer_whitelist,
                                              template),
                            strategy='recurse'
                        )
        else:
            if debug is True:
                log.debug(
                    'file_tree: no nodegroups found in file tree directory %s, skipping...',
                    ext_pillar_dirs
                )
    else:
        if debug is True:
            log.debug('file_tree: no nodegroups found in master configuration')

    host_dir = os.path.join(root_dir, 'hosts', minion_id)
    if not os.path.exists(host_dir):
        if debug is True:
            log.debug(
                'file_tree: no pillar data for minion %s found in file tree directory %s',
                minion_id,
                host_dir
            )
        return ngroup_pillar

    if not os.path.isdir(host_dir):
        log.error('file_tree: %s exists, but is not a directory', host_dir)
        return ngroup_pillar

    host_pillar = _construct_pillar(host_dir,
                                    follow_dir_links,
                                    keep_newline,
                                    render_default,
                                    renderer_blacklist,
                                    renderer_whitelist,
                                    template)
    return salt.utils.dictupdate.merge(ngroup_pillar,
                                       host_pillar,
                                       strategy='recurse')
