diff --git a/.gitignore b/.gitignore index 273b8432f..1901e87cf 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,8 @@ stackrc id_rsa* hosts *-hosts -ksgen_settings.yml instack_hosts doc/_build/ fence_xvm.key vm-host-table -tools/kcli/etc/kcli.cfg +tools/cli/etc/infrared.cfg diff --git a/.travis.yml b/.travis.yml index 996c2eb20..e8e1e8fca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ python: install: - pip install pep8 --use-mirrors - pip install https://github.com/dcramer/pyflakes/tarball/master - - pip install tools/kcli + - pip install tools/cli # command to run tests script: - - pep8 tools/kcli - - py.test tools/kcli + - pep8 tools/cli + - py.test tools/cli -v diff --git a/tools/__init__.py b/tools/__init__.py deleted file mode 100644 index 09668b120..000000000 --- a/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'yfried' diff --git a/tools/cli/README.rst b/tools/cli/README.rst new file mode 100644 index 000000000..503d9f057 --- /dev/null +++ b/tools/cli/README.rst @@ -0,0 +1,71 @@ +================= +InfraRed CLI tool +================= + +Intended to reduce InfraRed users' dependency on external CLI tools. + +Setup +===== + +.. note:: On Fedora 23 `BZ#1103566 `_ + calls for:: + + $ dnf install redhat-rpm-config + +Use pip to install from source:: + + $ pip install tools/cli + +.. note:: For development work it's better to install in editable mode:: + + $ pip install -e tools/cli + +Conf +==== + +.. note:: Assumes that ``infrared`` is installed, else follow Setup_. + +``infrared`` will look for ``infrared.cfg`` in the following order: + +#. In working directory: ``./infrared.cfg`` +#. In user home directory: ``~/.infrared.cfg`` +#. In system settings: ``/etc/infrared /infrared.cfg`` + +.. note:: To specify a different directory or different filename, override the + lookup order with ``IR_CONFIG`` environment variable:: + + $ IR_CONFIG=/my/config/file.ini infrared --help + +Running InfraRed +================ + +.. note:: Assumes that ``infrared`` is installed, else follow Setup_. + +You can get general usage information with the ``--help`` option:: + + infrared --help + +This displays options you can pass to ``infrared``. + +Extra-Vars +---------- +One can set/overwrite settings in the output file using the '-e/--extra-vars' +option. There are 2 ways of doing so: + +1. specific settings: (``key=value`` form) + ``--extra-vars provisioner.site.user=a_user`` +2. path to a settings file: (starts with ``@``) + ``--extra-vars @path/to/a/settings_file.yml`` + +The ``-e``/``--extra-vars`` can be used more than once. + +Merging order +------------- +Except options based on the settings dir structure, ``infrared`` accepts input of +predefined settings files (with ``-n``/``--input``) and user defined specific options +(``-e``/``--extra-vars``). +The merging priority order listed below: + +1. Input files +2. Settings dir based options +3. Extra Vars diff --git a/tools/kcli/kcli/__init__.py b/tools/cli/cli/__init__.py similarity index 100% rename from tools/kcli/kcli/__init__.py rename to tools/cli/cli/__init__.py diff --git a/tools/kcli/kcli/conf.py b/tools/cli/cli/conf.py similarity index 68% rename from tools/kcli/kcli/conf.py rename to tools/cli/cli/conf.py index faf86bbaa..7634485cd 100644 --- a/tools/kcli/kcli/conf.py +++ b/tools/cli/cli/conf.py @@ -1,15 +1,17 @@ import ConfigParser import os +import time -from kcli import exceptions +from cli import exceptions -from kcli.exceptions import IRFileNotFoundException - -ENV_VAR_NAME = "KCLI_CONFIG" -KCLI_CONF_FILE = 'kcli.cfg' -CWD_PATH = os.path.join(os.getcwd(), KCLI_CONF_FILE) -USER_PATH = os.path.expanduser('~/.' + KCLI_CONF_FILE) -SYSTEM_PATH = os.path.join('/etc/khaleesi', KCLI_CONF_FILE) +ENV_VAR_NAME = "IR_CONFIG" +IR_CONF_FILE = 'infrared.cfg' +CWD_PATH = os.path.join(os.getcwd(), IR_CONF_FILE) +USER_PATH = os.path.expanduser('~/.' + IR_CONF_FILE) +SYSTEM_PATH = os.path.join('/etc/infrared', IR_CONF_FILE) +YAML_EXT = ".yml" +TMP_OUTPUT_FILE = 'ir_settings_' + str(time.time()) + YAML_EXT +INFRARED_DIR_ENV_VAR = 'IR_SETTINGS' def load_config_file(): @@ -22,7 +24,7 @@ def load_config_file(): if env_path is not None: env_path = os.path.expanduser(env_path) if os.path.isdir(env_path): - env_path = os.path.join(env_path, KCLI_CONF_FILE) + env_path = os.path.join(env_path, IR_CONF_FILE) for path in (env_path, CWD_PATH, USER_PATH, SYSTEM_PATH): if path is not None and os.path.exists(path): _config.read(path) diff --git a/tools/kcli/kcli/exceptions.py b/tools/cli/cli/exceptions.py similarity index 69% rename from tools/kcli/kcli/exceptions.py rename to tools/cli/cli/exceptions.py index 50b470fc7..b79815fd8 100644 --- a/tools/kcli/kcli/exceptions.py +++ b/tools/cli/cli/exceptions.py @@ -38,3 +38,18 @@ class IRPlaybookFailedException(IRException): def __init__(self, playbook): super(self.__class__, self).__init__( 'Playbook "%s" failed!' % playbook) + + +class IRYAMLConstructorError(IRException): + def __init__(self, mark_obj, where): + self.message = mark_obj.problem + pm = mark_obj.problem_mark + self.message += ' in:\n "{where}", line {line_no}, column ' \ + '{column_no}'.format(where=where, + line_no=pm.line + 1, + column_no=pm.column + 1) + + +class IRPlaceholderException(IRException): + def __init__(self, trace_message): + self.message = 'Mandatory value is missing.\n' + trace_message diff --git a/tools/kcli/kcli/execute/execute.py b/tools/cli/cli/execute.py similarity index 94% rename from tools/kcli/kcli/execute/execute.py rename to tools/cli/cli/execute.py index 47efb6f9a..66d17826e 100644 --- a/tools/kcli/kcli/execute/execute.py +++ b/tools/cli/cli/execute.py @@ -6,14 +6,19 @@ import ansible.utils from ansible import callbacks -from kcli import conf, exceptions -from kcli.execute import core +from cli import conf, exceptions, logger +LOG = logger.LOG + +VERBOSITY = 0 HOSTS_FILE = "hosts" LOCAL_HOSTS = "local_hosts" PROVISION = "provision" PLAYBOOKS = [PROVISION, "install", "test", "collect-logs", "cleanup"] +assert "playbooks" == path.basename(conf.PLAYBOOKS_DIR), \ + "Bad path to playbooks" + # ansible-playbook # https://github.com/ansible/ansible/blob/devel/bin/ansible-playbook @@ -135,11 +140,11 @@ def execute_ansible(playbook, args): def ansible_wrapper(args): - """Wraps the 'anisble-playbook' CLI.""" + """ Wraps the 'ansible-playbook' CLI. """ playbooks = [p for p in PLAYBOOKS if getattr(args, p, False)] if not playbooks: - core.parser.error("No playbook to execute (%s)" % PLAYBOOKS) + LOG.error("No playbook to execute (%s)" % PLAYBOOKS) for playbook in (p for p in PLAYBOOKS if getattr(args, p, False)): print "Executing Playbook: %s" % playbook diff --git a/tools/cli/cli/logger.py b/tools/cli/cli/logger.py new file mode 100644 index 000000000..07eac01cd --- /dev/null +++ b/tools/cli/cli/logger.py @@ -0,0 +1,55 @@ +import logging +import sys +import traceback + +import colorlog + +from cli import exceptions + +logger_formatter = colorlog.ColoredFormatter( + "%(log_color)s%(levelname)-8s%(message)s", + log_colors=dict( + DEBUG='blue', + INFO='green', + WARNING='yellow', + ERROR='red', + CRITICAL='bold_red,bg_white', + ) +) + +LOGGER_NAME = "IRLogger" +DEFAULT_LOG_LEVEL = logging.WARNING + +LOG = logging.getLogger(LOGGER_NAME) +LOG.setLevel(DEFAULT_LOG_LEVEL) + +# Create stream handler with debug level +sh = logging.StreamHandler() +sh.setLevel(logging.DEBUG) + +# Add the logger_formatter to sh +sh.setFormatter(logger_formatter) + +# Create logger and add handler to it +LOG.addHandler(sh) + + +def ir_excepthook(exc_type, exc_value, exc_traceback): + """ + exception hook that sends IRException to log and other exceptions to + stderr (default excepthook) + """ + + # sends full exception with trace to log + if not isinstance(exc_value, exceptions.IRException): + return sys.__excepthook__(exc_type, exc_value, exc_traceback) + + if LOG.getEffectiveLevel() <= logging.DEBUG: + formated_exception = "".join( + traceback.format_exception(exc_type, exc_value, exc_traceback)) + LOG.error(formated_exception + exc_value.message) + else: + LOG.error(exc_value.message) + + +sys.excepthook = ir_excepthook diff --git a/tools/cli/cli/main.py b/tools/cli/cli/main.py new file mode 100755 index 000000000..224f7d48b --- /dev/null +++ b/tools/cli/cli/main.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +import logging +import os + +import yaml + +# logger creation is first thing to be done +from cli import logger + +from cli import conf +from cli import options as cli_options +from cli import execute +from cli import parse +from cli import utils +import cli.yamls + +LOG = logger.LOG +CONF = conf.config + + +def main(): + options_trees = [] + settings_files = [] + settings_dir = utils.validate_settings_dir( + CONF.get('DEFAULTS', 'SETTINGS_DIR')) + + for option in CONF.options('ROOT_OPTS'): + options_trees.append(cli_options.OptionsTree(settings_dir, option)) + + parser = parse.create_parser(options_trees) + args = parser.parse_args() + + verbose = int(args.verbose) + + if args.verbose == 0: + args.verbose = logging.WARNING + elif args.verbose == 1: + args.verbose = logging.INFO + else: + args.verbose = logging.DEBUG + + LOG.setLevel(args.verbose) + + # settings generation stage + if args.which.lower() != 'execute': + for input_file in args.input: + settings_files.append(utils.normalize_file(input_file)) + + for options_tree in options_trees: + options = {key: value for key, value in vars(args).iteritems() + if value and key.startswith(options_tree.name)} + + settings_files += (options_tree.get_options_ymls(options)) + + LOG.debug("All settings files to be loaded:\n%s" % settings_files) + + cli.yamls.Lookup.settings = utils.generate_settings(settings_files, + args.extra_vars) + + LOG.debug("Dumping settings...") + output = yaml.safe_dump(cli.yamls.Lookup.settings, + default_flow_style=False) + + if args.output_file: + with open(args.output_file, 'w') as output_file: + output_file.write(output) + else: + print output + + exec_playbook = (args.which == 'execute') or \ + (not args.dry_run and args.which in CONF.options( + 'AUTO_EXEC_OPTS')) + + # playbook execution stage + if exec_playbook: + if args.which == 'execute': + execute_args = parser.parse_args() + elif args.which not in execute.PLAYBOOKS: + LOG.debug("No playbook named \"%s\", nothing to execute.\n" + "Please choose from: %s" % (args.which, + execute.PLAYBOOKS)) + return + else: + args_list = ["execute"] + if verbose: + args_list.append('-%s' % ('v' * verbose)) + if 'inventory' in args: + inventory = args.inventory + else: + inventory = 'local_hosts' if args.which == 'provision' \ + else 'hosts' + args_list.append('--inventory=%s' % inventory) + args_list.append('--' + args.which) + args_list.append('--collect-logs') + if args.output_file: + LOG.debug('Using the newly created settings file: "%s"' + % args.output_file) + args_list.append('--settings=%s' % args.output_file) + else: + with open(conf.TMP_OUTPUT_FILE, 'w') as output_file: + output_file.write(output) + LOG.debug('Temporary settings file "%s" has been created for ' + 'execution purpose only.' % conf.TMP_OUTPUT_FILE) + args_list.append('--settings=%s' % conf.TMP_OUTPUT_FILE) + + execute_args = parser.parse_args(args_list) + + LOG.debug("execute parser args: %s" % args) + execute_args.func(execute_args) + + if not args.output_file and args.which != 'execute': + LOG.debug('Temporary settings file "%s" has been deleted.' + % conf.TMP_OUTPUT_FILE) + os.remove(conf.TMP_OUTPUT_FILE) + + +if __name__ == '__main__': + main() diff --git a/tools/cli/cli/options.py b/tools/cli/cli/options.py new file mode 100644 index 000000000..942fad437 --- /dev/null +++ b/tools/cli/cli/options.py @@ -0,0 +1,160 @@ +""" +This module is the building blocks for the options parsing tree. +It contains the data structures that hold available options & values in a +given directory. +""" + +import os + +import yaml + +from cli import conf +from cli import exceptions +from cli import logger + +LOG = logger.LOG + + +class OptionNode(object): + """ + represents an option and its properties: + - parent option + - available values + - option's path + - sub options + """ + def __init__(self, path, parent=None): + self.path = path + self.option = self.path.split("/")[-1] + self.parent = parent + self.parent_value = None + if parent: + self.option = "-".join([self.parent.option, self.option]) + self.values = self._get_values() + self.children = {i: dict() for i in self._get_sub_options()} + + if self.parent: + self.parent_value = self.path.split("/")[-2] + self.parent.children[self.parent_value][self.option] = self + + def _get_values(self): + """Returns a sorted list of values available for the current option""" + values = [a_file.split(conf.YAML_EXT)[0] + for a_file in os.listdir(self.path) + if os.path.isfile(os.path.join(self.path, a_file)) and + a_file.endswith(conf.YAML_EXT)] + + values.sort() + return values + + def _get_sub_options(self): + """ + Returns a sorted list of sup-options available for the current option + """ + options = [options_dir for options_dir in os.listdir(self.path) + if os.path.isdir(os.path.join(self.path, options_dir)) and + options_dir in self.values] + + options.sort() + return options + + +class OptionsTree(object): + """ + Tree represents hierarchy of options from rhe same kind (provisioner, + installer etc...) + """ + def __init__(self, settings_dir, option): + self.root = None + self.name = option + self.action = option[:-2] if option.endswith('er') else option + self.options_dict = {} + self.root_dir = os.path.join(settings_dir, self.name) + + self.build_tree() + self.init_options_dict(self.root) + + def build_tree(self): + """Builds the OptionsTree""" + self.add_node(self.root_dir, None) + + def add_node(self, path, parent): + """ + Adds OptionNode object to the tree + :param path: Path to option dir + :param parent: Parent option (OptionNode) + """ + node = OptionNode(path, parent) + + if not self.root: + self.root = node + + for child in node.children: + sub_options_dir = os.path.join(node.path, child) + sub_options = [a_dir for a_dir in os.listdir(sub_options_dir) if + os.path.isdir(os.path.join(sub_options_dir, a_dir))] + + for sub_option in sub_options: + self.add_node(os.path.join(sub_options_dir, sub_option), node) + + def init_options_dict(self, node): + """ + Initialize "options_dict" dictionary to store all options and their + valid values + :param node: OptionNode object + """ + if node.option not in self.options_dict: + self.options_dict[node.option] = {} + + if node.parent_value: + self.options_dict[node.option][node.parent_value] = node.values + + if 'ALL' not in self.options_dict[node.option]: + self.options_dict[node.option]['ALL'] = set() + + self.options_dict[node.option]['ALL'].update(node.values) + + for pre_value in node.children: + for child in node.children[pre_value].values(): + self.init_options_dict(child) + + def get_options_ymls(self, options): + """return list of paths to settings YAML files for a given options + dictionary + + :param options: dictionary of options to get path of their settings + files + :return: list of paths to settings files of 'options' + """ + ymls = [] + if not options: + return ymls + + keys = options.keys() + keys.sort() + + def step_in(key, node): + """recursive method that returns the settings files of a given + options + """ + keys.remove(key) + if node.option != key.replace("_", "-"): + raise exceptions.IRMissingAncestorException(key) + + ymls.append(os.path.join(node.path, options[key] + ".yml")) + child_keys = [child_key for child_key in keys + if child_key.startswith(key) and + len(child_key.split("_")) == len(key.split("_")) + 1 + ] + + for child_key in child_keys: + step_in(child_key, node.children[options[key]][ + child_key.replace("_", "-")]) + + step_in(keys[0], self.root) + LOG.debug("%s tree settings files:\n%s" % (self.name, ymls)) + + return ymls + + def __str__(self): + return yaml.safe_dump(self.options_dict, default_flow_style=False) diff --git a/tools/kcli/kcli/parse.py b/tools/cli/cli/parse.py similarity index 74% rename from tools/kcli/kcli/parse.py rename to tools/cli/cli/parse.py index fb50be260..5325244bc 100644 --- a/tools/kcli/kcli/parse.py +++ b/tools/cli/cli/parse.py @@ -1,7 +1,6 @@ from argparse import ArgumentParser, RawTextHelpFormatter -from execute.core import * -from execute.execute import * +from cli import conf, execute, utils def create_parser(options_trees): @@ -10,20 +9,20 @@ def create_parser(options_trees): :param options_trees: An iterable with OptionsTree objects :return: Namespace object """ - parser = ArgumentParser(prog="kcli", - formatter_class=RawTextHelpFormatter) + parser = ArgumentParser(formatter_class=RawTextHelpFormatter) sub_parsers = parser.add_subparsers() execute_parser = sub_parsers.add_parser('execute') execute_parser.add_argument('-i', '--inventory', default=None, - type=lambda x: core.file_exists( - execute_parser, x), + type=lambda file_path: utils.normalize_file( + file_path), help="Inventory file to use. " "Default: {lcl}. " "NOTE: to reuse old environment use {" "host}". - format(lcl=LOCAL_HOSTS, host=HOSTS_FILE)) + format(lcl=execute.LOCAL_HOSTS, + host=execute.HOSTS_FILE)) execute_parser.add_argument("-v", "--verbose", help="verbosity", action='count', default=0) execute_parser.add_argument("--provision", action="store_true", @@ -37,10 +36,11 @@ def create_parser(options_trees): execute_parser.add_argument("--cleanup", action="store_true", help="cleanup nodes") execute_parser.add_argument("--settings", - type=lambda x: file_exists(parser, x), + type=lambda file_path: utils.normalize_file( + file_path), help="settings file to use. default: %s" - % conf.KCLI_SETTINGS_YML) - execute_parser.set_defaults(func=ansible_wrapper) + % conf.IR_SETTINGS_YML) + execute_parser.set_defaults(func=execute.ansible_wrapper) execute_parser.set_defaults(which='execute') for options_tree in options_trees: @@ -52,11 +52,14 @@ def create_parser(options_trees): sub_parser.add_argument("-d", "--dry-run", action='store_true', help="skip playbook execution stage") sub_parser.add_argument("-e", "--extra-vars", default=list(), - action='append', help="Provide extra vars") + action='append', help="Provide extra vars. " + "(key=value, or a path " + "to settings file if " + "starts with '@')") sub_parser.add_argument("-n", "--input", action='append', - help="a settings file that will be loaded " - "first, all other settings file will be" - " merged with it", default=list()) + help="Settings files to be loaded first," + " other settings files will be" + " merged with them", default=list()) sub_parser.add_argument("-o", "--output-file", help="file to dump the settings into") sub_parser.add_argument("-v", "--verbose", help="verbosity", diff --git a/tools/cli/cli/utils.py b/tools/cli/cli/utils.py new file mode 100644 index 000000000..36f3cef9d --- /dev/null +++ b/tools/cli/cli/utils.py @@ -0,0 +1,141 @@ +""" +This module provide some general helper methods +""" + +import os +import re + +import configure +import yaml + +import cli.yamls +import cli.conf +from cli import exceptions +from cli import logger + +LOG = logger.LOG + + +def dict_insert(dic, val, key, *keys): + """insert a value of a nested key into a dictionary + + to insert value for a nested key, all ancestor keys should be given as + method's arguments + + example: + dict_insert({}, 'val', 'key1.key2'.split('.')) + + :param dic: a dictionary object to insert the nested key value into + :param val: a value to insert to the given dictionary + :param key: first key in a chain of key that will store the value + :param keys: sub keys in the keys chain + """ + if not keys: + dic[key] = val + return + + dict_insert(dic.setdefault(key, {}), val, *keys) + + +# TODO: remove "settings" references in project +def validate_settings_dir(settings_dir=None): + """Checks & returns the full path to the settings dir. + + Path is set in the following priority: + 1. Method argument + 2. System environment variable + + :param settings_dir: path given as argument by a user + :return: path to settings dir (str) + :raise: IRFileNotFoundException: when the path to the settings dir doesn't + exist + """ + settings_dir = settings_dir or os.environ.get( + cli.conf.INFRARED_DIR_ENV_VAR) + + if not os.path.exists(settings_dir): + raise exceptions.IRFileNotFoundException( + settings_dir, + "Settings dir doesn't exist: ") + + return settings_dir + + +def update_settings(settings, file_path): + """merge settings in 'file_path' with 'settings' + + :param settings: settings to be merge with (configure.Configuration) + :param file_path: path to file with settings to be merged + :return: merged settings + """ + LOG.debug("Loading setting file: %s" % file_path) + if not os.path.exists(file_path): + raise exceptions.IRFileNotFoundException(file_path) + + try: + loaded_file = configure.Configuration.from_file(file_path).configure() + placeholders_list = cli.yamls.Placeholder.placeholders_list + for placeholder in placeholders_list[::-1]: + if placeholders_list[-1].file_path is None: + placeholder.file_path = file_path + else: + break + except yaml.constructor.ConstructorError as e: + raise exceptions.IRYAMLConstructorError(e, file_path) + + settings = settings.merge(loaded_file) + + return settings + + +def generate_settings(settings_files, extra_vars): + """ Generates one settings object (configure.Configuration) by merging all + files in settings file & extra-vars + + files in 'settings_files' are the first to be merged and after them the + 'extra_vars' + + :param settings_files: list of paths to settings files + :param extra_vars: list of extra-vars + :return: Configuration object with merging results of all settings + files and extra-vars + """ + settings = configure.Configuration.from_dict({}) + + for settings_file in settings_files: + settings = update_settings(settings, settings_file) + + for extra_var in extra_vars: + if extra_var.startswith('@'): + settings_file = normalize_file(extra_var[1:]) + settings = update_settings(settings, settings_file) + + else: + if '=' not in extra_var: + raise exceptions.IRExtraVarsException(extra_var) + key, value = extra_var.split("=") + dict_insert(settings, value, *key.split(".")) + + return settings + + +# todo: convert into a file object to be consumed by argparse +def normalize_file(file_path): + """Return a normalized absolutized version of a file + + :param file_path: path to file to be normalized + :return: normalized path of a file + :raise: IRFileNotFoundException if the file doesn't exist + """ + if not os.path.isabs(file_path): + abspath = os.path.abspath(file_path) + LOG.debug( + 'Setting the absolute path of "%s" to: "%s"' + % (file_path, abspath) + ) + file_path = abspath + + if not os.path.exists(file_path): + raise exceptions.IRFileNotFoundException(file_path) + + return file_path diff --git a/tools/cli/cli/yamls.py b/tools/cli/cli/yamls.py new file mode 100644 index 000000000..089c52fc1 --- /dev/null +++ b/tools/cli/cli/yamls.py @@ -0,0 +1,308 @@ +""" +This module contains the tools for handling YAML files and tags. +""" + +import logging +import re +import sys +import string + +import configure +import yaml + +from cli import exceptions +from cli import logger + +LOG = logger.LOG + +# Representer for Configuration object +yaml.SafeDumper.add_representer( + configure.Configuration, + lambda dumper, value: + yaml.representer.BaseRepresenter.represent_mapping + (dumper, u'tag:yaml.org,2002:map', value)) + + +def random_generator(size=32, chars=string.ascii_lowercase + string.digits): + import random + + return ''.join(random.choice(chars) for _ in range(size)) + + +@configure.Configuration.add_constructor('join') +def _join_constructor(loader, node): + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + + +@configure.Configuration.add_constructor('random') +def _random_constructor(loader, node): + """ + usage: + !random + returns a random string of characters + """ + + num_chars = loader.construct_scalar(node) + return random_generator(int(num_chars)) + + +def _limit_chars(_string, length): + length = int(length) + if length < 0: + raise exceptions.IRException('length to crop should be int, not ' + + str(length)) + + return _string[:length] + + +@configure.Configuration.add_constructor('limit_chars') +def _limit_chars_constructor(loader, node): + """ + Usage: + !limit_chars [, ] + Method returns first param cropped to chars. + """ + + params = loader.construct_sequence(node) + if len(params) != 2: + raise exceptions.IRException( + 'limit_chars requires two params: string length') + return _limit_chars(params[0], params[1]) + + +@configure.Configuration.add_constructor('env') +def _env_constructor(loader, node): + """ + usage: + !env + !env [, [default]] + !env [, [default], [length]] + returns value for the environment var-name + default may be specified by passing a second parameter in a list + length is maximum length of output (croped to that length) + """ + + import os + # scalar node or string has no defaults, + # raise IRUndefinedEnvironmentVariableExcption if absent + if isinstance(node, yaml.nodes.ScalarNode): + try: + return os.environ[loader.construct_scalar(node)] + except KeyError: + raise exceptions.IRUndefinedEnvironmentVariableExcption(node.value) + + seq = loader.construct_sequence(node) + var = seq[0] + if len(seq) >= 2: + ret = os.getenv(var, seq[1]) # second item is default val + + # third item is max. length + if len(seq) == 3: + ret = _limit_chars(ret, seq[2]) + return ret + + return os.environ[var] + + +class Lookup(yaml.YAMLObject): + yaml_tag = u'!lookup' + yaml_dumper = yaml.SafeDumper + + settings = None + handling_nested_lookups = False + + def __init__(self, key, old_style_lookup=False): + self.key = key + if old_style_lookup: + self.convert_old_style_lookup() + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, self.key) + + def convert_old_style_lookup(self): + self.key = '{{!lookup %s}}' % self.key + + parser = re.compile('\[\s*\!lookup\s*[\w.]*\s*\]') + lookups = parser.findall(self.key) + + for lookup in lookups: + self.key = self.key.replace(lookup, '.{{%s}}' % lookup[1:-1]) + + def replace_lookup(self): + """ + Replace any !lookup with the corresponding value from settings table + """ + while True: + parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') + lookups = parser.findall(self.key) + + if not lookups: + break + + for a_lookup in lookups: + lookup_key = re.search('(\w+\.?)+ *?\}\}', a_lookup) + lookup_key = lookup_key.group(0).strip()[:-2].strip() + lookup_value = self.dict_lookup(lookup_key.split(".")) + + if isinstance(lookup_value, Lookup): + return + + lookup_value = str(lookup_value) + + self.key = re.sub('\{\{\s*\!lookup\s*[\w.]*\s*\}\}', + lookup_value, self.key, count=1) + + def dict_lookup(self, keys, dic=None): + """ Returns the value of a given key from the settings class variable + + to get the value of a nested key, all ancestor keys should be given as + method's arguments + + example: + if one want to get the value of 'key3' in: + {'key1': {'key2': {'key3': 'val1'}}} + + dict_lookup(['key1', 'key2', 'key3']) + return value: + 'val1' + + :param keys: list with keys describing the path to the target key + :param dic: mapping object holds settings. (self.settings by default) + + :return: value of the target key + """ + if LOG.getEffectiveLevel() <= logging.DEBUG: + calling_method_name = sys._getframe().f_back.f_code.co_name + current_method_name = sys._getframe().f_code.co_name + if current_method_name != calling_method_name: + LOG.debug( + 'looking up the value of "{keys}"'.format( + keys=".".join(keys))) + + if dic is None: + dic = self.settings + + key = keys.pop(0) + + if key not in dic: + if isinstance(key, str) and key.isdigit(): + key = int(key) + elif isinstance(key, int): + key = str(key) + + try: + if keys: + return self.dict_lookup(keys, dic[key]) + + value = dic[key] + except KeyError: + raise exceptions.IRKeyNotFoundException(key, dic) + + LOG.debug('value has been found: "{value}"'.format(value=value)) + return value + + @classmethod + def in_string_lookup(cls, settings_dic=None): + """ convert strings containing '!lookup' in them, and didn't already + converted into Lookup objects. + (in case when the strings don't start with '!lookup') + + :param settings_dic: a settings dictionary to search and convert + lookup from + """ + + if settings_dic is None: + settings_dic = cls.settings + + my_iter = settings_dic.iteritems() if isinstance(settings_dic, dict) \ + else enumerate(settings_dic) + + for idx_key, value in my_iter: + if isinstance(value, dict): + cls.in_string_lookup(settings_dic[idx_key]) + elif isinstance(value, list): + cls.in_string_lookup(value) + elif isinstance(value, str): + parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') + lookups = parser.findall(value) + + if lookups: + settings_dic[idx_key] = cls(value) + + @classmethod + def handle_nested_lookup(cls): + """ handles lookup to lookup (nested lookup scenario) + + load and dump 'settings' again and again until all lookups strings + are converted into Lookup objects + """ + + # because there is a call to 'yaml.safe_dump' which call to this + # method, the 'handling_nested_lookups' flag is being set & unset to + # prevent infinite loop between the method + cls.handling_nested_lookups = True + + first_dump = True + settings = cls.settings + + while True: + if not first_dump: + cls.settings = settings + settings = yaml.load(output) + + cls.in_string_lookup() + output = yaml.safe_dump(cls.settings, default_flow_style=False) + + if first_dump: + first_dump = False + continue + + if not cmp(settings, cls.settings): + break + + cls.handling_nested_lookups = False + + @classmethod + def from_yaml(cls, loader, node): + return Lookup(loader.construct_scalar(node), old_style_lookup=True) + + @classmethod + def to_yaml(cls, dumper, node): + if not cls.handling_nested_lookups: + cls.handle_nested_lookup() + + if node.settings: + node.replace_lookup() + + return dumper.represent_data("%s" % node.key) + + +class Placeholder(yaml.YAMLObject): + """ Raises 'IRPlaceholderException' when dumping Placeholder objects. + + Objects created by 'from_yaml' method are automatically added to the + 'placeholders_list' class variable so it'll be possible to add for each + object the path to the file where it stored. + """ + yaml_tag = u'!placeholder' + yaml_dumper = yaml.SafeDumper + + # Refs for all Placeholder's objects + placeholders_list = [] + + def __init__(self, message): + self.message = message + self.file_path = None + + @classmethod + def from_yaml(cls, loader, node): + # Create & save references to Placeholder objects + placeholder = Placeholder(str(node.start_mark)) + cls.placeholders_list.append(placeholder) + return placeholder + + @classmethod + def to_yaml(cls, dumper, node): + message = re.sub("", node.file_path, node.message) + raise exceptions.IRPlaceholderException(message) diff --git a/tools/cli/etc/infrared.cfg.example b/tools/cli/etc/infrared.cfg.example new file mode 100644 index 000000000..988365385 --- /dev/null +++ b/tools/cli/etc/infrared.cfg.example @@ -0,0 +1,19 @@ +[DEFAULTS] +INFRARED_DIR = /home/aopincar/Git/Repos/Infrared +SETTINGS_DIR = %(INFRARED_DIR)s/settings +MODULES_DIR = %(INFRARED_DIR)s/library +ROLES_DIR = %(INFRARED_DIR)s/roles +PLAYBOOKS_DIR = %(INFRARED_DIR)s/playbooks +IR_SETTINGS_YML = ir_settings.yml + +[ROOT_OPTS] +provisioner +installer +tester +distro +product + +[AUTO_EXEC_OPTS] +provision +install +test diff --git a/tools/kcli/requirements.txt b/tools/cli/requirements.txt similarity index 100% rename from tools/kcli/requirements.txt rename to tools/cli/requirements.txt diff --git a/tools/kcli/setup.py b/tools/cli/setup.py similarity index 84% rename from tools/kcli/setup.py rename to tools/cli/setup.py index 83f44a587..ed05ecf84 100644 --- a/tools/kcli/setup.py +++ b/tools/cli/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages from os.path import join, dirname, abspath -import kcli +import cli # parse_requirements() returns generator of pip.req.InstallRequirement objects install_reqs = req.parse_requirements('requirements.txt', session=False) @@ -13,12 +13,12 @@ prj_dir = dirname(abspath(__file__)) setup( - name='kcli', - version=kcli.__VERSION__, + name='infrared', + version=cli.__VERSION__, packages=find_packages(), long_description=open(join(prj_dir, 'README.rst')).read(), entry_points={ - 'console_scripts': ['kcli = kcli.main:main'] + 'console_scripts': ['infrared = cli.main:main'] }, install_requires=reqs, author='Yair Fried', diff --git a/tools/kcli/tests/__init__.py b/tools/cli/tests/__init__.py similarity index 100% rename from tools/kcli/tests/__init__.py rename to tools/cli/tests/__init__.py diff --git a/tools/cli/tests/test_conf.py b/tools/cli/tests/test_conf.py new file mode 100644 index 000000000..75535bc56 --- /dev/null +++ b/tools/cli/tests/test_conf.py @@ -0,0 +1,13 @@ +import os + +from test_cwd import utils + +our_cwd_setup = utils.our_cwd_setup + + +def test_get_config_dir(our_cwd_setup): + from cli import conf + conf_file = conf.load_config_file() + assert os.path.abspath( + conf_file.get("DEFAULTS", "INFRARED_DIR")) == os.path.abspath( + utils.TESTS_CWD) diff --git a/tools/cli/tests/test_cwd/IRYAMLConstructorError.yml b/tools/cli/tests/test_cwd/IRYAMLConstructorError.yml new file mode 100644 index 000000000..e567c5144 --- /dev/null +++ b/tools/cli/tests/test_cwd/IRYAMLConstructorError.yml @@ -0,0 +1,3 @@ +--- +key: + sub_key: !notag diff --git a/tools/kcli/tests/test_cwd/__init__.py b/tools/cli/tests/test_cwd/__init__.py similarity index 100% rename from tools/kcli/tests/test_cwd/__init__.py rename to tools/cli/tests/test_cwd/__init__.py diff --git a/tools/cli/tests/test_cwd/infrared.cfg b/tools/cli/tests/test_cwd/infrared.cfg new file mode 100644 index 000000000..7964cf6c8 --- /dev/null +++ b/tools/cli/tests/test_cwd/infrared.cfg @@ -0,0 +1,3 @@ +[DEFAULTS] +INFRARED_DIR = . +SETTINGS_DIR = %(INFRARED_DIR)s/settings diff --git a/tools/cli/tests/test_cwd/placeholder_injector.yml b/tools/cli/tests/test_cwd/placeholder_injector.yml new file mode 100644 index 000000000..150cc1285 --- /dev/null +++ b/tools/cli/tests/test_cwd/placeholder_injector.yml @@ -0,0 +1,5 @@ +--- +place: + holder: + validator: + !placeholder "tester for '!placeholder' tag" diff --git a/tools/cli/tests/test_cwd/placeholder_overwriter.yml b/tools/cli/tests/test_cwd/placeholder_overwriter.yml new file mode 100644 index 000000000..bedb599e7 --- /dev/null +++ b/tools/cli/tests/test_cwd/placeholder_overwriter.yml @@ -0,0 +1,5 @@ +--- +place: + holder: + validator: + "'!placeholder' has been overwritten" diff --git a/tools/cli/tests/test_cwd/utils.py b/tools/cli/tests/test_cwd/utils.py new file mode 100644 index 000000000..db4cb3dfb --- /dev/null +++ b/tools/cli/tests/test_cwd/utils.py @@ -0,0 +1,33 @@ +import os +import pytest + +TESTS_CWD = os.path.dirname(__file__) +SETTINGS_PATH = os.path.join(TESTS_CWD, "settings") + + +@pytest.yield_fixture +def os_environ(): + """ Backups env var from os.environ and restores it at teardown. """ + + from cli import conf + + backup_flag = False + if conf.ENV_VAR_NAME in os.environ: + backup_flag = True + backup_value = os.environ.get(conf.ENV_VAR_NAME) + yield os.environ + if backup_flag: + os.environ[conf.ENV_VAR_NAME] = backup_value + + +@pytest.fixture() +def our_cwd_setup(request): + """ Change cwd to test_cwd dir. Revert to original dir on teardown. """ + + bkp = os.getcwd() + + def our_cwd_teardown(): + os.chdir(bkp) + + request.addfinalizer(our_cwd_teardown) + os.chdir(TESTS_CWD) diff --git a/tools/cli/tests/test_utils.py b/tools/cli/tests/test_utils.py new file mode 100644 index 000000000..5d35cda04 --- /dev/null +++ b/tools/cli/tests/test_utils.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.mark.parametrize('tested, val, key, expected', [ + ({}, 'val', ['key'], {'key': 'val'}), + + ({}, 'val', ['key1', 'key2', 'key3'], {'key1': {'key2': {'key3': 'val'}}}), + + ({'a_key': 'a_val', 'b_key1': {'b_key2': {'b_key3': 'b_val'}}}, 'x_val', + ['b_key1', 'b_key2'], {'a_key': 'a_val', 'b_key1': {'b_key2': 'x_val'}}), +]) +def test_dict_insert(tested, val, key, expected): + from cli import utils + utils.dict_insert(tested, val, *key) + assert tested == expected diff --git a/tools/cli/tests/test_yaml.py b/tools/cli/tests/test_yaml.py new file mode 100644 index 000000000..7247e32c8 --- /dev/null +++ b/tools/cli/tests/test_yaml.py @@ -0,0 +1,47 @@ +import os.path + +import configure +import pytest +import yaml + +from tests.test_cwd import utils + +our_cwd_setup = utils.our_cwd_setup + + +def test_unsupported_yaml_constructor(our_cwd_setup): + from cli.utils import update_settings + from cli.exceptions import IRYAMLConstructorError + tester_file = 'IRYAMLConstructorError.yml' + settings = configure.Configuration.from_dict({}) + with pytest.raises(IRYAMLConstructorError): + update_settings(settings, os.path.join(utils.TESTS_CWD, tester_file)) + + +def test_placeholder_validator(our_cwd_setup): + from cli.utils import update_settings + from cli.exceptions import IRPlaceholderException + from cli.yamls import Placeholder + + injector = 'placeholder_injector.yml' + overwriter = 'placeholder_overwriter.yml' + + # Checks that 'IRPlaceholderException' is raised if value isn't been + # overwritten + settings = configure.Configuration.from_dict({}) + settings = update_settings(settings, + os.path.join(utils.TESTS_CWD, injector)) + + assert isinstance(settings['place']['holder']['validator'], Placeholder) + with pytest.raises(IRPlaceholderException) as exc: + yaml.safe_dump(settings, default_flow_style=False) + assert "Mandatory value is missing." in str(exc.value.message) + + # Checks that exceptions haven't been raised after overwriting the + # placeholder + settings = update_settings(settings, + os.path.join(utils.TESTS_CWD, overwriter)) + + assert settings['place']['holder'][ + 'validator'] == "'!placeholder' has been overwritten" + yaml.safe_dump(settings, default_flow_style=False) diff --git a/tools/kcli/README.rst b/tools/kcli/README.rst deleted file mode 100644 index c873bfd9d..000000000 --- a/tools/kcli/README.rst +++ /dev/null @@ -1,57 +0,0 @@ -======================== -kcli - Khaleesi CLI tool -======================== - -``kcli`` is intended to reduce Khaleesi users' dependency on external CLI tools. - -Setup -===== - -.. note:: On Fedora 23 `BZ#1103566 `_ - calls for:: - - $ dnf install redhat-rpm-config - -Use pip to install from source:: - - $ pip install tools/kcli - -.. note:: For development work it's better to install in editable mode:: - - $ pip install -e tools/kcli - -Conf -==== - -.. note:: Assumes that ``kcli`` is installed, else follow Setup_. - -``kcli`` will look for ``kcli.cfg`` in the following order: - -#. In working directory: ``./kcli.cfg`` -#. In user home directory: ``~/.kcli.cfg`` -#. In system settings: ``/etc/khaleesi/kcli.cfg`` - -.. note:: To specify a different directory or different filename, override the - lookup order with ``KCLI_CONFIG`` environment variable:: - - $ KCLI_CONFIG=/my/config/file.ini kcli --help - -Running kcli -============ - -.. note:: Assumes that ``kcli`` is installed, else follow Setup_. - -You can get general usage information with the ``--help`` option:: - - kcli --help - -This displays options you can pass to ``kcli``. - -.. note:: Some setting files are hard-coded to look for the ``$WORKSPACE`` - environment variable (see `Khaleesi - Cookbook`) that should point to the - directory where ``khaleesi`` and ``khaleesi-settings`` have been cloned. You - can define it manually to work around that:: - - $ export WORKSAPCE=$(dirname `pwd`) - - diff --git a/tools/kcli/etc/kcli.cfg.example b/tools/kcli/etc/kcli.cfg.example deleted file mode 100644 index eae881d2f..000000000 --- a/tools/kcli/etc/kcli.cfg.example +++ /dev/null @@ -1,19 +0,0 @@ -[DEFAULTS] -KHALEESI_DIR = /home/aopincar/Git/Repos/khaleesi -SETTINGS_DIR = %(KHALEESI_DIR)s/settings -MODULES_DIR = %(KHALEESI_DIR)s/library -ROLES_DIR = %(KHALEESI_DIR)s/roles -PLAYBOOKS_DIR = %(KHALEESI_DIR)s/playbooks -KCLI_SETTINGS_YML = kcli_settings.yml - -[ROOT_OPTS] -provisioner -installer -tester -distro -product - -[AUTO_EXEC_OPTS] -provision -install -test diff --git a/tools/kcli/kcli/execute/__init__.py b/tools/kcli/kcli/execute/__init__.py deleted file mode 100644 index c8cc42b81..000000000 --- a/tools/kcli/kcli/execute/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# from kcli.execute import core -# from kcli.execute import execute diff --git a/tools/kcli/kcli/execute/core.py b/tools/kcli/kcli/execute/core.py deleted file mode 100644 index 1882534a1..000000000 --- a/tools/kcli/kcli/execute/core.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -import argparse -import os -import sys - -from kcli import conf - -assert "playbooks" == os.path.basename(conf.PLAYBOOKS_DIR), \ - "Bad path to playbooks" - -VERBOSITY = 0 - - -def file_exists(prs, filename): - if not os.path.exists(filename): - prs.error("The file %s does not exist!" % filename) - return filename - - -def main(): - args = parser.parse_args() - args.func(args) - - -parser = argparse.ArgumentParser() -parser.add_argument('-v', '--verbose', default=VERBOSITY, action="count", - help="verbose mode (-vvv for more," - " -vvvv to enable connection debugging)") -parser.add_argument("--settings", - default=conf.KCLI_SETTINGS_YML, - type=lambda x: file_exists(parser, x), - help="settings file to use. default: %s" - % conf.KCLI_SETTINGS_YML) -subparsers = parser.add_subparsers(metavar="COMMAND") - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/kcli/kcli/logger.py b/tools/kcli/kcli/logger.py deleted file mode 100644 index 445d4d404..000000000 --- a/tools/kcli/kcli/logger.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging -import sys -import traceback - -from colorlog import ColoredFormatter - -from kcli import exceptions - -LOGGER_NAME = "IRLogger" -DEFAULT_LOGLEVEL = logging.WARNING - -debug_formatter = ColoredFormatter( - "%(log_color)s%(levelname)-8s%(message)s", - log_colors=dict( - DEBUG='blue', - INFO='green', - WARNING='yellow', - ERROR='red', - CRITICAL='bold_red,bg_white', - ) -) - - -def kcli_traceback_handler(log): - """Creates exception hook that sends IRException to log and other - exceptions to stdout (default excepthook) - :param log: logger to log trace - """ - - def my_excepthook(exc_type, exc_value, exc_traceback): - # sends full exception with trace to log - if not isinstance(exc_value, exceptions.IRException): - return sys.__excepthook__(exc_type, exc_value, exc_traceback) - - if log.getEffectiveLevel() <= logging.DEBUG: - formated_exception = "".join( - traceback.format_exception(exc_type, exc_value, exc_traceback)) - log.error(formated_exception + exc_value.message) - else: - log.error(exc_value.message) - - sys.excepthook = my_excepthook - - -LOG = logging.getLogger(LOGGER_NAME) -LOG.setLevel(DEFAULT_LOGLEVEL) - -# Create stream handler with debug level -sh = logging.StreamHandler() -sh.setLevel(logging.DEBUG) - -# Add the debug_formatter to sh -sh.setFormatter(debug_formatter) - -# Create logger and add handler to it -LOG.addHandler(sh) - -kcli_traceback_handler(LOG) diff --git a/tools/kcli/kcli/main.py b/tools/kcli/kcli/main.py deleted file mode 100755 index 683212397..000000000 --- a/tools/kcli/kcli/main.py +++ /dev/null @@ -1,475 +0,0 @@ -#!/usr/bin/env python - -import logging -import os -import re -import sys - -import yaml -import configure - -from kcli import conf -from kcli.exceptions import * -from kcli.execute.execute import PLAYBOOKS -from kcli import logger -from kcli import parse -# Contains meta-classes so we need to import it without using. -from kcli import yamls - -SETTING_FILE_EXT = ".yml" -LOG = logger.LOG -kcli_conf = conf.config - -# Representer for Configuration object -yaml.SafeDumper.add_representer( - configure.Configuration, - lambda dumper, value: - yaml.representer.BaseRepresenter.represent_mapping - (dumper, u'tag:yaml.org,2002:map', value)) - - -def dict_lookup(dic, key, *keys): - if LOG.getEffectiveLevel() <= logging.DEBUG: - calling_method_name = sys._getframe().f_back.f_code.co_name - current_method_name = sys._getframe().f_code.co_name - if current_method_name != calling_method_name: - full_key = list(keys) - full_key.insert(0, key) - LOG.debug("looking up the value of \"%s\"" % ".".join(full_key)) - - if key not in dic: - if isinstance(key, str) and key.isdigit(): - key = int(key) - elif isinstance(key, int): - key = str(key) - - if keys: - return dict_lookup(dic.get(key, {}), *keys) - - try: - value = dic[key] - except KeyError: - raise exceptions.IRKeyNotFoundException(key, dic) - - LOG.debug("value has been found: \"%s\"" % value) - return value - - -def dict_insert(dic, val, key, *keys): - if not keys: - dic[key] = val - return - - if key not in dic: - dic[key] = {} - - dict_insert(dic[key], val, *keys) - - -def validate_settings_dir(settings_dir=None): - """ - Checks & returns the full path to the settings dir. - Path is set in the following priority: - 1. Method argument - 2. System environment variable - 3. Settings dir in the current working dir - :param settings_dir: path given as argument by a user - :return: path to settings dir (str) - :raise: IRFileNotFoundException: when the path to the settings dir doesn't - exist - """ - settings_dir = settings_dir or os.environ.get( - 'KHALEESI_SETTINGS') or os.path.join(os.getcwd(), "settings", "") - - if not os.path.exists(settings_dir): - raise exceptions.IRFileNotFoundException( - settings_dir, - "Settings dir doesn't exist: ") - - return settings_dir - - -class Lookup(yaml.YAMLObject): - yaml_tag = u'!lookup' - yaml_dumper = yaml.SafeDumper - - settings = None - - def __init__(self, key, old_style_lookup=False): - self.key = key - if old_style_lookup: - self.convert_old_style_lookup() - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, self.key) - - def convert_old_style_lookup(self): - self.key = '{{!lookup %s}}' % self.key - - parser = re.compile('\[\s*\!lookup\s*[\w.]*\s*\]') - lookups = parser.findall(self.key) - - for lookup in lookups: - self.key = self.key.replace(lookup, '.{{%s}}' % lookup[1:-1]) - - def replace_lookup(self): - """ - Replace any !lookup with the corresponding value from settings table - """ - while True: - parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') - lookups = parser.findall(self.key) - - if not lookups: - break - - for a_lookup in lookups: - lookup_key = re.search('(\w+\.?)+ *?\}\}', a_lookup) - lookup_key = lookup_key.group(0).strip()[:-2].strip() - lookup_value = dict_lookup( - self.settings, *lookup_key.split(".")) - - if isinstance(lookup_value, Lookup): - return - - lookup_value = str(lookup_value) - - self.key = re.sub('\{\{\s*\!lookup\s*[\w.]*\s*\}\}', - lookup_value, self.key, count=1) - - @classmethod - def from_yaml(cls, loader, node): - return Lookup(loader.construct_scalar(node), old_style_lookup=True) - - @classmethod - def to_yaml(cls, dumper, node): - if node.settings: - node.replace_lookup() - - return dumper.represent_data("%s" % node.key) - - -class OptionNode(object): - def __init__(self, path, parent=None): - self.path = path - self.option = self.path.split("/")[-1] - self.parent = parent - self.parent_value = None - if parent: - self.option = "-".join([self.parent.option, self.option]) - self.values = self._get_values() - self.children = {i: dict() for i in self._get_sub_options()} - - if self.parent: - self.parent_value = self.path.split("/")[-2] - self.parent.children[self.parent_value][self.option] = self - - def _get_values(self): - """ - Returns a sorted list of values available for the current option - """ - values = [a_file.split(SETTING_FILE_EXT)[0] - for a_file in os.listdir(self.path) - if os.path.isfile(os.path.join(self.path, a_file)) and - a_file.endswith(SETTING_FILE_EXT)] - - values.sort() - return values - - def _get_sub_options(self): - """ - Returns a sorted list of sup-options available for the current option - """ - options = [options_dir for options_dir in os.listdir(self.path) - if os.path.isdir(os.path.join(self.path, options_dir)) and - options_dir in self.values] - - options.sort() - return options - - -class OptionsTree(object): - def __init__(self, settings_dir, option): - self.root = None - self.name = option - self.action = option[:-2] if option.endswith('er') else option - self.options_dict = {} - self.root_dir = os.path.join(settings_dir, self.name) - - self.build_tree() - self.init_options_dict(self.root) - - def build_tree(self): - """ - Builds the OptionsTree - """ - self.add_node(self.root_dir, None) - - def add_node(self, path, parent): - """ - Adds OptionNode object to the tree - :param path: Path to option dir - :param parent: Parent option (OptionNode) - """ - node = OptionNode(path, parent) - - if not self.root: - self.root = node - - for child in node.children: - sub_options_dir = os.path.join(node.path, child) - sub_options = [a_dir for a_dir in os.listdir(sub_options_dir) if - os.path.isdir(os.path.join(sub_options_dir, a_dir))] - - for sub_option in sub_options: - self.add_node(os.path.join(sub_options_dir, sub_option), node) - - def init_options_dict(self, node): - """ - Initialize "options_dict" dictionary to store all options and their - valid values - :param node: OptionNode object - """ - if node.option not in self.options_dict: - self.options_dict[node.option] = {} - - if node.parent_value: - self.options_dict[node.option][node.parent_value] = node.values - - if 'ALL' not in self.options_dict[node.option]: - self.options_dict[node.option]['ALL'] = set() - - self.options_dict[node.option]['ALL'].update(node.values) - - for pre_value in node.children: - for child in node.children[pre_value].values(): - self.init_options_dict(child) - - def get_options_ymls(self, options): - ymls = [] - if not options: - return ymls - - keys = options.keys() - keys.sort() - - def step_in(key, node): - keys.remove(key) - if node.option != key.replace("_", "-"): - raise exceptions.IRMissingAncestorException(key) - - ymls.append(os.path.join(node.path, options[key] + ".yml")) - child_keys = [child_key for child_key in keys - if child_key.startswith(key) and - len(child_key.split("_")) == len(key.split("_")) + 1 - ] - - for child_key in child_keys: - step_in(child_key, node.children[options[key]][ - child_key.replace("_", "-")]) - - step_in(keys[0], self.root) - LOG.debug("%s tree settings files:\n%s" % (self.name, ymls)) - - return ymls - - def __str__(self): - return yaml.safe_dump(self.options_dict, default_flow_style=False) - - -def merge_settings(settings, file_path): - LOG.debug("Loading setting file: %s" % file_path) - if not os.path.exists(file_path): - raise exceptions.IRFileNotFoundException(file_path) - - loaded_file = configure.Configuration.from_file(file_path).configure() - settings = settings.merge(loaded_file) - - return settings - - -def generate_settings_file(settings_files, extra_vars): - settings = configure.Configuration.from_dict({}) - - for settings_file in settings_files: - settings = merge_settings(settings, settings_file) - - for extra_var in extra_vars: - if extra_var.startswith('@'): - settings_file = normalize_file(extra_var[1:]) - settings = merge_settings(settings, settings_file) - - else: - if '=' not in extra_var: - raise exceptions.IRExtraVarsException(extra_var) - key, value = extra_var.split("=") - dict_insert(settings, value, *key.split(".")) - - # Dump & load again settings, because 'in_string_lookup' can't work with - # 'Configuration' object. - dumped_settings = yaml.safe_dump(settings, default_flow_style=False) - settings = yaml.safe_load(dumped_settings) - - return settings - - -def in_string_lookup(settings): - """ - Convert strings contain the '!lookup' tag in them and don't - already converted into Lookup objects. - """ - if Lookup.settings is None: - Lookup.settings = settings - - my_iter = settings.iteritems() if isinstance(settings, dict) \ - else enumerate(settings) - - for idx_key, value in my_iter: - if isinstance(value, dict): - in_string_lookup(settings[idx_key]) - elif isinstance(value, list): - in_string_lookup(value) - elif isinstance(value, str): - parser = re.compile('\{\{\s*\!lookup\s*[\w.]*\s*\}\}') - lookups = parser.findall(value) - - if lookups: - settings[idx_key] = Lookup(value) - - -def normalize_file(file_path): - """ - Return a normalized absolutized version of a file - """ - if not os.path.isabs(file_path): - abspath = os.path.abspath(file_path) - LOG.debug( - "Setting the absolute path of \"%s\" to: \"%s\"" - % (file_path, abspath) - ) - file_path = abspath - - if not os.path.exists(file_path): - raise exceptions.IRFileNotFoundException(file_path) - - return file_path - - -def lookup2lookup(settings): - first_dump = True - while True: - if not first_dump: - Lookup.settings = settings - settings = yaml.load(output) - - in_string_lookup(settings) - output = yaml.safe_dump(settings, default_flow_style=False) - - if first_dump: - first_dump = False - continue - - if not cmp(settings, Lookup.settings): - break - - return output - - -def main(): - options_trees = [] - settings_files = [] - settings_dir = validate_settings_dir(kcli_conf.get('DEFAULTS', - 'SETTINGS_DIR')) - - for option in kcli_conf.options('ROOT_OPTS'): - options_trees.append(OptionsTree(settings_dir, option)) - - parser = parse.create_parser(options_trees) - args = parser.parse_args() - - verbose = int(args.verbose) - - if args.verbose == 0: - args.verbose = logging.WARNING - elif args.verbose == 1: - args.verbose = logging.INFO - else: - args.verbose = logging.DEBUG - - LOG.setLevel(args.verbose) - - # settings generation stage - if args.which.lower() != 'execute': - for input_file in args.input: - settings_files.append(normalize_file(input_file)) - - for options_tree in options_trees: - options = {key: value for key, value in vars(args).iteritems() - if value and key.startswith(options_tree.name)} - - settings_files += (options_tree.get_options_ymls(options)) - - LOG.debug("All settings files to be loaded:\n%s" % settings_files) - - settings = generate_settings_file(settings_files, args.extra_vars) - - output = lookup2lookup(settings) - - if args.output_file: - with open(args.output_file, 'w') as output_file: - output_file.write(output) - else: - print output - - exec_playbook = (args.which == 'execute') or \ - (not args.dry_run and args.which in kcli_conf.options( - 'AUTO_EXEC_OPTS')) - - # playbook execution stage - if exec_playbook: - if args.which == 'execute': - execute_args = parser.parse_args() - elif args.which not in PLAYBOOKS: - LOG.debug("No playbook named \"%s\", nothing to execute.\n" - "Please choose from: %s" % (args.which, PLAYBOOKS)) - return - else: - args_list = ["execute"] - if verbose: - args_list.append('-%s' % ('v' * verbose)) - if 'inventory' in args: - inventory = args.inventory - else: - inventory = 'local_hosts' if args.which == 'provision' \ - else 'hosts' - args_list.append('--inventory=%s' % inventory) - args_list.append('--' + args.which) - args_list.append('--collect-logs') - if args.output_file: - LOG.debug('Using the newly created settings file: "%s"' - % args.output_file) - args_list.append('--settings=%s' % args.output_file) - else: - from time import time - - tmp_settings_file = 'kcli_settings_' + str(time()) + \ - SETTING_FILE_EXT - with open(tmp_settings_file, 'w') as output_file: - output_file.write(output) - LOG.debug('Temporary settings file "%s" has been created for ' - 'execution purpose only.' % tmp_settings_file) - args_list.append('--settings=%s' % tmp_settings_file) - - execute_args = parser.parse_args(args_list) - - LOG.debug("execute parser args: %s" % args) - execute_args.func(execute_args) - - if not args.output_file and args.which != 'execute': - LOG.debug('Temporary settings file "%s" has been deleted.' - % tmp_settings_file) - os.remove(tmp_settings_file) - - -if __name__ == '__main__': - main() diff --git a/tools/kcli/kcli/yamls.py b/tools/kcli/kcli/yamls.py deleted file mode 100644 index 4d5298adc..000000000 --- a/tools/kcli/kcli/yamls.py +++ /dev/null @@ -1,91 +0,0 @@ -import string - -from configure import Configuration -import yaml - -from kcli import logger -from kcli import exceptions - -LOG = logger.LOG - - -def random_generator(size=32, chars=string.ascii_lowercase + string.digits): - import random - - return ''.join(random.choice(chars) for _ in range(size)) - - -@Configuration.add_constructor('join') -def _join_constructor(loader, node): - seq = loader.construct_sequence(node) - return ''.join([str(i) for i in seq]) - - -@Configuration.add_constructor('random') -def _random_constructor(loader, node): - """ - usage: - !random - returns a random string of characters - """ - - num_chars = loader.construct_scalar(node) - return random_generator(int(num_chars)) - - -def _limit_chars(_string, length): - length = int(length) - if length < 0: - raise exceptions.IRException('length to crop should be int, not ' + - str(length)) - - return _string[:length] - - -@Configuration.add_constructor('limit_chars') -def _limit_chars_constructor(loader, node): - """ - Usage: - !limit_chars [, ] - Method returns first param cropped to chars. - """ - - params = loader.construct_sequence(node) - if len(params) != 2: - raise exceptions.IRException( - 'limit_chars requires two params: string length') - return _limit_chars(params[0], params[1]) - - -@Configuration.add_constructor('env') -def _env_constructor(loader, node): - """ - usage: - !env - !env [, [default]] - !env [, [default], [length]] - returns value for the environment var-name - default may be specified by passing a second parameter in a list - length is maximum length of output (croped to that length) - """ - - import os - # scalar node or string has no defaults, - # raise IRUndefinedEnvironmentVariableExcption if absent - if isinstance(node, yaml.nodes.ScalarNode): - try: - return os.environ[loader.construct_scalar(node)] - except KeyError: - raise exceptions.IRUndefinedEnvironmentVariableExcption(node.value) - - seq = loader.construct_sequence(node) - var = seq[0] - if len(seq) >= 2: - ret = os.getenv(var, seq[1]) # second item is default val - - # third item is max. length - if len(seq) == 3: - ret = _limit_chars(ret, seq[2]) - return ret - - return os.environ[var] diff --git a/tools/kcli/tests/test_conf.py b/tools/kcli/tests/test_conf.py deleted file mode 100644 index dc70b02dd..000000000 --- a/tools/kcli/tests/test_conf.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -import pytest - -from tests.test_cwd import consts as test_const - -MYCWD = test_const.TESTS_CWD - - -@pytest.fixture() -def our_cwd_setup(request): - """Change cwd to test_cwd dir. Revert to original dir on teardown. """ - - bkp = os.getcwd() - - def our_cwd_teardown(): - os.chdir(bkp) - - request.addfinalizer(our_cwd_teardown) - os.chdir(MYCWD) - - -@pytest.yield_fixture -def os_environ(): - """Backups env var from os.environ and restores it at teardown. """ - - from kcli import conf - - backup_flag = False - if conf.ENV_VAR_NAME in os.environ: - backup_flag = True - backup_value = os.environ.get(conf.ENV_VAR_NAME) - yield os.environ - if backup_flag: - os.environ[conf.ENV_VAR_NAME] = backup_value - - -def test_get_config_dir(our_cwd_setup): - from kcli import conf - conf_file = conf.load_config_file() - assert os.path.abspath(conf_file.get("DEFAULTS", "KHALEESI_DIR")) == \ - os.path.abspath(MYCWD) diff --git a/tools/kcli/tests/test_cwd/consts.py b/tools/kcli/tests/test_cwd/consts.py deleted file mode 100644 index 534bd4a57..000000000 --- a/tools/kcli/tests/test_cwd/consts.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -TESTS_CWD = os.path.dirname(__file__) -SETTINGS_PATH = os.path.join(TESTS_CWD, "settings") -# CONFIG_FILE = os.path.join(TESTS_CWD, conf.KCLI_CONF_FILE) diff --git a/tools/kcli/tests/test_cwd/kcli.cfg b/tools/kcli/tests/test_cwd/kcli.cfg deleted file mode 100644 index 27f3d8ac4..000000000 --- a/tools/kcli/tests/test_cwd/kcli.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULTS] -KHALEESI_DIR = . -SETTINGS_DIR = %(KHALEESI_DIR)s/settings