From 0cbdd8cb1d46ec98d21c5231e6d6e91eea253222 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Sun, 27 Dec 2015 08:56:08 +0000 Subject: [PATCH] added qubesctl qubesformation automation technology --- README.md | 5 +- ansible/action_plugins/commonlib.py | 113 ++++++++++++++++ ansible/action_plugins/qubesformation.py | 157 +++++++++++++++++++++++ ansible/action_plugins/qubessls.py | 43 +++++++ ansible/library/qubesformation.py | 33 +++++ ansible/library/qubessls.py | 30 +++++ 6 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 ansible/action_plugins/commonlib.py create mode 100644 ansible/action_plugins/qubesformation.py create mode 100644 ansible/action_plugins/qubessls.py create mode 100644 ansible/library/qubesformation.py create mode 100644 ansible/library/qubessls.py diff --git a/README.md b/README.md index e93cbc7..b1d14e4 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,10 @@ The software in this kit includes the following: 3. A [set of commands for SaltStack `salt-ssh`](./bin/) that fake SSH and SCP using `bombshell-client` to enable SaltStack management of Qubes OS VMs. -4. A [set of DevOps automation skeletons / examples](./examples/) to get you up and +4. A set of [action plugions for Ansible](./ansible/action_plugins/) that + interface with the new + [Qubes OS 3.1 Salt management stack](https://www.qubes-os.org/news/2015/12/14/mgmt-stack/). +5. A [set of DevOps automation skeletons / examples](./examples/) to get you up and running without having to construct everything yourself. `bombshell-client` and the other programs in this toolkit that diff --git a/ansible/action_plugins/commonlib.py b/ansible/action_plugins/commonlib.py new file mode 100644 index 0000000..c789d1f --- /dev/null +++ b/ansible/action_plugins/commonlib.py @@ -0,0 +1,113 @@ +import collections +import copy + + +def inject_qubes(inject): + myname = inject["inventory_hostname"] + akk = collections.OrderedDict() + all_pcidevs = dict() + for invhostname in inject["groups"]["all"]: + if invhostname == myname: + continue + hostvars = inject["hostvars"][invhostname] + if hostvars.get("qubes", {}).get("dom0_vm") == myname: + akk[invhostname] = copy.deepcopy(hostvars) + for invhostname, hostvars in akk.items(): + qubes = hostvars["qubes"] + dominv = qubes["dom0_vm"] + for dev in qubes.get("pcidevs", []): + if all_pcidevs.get(dev): + assert t in akk, ( + "while processing attribute pcidevs of VM %s: " + "device %s already in use by VM %s" + ) % ( + invhostname, + dev, + all_pcidevs[dev], + ) + all_pcidevs[dev] = invhostname + for vmitem in ["template_vm", "netvm_vm"]: + if vmitem == "template_vm": + if "_template" in hostvars and not "template_vm" in qubes: + qubes["template_vm"] = hostvars["_template"] + if vmitem in qubes: + t = qubes[vmitem] + if t is None or t.lower() == "none": + qubes[vmitem[:-3]] = None + else: + if t.startswith("sibling(") and t.endswith(")"): + t = t[len("sibling("):-1] + dompostfix = dominv.split(".")[1:] + dompostfix = "." + ".".join(dompostfix) if dompostfix else "" + t = t + dompostfix + assert t in akk, ( + "while processing attribute %s of VM %s: " + "%s not found in VMs of %s: %s" + ) % ( + vmitem, + invhostname, + t, + dominv, + ", ".join(akk) + ) + qubes[vmitem[:-3]] = t + del qubes[vmitem] + enabledservices = [] + disabledservices = [] + defaultservices = [] + for service, status in qubes.get("services", {}).items(): + if status == "default" or status == None: + defaultservices.append(service) + elif status == True: + enabledservices.append(service) + elif status == False: + disabledservices.append(service) + else: + assert 0, "while processing service %s of VM %s: invalid value %r" % ( + service, invhostname, status + ) + if enabledservices or disabledservices or defaultservices: + qubes["services"] = collections.OrderedDict() + if enabledservices: + qubes["services"]["enable"] = enabledservices + if disabledservices: + qubes["services"]["disable"] = disabledservices + if defaultservices: + qubes["services"]["disable"] = defaultservices + elif "services" in qubes: + del qubes["services"] + try: + vmtype = qubes["vm_type"] + except KeyError: + assert 0, "while processing attribute %s of VM %s: attribute not set" % ( + "vm_type", invhostname + ) + flags = qubes.get("flags", []) + def add(l, v): + if v not in l: + l.append(v) + if vmtype == "NetVM": + add(flags, "net") + elif vmtype == "StandaloneVM": + add(flags, "standalone") + elif vmtype == "AppVM": + pass + elif vmtype == "ProxyVM": + add(flags, "proxy") + elif vmtype == "TemplateVM": + try: + qubes["source"] = qubes["template"] + del qubes["template"] + except KeyError: + if "source" in qubes: + del qubes["source"] + else: + assert 0, "while processing attribute %s of VM %s: VM type %s unsupported" % ( + "vm_type", invhostname, vmtype + ) + if flags: + qubes["flags"] = flags + else: + if "flags" in qubes: + del qubes["flags"] + return akk diff --git a/ansible/action_plugins/qubesformation.py b/ansible/action_plugins/qubesformation.py new file mode 100644 index 0000000..fc1d9e0 --- /dev/null +++ b/ansible/action_plugins/qubesformation.py @@ -0,0 +1,157 @@ +import collections +import os +import sys +import tempfile +from ansible import errors +from ansible.runner.action_plugins import template + +sys.path.insert(0, os.path.dirname(__file__)) +import commonlib + + +contents = """{{ vms | to_nice_yaml }}""" +topcontents = "{{ saltenv }}:\n '*':\n - {{ recipename }}\n" + + +def generate_datastructure(vms): + dc = collections.OrderedDict + d = dc() + for n, data in vms.items(): + qubes = data['qubes'] + d[n] = dc(qvm=['vm']) + vm = d[n] + qvm = vm['qvm'] + actions = [] + qvm.append(dc(actions=actions)) + + # Setup creation / cloning / existence test. + if 'template' in qubes: + creationparms = [ + {k: v} for k, v in qubes.items() + if k in ['template', 'label', 'mem', 'vcpus', 'flags'] + ] + actions.append('present') + qvm.append({'present': creationparms}) + elif 'source' in qubes: + assert qubes['vm_type'] in ['StandaloneVM', 'TemplateVM'], qubes['vm_type'] + cloneparms = [ + {k: v} for k, v in qubes.items() + if k in ['source'] + ] + actions.append('clone') + qvm.append({'clone': cloneparms}) + else: + actions.append('exists') + qvm.append({'exists': []}) + + # Setup preferences. + ignparm = ['guid', 'services', 'dom0_vm', + 'vm_type', 'flags', 'source'] + ignparm += ['netvm'] if qubes.get('vm_type') == 'NetVM' else [] + ignparm += ['template'] if qubes.get('vm_type') == 'StandaloneVM' else [] + prefsparms = [ + {k: v} for k, v in qubes.items() + if k not in ignparm + ] + if prefsparms: + actions.append('prefs') + qvm.append({'prefs': prefsparms}) + + # Setup services. + if'services' in qubes: + s = qubes['services'] + actions.append('service') + services = [] + qvm.append({'service': services}) + for act in ['enable', 'disable', 'default']: + if act in s: + services.append({act: s[act]}) + + # Setup autostart and execution. + if qubes.get('autostart'): + actions.append('start') + qvm.append({'start': []}) + + # Collate and setup dependencies. + template = qubes.get('template') or qubes.get('source') + netvm = qubes.get('netvm', None) + require = [] + if template: + require.append({'qvm': template}) + if netvm != None: + require.append({'qvm': netvm}) + if require: + qvm.append({'require': require}) + + return d + + +class ActionModule(object): + + TRANSFERS_FILES = True + + def __init__(self, runner): + self.ActionModule = template.ActionModule(runner) + + def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs): + ''' handler for launcher operations ''' + + if module_args: + raise errors.AnsibleError("This module does not accept simple module args: %r" % module_args) + new_inject = dict(inject) + qubesdata = commonlib.inject_qubes(inject) + new_inject["vms"] = generate_datastructure(qubesdata) + with tempfile.NamedTemporaryFile() as x: + x.write(contents) + x.flush() + new_complex_args = dict(complex_args) + new_complex_args["src"] = x.name + retval = self.ActionModule.run( + conn, + tmp, + 'template', + module_args, + inject=new_inject, + complex_args=new_complex_args + ) + if retval.result.get("failed"): + return retval + + with tempfile.NamedTemporaryFile() as y: + y.write(topcontents) + y.flush() + + # Create new tmp path -- the other was blown away. + tmp = self.ActionModule.runner._make_tmp_path(conn) + + new_complex_args = dict(complex_args) + new_complex_args["src"] = y.name + namenoext = os.path.splitext(complex_args["dest"])[0] + dest = namenoext + ".top" + new_complex_args["dest"] = dest + new_inject["recipename"] = os.path.basename(namenoext) + new_inject["saltenv"] = "user" if "user_salt" in dest.split(os.sep) else "base" + retval2 = self.ActionModule.run( + conn, + tmp, + 'template', + module_args, + inject=new_inject, + complex_args=new_complex_args + ) + if retval2.result.get("failed"): + return retval2 + if not retval.result['changed'] and not retval2.result['changed']: + for c in ('path', 'size'): + retval.result[c] = [x.result[c] for x in (retval, retval2) if c in x.result] + return retval + elif retval.result['changed'] and retval2.result['changed']: + for c in ('src', 'checksum', 'size', 'state', 'changed', 'md5sum', 'dest'): + retval.result[c] = [x.result[c] for x in (retval, retval2) if c in x.result] + return retval + elif retval.result['changed']: + return retval + elif retval2.result['changed']: + return retval2 + else: + assert 0, "not reached" diff --git a/ansible/action_plugins/qubessls.py b/ansible/action_plugins/qubessls.py new file mode 100644 index 0000000..18ac0b2 --- /dev/null +++ b/ansible/action_plugins/qubessls.py @@ -0,0 +1,43 @@ +import pipes +from ansible import errors + + +class ActionModule(object): + + TRANSFERS_FILES = True + + def __init__(self, runner): + self.runner = runner + + def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs): + ''' handler for launcher operations ''' + + if module_args: + raise errors.AnsibleError("This module does not accept simple module args: %r" % module_args) + + cmd = ["qubesctl"] + cmd.append('state.sls') + cmd.append(complex_args['sls']) + if 'env' in complex_args: + cmd.append("saltenv=%s" % (complex_args['env'],)) + if self.runner.noop_on_check(inject): + cmd.append("test=True") + + module_args = " ".join(pipes.quote(s) for s in cmd) + + retval = self.runner._execute_module( + conn, + tmp, + 'command', + module_args, + inject=inject, + complex_args=complex_args + ) + changeline = retval.result['stdout'].splitlines()[-4] + if self.runner.noop_on_check(inject): + numtasks = changeline.split()[1] + numunchanged = changeline.split("=")[1].split(')')[0] + retval.result['changed'] = numtasks != numunchanged + else: + retval.result['changed'] = 'changed=' in changeline + return retval diff --git a/ansible/library/qubesformation.py b/ansible/library/qubesformation.py new file mode 100644 index 0000000..07ec82b --- /dev/null +++ b/ansible/library/qubesformation.py @@ -0,0 +1,33 @@ +DOCUMENTATION = """ +--- +module: qubesformation +author: Manuel Amador (Rudd-O) +short_description: provision VMs via a generated Qubes Salt Management recipe. +version_added: 0.0 +description: + - 'This module lets you provision VMs and enforce VM settings on a + collection of VMs derived from your Ansible inventory. Note that + this module does not accept simple arguments -- you must specify + complex arguments in the form of a dictionary below the module name.' +options: + dest: + required: true + description: + - Where to deposit the recipe -- usually a path like + `/srv/user_salt/.sls`). Will create + two files: + * The file you specified in `description`. + * An additional file with a .top extension instead of the + original extension of the file you specified. + others: + description: + - All arguments accepted by the M(template) module also work here, + except for `src` and `content`. + required: false +""" + +EXAMPLES = r""" +# Would create `/srv/user_salt/formation.sls` and `/srv/user_salt/formation.top`. +- qubesformation: + dest: /srv/user_salt/formation.sls +""" diff --git a/ansible/library/qubessls.py b/ansible/library/qubessls.py new file mode 100644 index 0000000..8c29e2c --- /dev/null +++ b/ansible/library/qubessls.py @@ -0,0 +1,30 @@ +DOCUMENTATION = """ +--- +module: qubesformation +author: Manuel Amador (Rudd-O) +short_description: provision VMs via a generated Qubes Salt Management recipe. +version_added: 0.0 +description: + - 'This module lets you provision VMs and enforce VM settings on a + collection of VMs derived from your Ansible inventory. Note that + this module does not accept simple arguments -- you must specify + complex arguments in the form of a dictionary below the module name.' +options: + sls: + required: true + description: + - The name of the recipe (SLS and top files), as stored in either `/srv/salt` for + recipes in the `base` Salt enviornment, or `/srv/user_salt` for those in the `user` + environment. + env: + required: false + description: + - Which Salt environment to load the SLS from (default `base`, you can specify `user`). +""" + +EXAMPLES = r""" +# Would realize `/srv/salt/formation.sls` and `/srv/salt/formation.top`. +- qubessls: + env: base + sls: formation +"""