added qubesctl qubesformation automation technology

README.md

@@ -19,7 +19,10 @@ The software in this kit includes the following:
 3. A [set of commands for SaltStack `salt-ssh`](./bin/) that fake SSH
    and SCP using `bombshell-client` to enable SaltStack management
    of Qubes OS VMs.
-4. A [set of DevOps automation skeletons / examples](./examples/) to get you up and
+4. A set of [action plugions for Ansible](./ansible/action_plugins/) that
+   interface with the new
+   [Qubes OS 3.1 Salt management stack](https://www.qubes-os.org/news/2015/12/14/mgmt-stack/).
+5. A [set of DevOps automation skeletons / examples](./examples/) to get you up and
    running without having to construct everything yourself.
 
 `bombshell-client` and the other programs in this toolkit that

ansible/action_plugins/commonlib.py

@@ -0,0 +1,113 @@
+import collections
+import copy
+
+
+def inject_qubes(inject):
+    myname = inject["inventory_hostname"]
+    akk = collections.OrderedDict()
+    all_pcidevs = dict()
+    for invhostname in inject["groups"]["all"]:
+        if invhostname == myname:
+            continue
+        hostvars = inject["hostvars"][invhostname]
+        if hostvars.get("qubes", {}).get("dom0_vm") == myname:
+            akk[invhostname] = copy.deepcopy(hostvars)
+    for invhostname, hostvars in akk.items():
+        qubes = hostvars["qubes"]
+        dominv = qubes["dom0_vm"]
+        for dev in qubes.get("pcidevs", []):
+            if all_pcidevs.get(dev):
+                assert t in akk, (
+                    "while processing attribute pcidevs of VM %s: "
+                    "device %s already in use by VM %s"
+                ) % (
+                    invhostname,
+                    dev,
+                    all_pcidevs[dev],
+                )
+            all_pcidevs[dev] = invhostname
+        for vmitem in ["template_vm", "netvm_vm"]:
+            if vmitem == "template_vm":
+                if "_template" in hostvars and not "template_vm" in qubes:
+                    qubes["template_vm"] = hostvars["_template"]
+            if vmitem in qubes:
+                t = qubes[vmitem]
+                if t is None or t.lower() == "none":
+                    qubes[vmitem[:-3]] = None
+                else:
+                    if t.startswith("sibling(") and t.endswith(")"):
+                        t = t[len("sibling("):-1]
+                        dompostfix = dominv.split(".")[1:]
+                        dompostfix = "." + ".".join(dompostfix) if dompostfix else ""
+                        t = t + dompostfix
+                    assert t in akk, (
+                        "while processing attribute %s of VM %s: "
+                        "%s not found in VMs of %s: %s"
+                    ) % (
+                        vmitem,
+                        invhostname,
+                        t,
+                        dominv,
+                        ", ".join(akk)
+                    )
+                    qubes[vmitem[:-3]] = t
+                del qubes[vmitem]
+        enabledservices = []
+        disabledservices = []
+        defaultservices = []
+        for service, status in qubes.get("services", {}).items():
+            if status == "default" or status == None:
+                defaultservices.append(service)
+            elif status == True:
+                enabledservices.append(service)
+            elif status == False:
+                disabledservices.append(service)
+            else:
+                assert 0, "while processing service %s of VM %s: invalid value %r" % (
+                    service, invhostname, status
+                )
+        if enabledservices or disabledservices or defaultservices:
+            qubes["services"] = collections.OrderedDict()
+            if enabledservices:
+                qubes["services"]["enable"] = enabledservices
+            if disabledservices:
+                qubes["services"]["disable"] = disabledservices
+            if defaultservices:
+                qubes["services"]["disable"] = defaultservices
+        elif "services" in qubes:
+            del qubes["services"]
+        try:
+            vmtype = qubes["vm_type"]
+        except KeyError:
+            assert 0, "while processing attribute %s of VM %s: attribute not set" % (
+                "vm_type", invhostname
+            )
+        flags = qubes.get("flags", [])
+        def add(l, v):
+            if v not in l:
+                l.append(v)
+        if vmtype == "NetVM":
+            add(flags, "net")
+        elif vmtype == "StandaloneVM":
+            add(flags, "standalone")
+        elif vmtype == "AppVM":
+            pass
+        elif vmtype == "ProxyVM":
+            add(flags, "proxy")
+        elif vmtype == "TemplateVM":
+            try:
+                qubes["source"] = qubes["template"]
+                del qubes["template"]
+            except KeyError:
+                if "source" in qubes:
+                    del qubes["source"]
+        else:
+            assert 0, "while processing attribute %s of VM %s: VM type %s unsupported" % (
+                "vm_type", invhostname, vmtype
+            )
+        if flags:
+            qubes["flags"] = flags
+        else:
+            if "flags" in qubes:
+                del qubes["flags"]
+    return akk

ansible/action_plugins/qubesformation.py

@@ -0,0 +1,157 @@
+import collections
+import os
+import sys
+import tempfile
+from ansible import errors
+from ansible.runner.action_plugins import template
+
+sys.path.insert(0, os.path.dirname(__file__))
+import commonlib
+
+
+contents = """{{ vms | to_nice_yaml }}"""
+topcontents = "{{ saltenv }}:\n  '*':\n  - {{ recipename }}\n"
+
+
+def generate_datastructure(vms):
+    dc = collections.OrderedDict
+    d = dc()
+    for n, data in vms.items():
+        qubes = data['qubes']
+        d[n] = dc(qvm=['vm'])
+        vm = d[n]
+        qvm = vm['qvm']
+        actions = []
+        qvm.append(dc(actions=actions))
+
+        # Setup creation / cloning / existence test.
+        if 'template' in qubes:
+            creationparms = [
+                {k: v} for k, v in qubes.items()
+                if k in ['template', 'label', 'mem', 'vcpus', 'flags']
+            ]
+            actions.append('present')
+            qvm.append({'present': creationparms})
+        elif 'source' in qubes:
+            assert qubes['vm_type'] in ['StandaloneVM', 'TemplateVM'], qubes['vm_type']
+            cloneparms = [
+                {k: v} for k, v in qubes.items()
+                if k in ['source']
+            ]
+            actions.append('clone')
+            qvm.append({'clone': cloneparms})
+        else:
+            actions.append('exists')
+            qvm.append({'exists': []})
+
+        # Setup preferences.
+        ignparm = ['guid', 'services', 'dom0_vm',
+                   'vm_type', 'flags', 'source']
+        ignparm += ['netvm'] if qubes.get('vm_type') == 'NetVM' else []
+        ignparm += ['template'] if qubes.get('vm_type') == 'StandaloneVM' else []
+        prefsparms = [
+            {k: v} for k, v in qubes.items()
+            if k not in ignparm
+        ]
+        if prefsparms:
+            actions.append('prefs')
+            qvm.append({'prefs': prefsparms})
+
+        # Setup services.
+        if'services' in qubes:
+            s = qubes['services']
+            actions.append('service')
+            services = []
+            qvm.append({'service': services})
+            for act in ['enable', 'disable', 'default']:
+                if act in s:
+                    services.append({act: s[act]})
+
+        # Setup autostart and execution.
+        if qubes.get('autostart'):
+            actions.append('start')
+            qvm.append({'start': []})
+
+        # Collate and setup dependencies.
+        template = qubes.get('template') or qubes.get('source')
+        netvm = qubes.get('netvm', None)
+        require = []
+        if template:
+            require.append({'qvm': template})
+        if netvm != None:
+            require.append({'qvm': netvm})
+        if require:
+            qvm.append({'require': require})
+
+    return d
+
+
+class ActionModule(object):
+
+    TRANSFERS_FILES = True
+
+    def __init__(self, runner):
+        self.ActionModule = template.ActionModule(runner)
+
+    def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs):
+        ''' handler for launcher operations '''
+
+        if module_args:
+            raise errors.AnsibleError("This module does not accept simple module args: %r" % module_args)
+        new_inject = dict(inject)
+        qubesdata = commonlib.inject_qubes(inject)
+        new_inject["vms"] = generate_datastructure(qubesdata)
+        with tempfile.NamedTemporaryFile() as x:
+            x.write(contents)
+            x.flush()
+            new_complex_args = dict(complex_args)
+            new_complex_args["src"] = x.name
+            retval = self.ActionModule.run(
+                conn,
+                tmp,
+                'template',
+                module_args,
+                inject=new_inject,
+                complex_args=new_complex_args
+            )
+            if retval.result.get("failed"):
+                return retval
+
+            with tempfile.NamedTemporaryFile() as y:
+                y.write(topcontents)
+                y.flush()
+
+                # Create new tmp path -- the other was blown away.
+                tmp = self.ActionModule.runner._make_tmp_path(conn)
+
+                new_complex_args = dict(complex_args)
+                new_complex_args["src"] = y.name
+                namenoext = os.path.splitext(complex_args["dest"])[0]
+                dest = namenoext + ".top"
+                new_complex_args["dest"] = dest
+                new_inject["recipename"] = os.path.basename(namenoext)
+                new_inject["saltenv"] = "user" if "user_salt" in dest.split(os.sep) else "base"
+                retval2 = self.ActionModule.run(
+                    conn,
+                    tmp,
+                    'template',
+                    module_args,
+                    inject=new_inject,
+                    complex_args=new_complex_args
+                )
+                if retval2.result.get("failed"):
+                    return retval2
+            if not retval.result['changed'] and not retval2.result['changed']:
+                for c in ('path', 'size'):
+                    retval.result[c] = [x.result[c] for x in (retval, retval2) if c in x.result]
+                return retval
+            elif retval.result['changed'] and retval2.result['changed']:
+                for c in ('src', 'checksum', 'size', 'state', 'changed', 'md5sum', 'dest'):
+                    retval.result[c] = [x.result[c] for x in (retval, retval2) if c in x.result]
+                return retval
+            elif retval.result['changed']:
+                return retval
+            elif retval2.result['changed']:
+                return retval2
+            else:
+                assert 0, "not reached"

ansible/action_plugins/qubessls.py

@@ -0,0 +1,43 @@
+import pipes
+from ansible import errors
+
+
+class ActionModule(object):
+
+    TRANSFERS_FILES = True
+
+    def __init__(self, runner):
+        self.runner = runner
+
+    def run(self, conn, tmp, module_name, module_args, inject, complex_args=None, **kwargs):
+        ''' handler for launcher operations '''
+
+        if module_args:
+            raise errors.AnsibleError("This module does not accept simple module args: %r" % module_args)
+
+        cmd = ["qubesctl"]
+        cmd.append('state.sls')
+        cmd.append(complex_args['sls'])
+        if 'env' in complex_args:
+            cmd.append("saltenv=%s" % (complex_args['env'],))
+        if self.runner.noop_on_check(inject):
+            cmd.append("test=True")
+
+        module_args = " ".join(pipes.quote(s) for s in cmd)
+
+        retval = self.runner._execute_module(
+            conn,
+            tmp,
+            'command',
+            module_args,
+            inject=inject,
+            complex_args=complex_args
+        )
+        changeline = retval.result['stdout'].splitlines()[-4]
+        if self.runner.noop_on_check(inject):
+            numtasks = changeline.split()[1]
+            numunchanged = changeline.split("=")[1].split(')')[0]
+            retval.result['changed'] = numtasks != numunchanged
+        else:
+            retval.result['changed'] = 'changed=' in changeline
+        return retval

ansible/library/qubesformation.py

@@ -0,0 +1,33 @@
+DOCUMENTATION = """
+---
+module: qubesformation
+author: Manuel Amador (Rudd-O) <rudd-o@rudd-o.com>
+short_description: provision VMs via a generated Qubes Salt Management recipe.
+version_added: 0.0
+description:
+  - 'This module lets you provision VMs and enforce VM settings on a
+    collection of VMs derived from your Ansible inventory.  Note that
+    this module does not accept simple arguments -- you must specify
+    complex arguments in the form of a dictionary below the module name.'
+options:
+  dest:
+    required: true
+    description:
+      - Where to deposit the recipe -- usually a path like
+        `/srv/user_salt/<formation name>.sls`).  Will create
+        two files:
+        * The file you specified in `description`.
+        * An additional file with a .top extension instead of the
+          original extension of the file you specified.
+  others:
+    description:
+      - All arguments accepted by the M(template) module also work here,
+        except for `src` and `content`.
+    required: false
+"""
+
+EXAMPLES = r"""
+# Would create `/srv/user_salt/formation.sls` and `/srv/user_salt/formation.top`.
+- qubesformation:
+    dest: /srv/user_salt/formation.sls
+"""

ansible/library/qubessls.py

@@ -0,0 +1,30 @@
+DOCUMENTATION = """
+---
+module: qubesformation
+author: Manuel Amador (Rudd-O) <rudd-o@rudd-o.com>
+short_description: provision VMs via a generated Qubes Salt Management recipe.
+version_added: 0.0
+description:
+  - 'This module lets you provision VMs and enforce VM settings on a
+    collection of VMs derived from your Ansible inventory.  Note that
+    this module does not accept simple arguments -- you must specify
+    complex arguments in the form of a dictionary below the module name.'
+options:
+  sls:
+    required: true
+    description:
+      - The name of the recipe (SLS and top files), as stored in either `/srv/salt` for
+        recipes in the `base` Salt enviornment, or `/srv/user_salt` for those in the `user`
+        environment.
+  env:
+    required: false
+    description:
+      - Which Salt environment to load the SLS from (default `base`, you can specify `user`).
+"""
+
+EXAMPLES = r"""
+# Would realize `/srv/salt/formation.sls` and `/srv/salt/formation.top`.
+- qubessls:
+    env: base
+    sls: formation
+"""