Merge pull request #40 from ryran/whatness-medimn

implement #38 & #39

README.md

@@ -15,6 +15,9 @@ upvm might not be for you if:
 - you already have all the images you need in a RHEV environment
 - you want to spin up new machines in an OpenStack cloud
 
+#### I don't like image files! LVM is the best!
+I used to use LVM for all my VMs too ... so upvm can use `virt-builder` to generate an image and then write that image out to a block device you specify with the `--primary-blockdev` option. This optional feature of course requires some sudo access (to run [dd-helper.py](https://github.com/ryran/upvm/blob/master/dd-helper.py) as root) which upvm will walk you through the first time you do it.
+
 ## Install
 
 #### yum/dnf (RPM) install in RHEL 7.2+ or Fedora 22+:
@@ -51,20 +54,20 @@ Are you annoyed with always having to open the `virt-manager` GUI to do stuff wi
 ## Help page
 
 ```
-$ upvm --help
 usage: upvm [--loglevel {debug,info,error}] [--build-image-only] [--nocolor]
             [--noconsole] [--cachedir VBCACHEDIR] [-h] [--help] [-l]
             [--arch {x86_64,i386,i686,ia64,armv71,ppc,ppc64,ppc64le,aarch64,sparc64,s390,s390x}]
             [--info] [-n VMNAME] [--os-variant OS] [--root-password SELECTOR]
             [--dnsdomain DOMAIN] [--hostname HOSTNAME | --hostname-prompt]
             [--no-dhcphostname] [--img-dir DIR] [--img-size SIZE]
             [--img-format {auto,raw,qcow2}] [--timezone TIMEZONE]
-            [--rhsm-key ORGID:KEY] [--ssh-pubkey FILE] [--upload FILE:DEST]
-            [--run SCRIPT] [--run-command CMD+ARGS] [--firstboot SCRIPT]
+            [--ssh-pubkey FILE] [--upload FILE:DEST] [--run SCRIPT]
+            [--run-command CMD+ARGS] [--firstboot SCRIPT]
             [--firstboot-command CMD+ARGS] [--install PKG,PKG,@GROUP...]
             [--firstboot-install PKG,PKG,@GROUP...] [--selinux-relabel]
-            [--vbuilder-arg ARG] [-m OPTIONS] [-c OPTIONS] [-d OPTIONS]
-            [-w OPTIONS] [--boot OPTIONS | --uefi] [--vinstall-arg ARG]
+            [--vbuilder-arg ARG] [--primary-blockdev BLOCKDEV] [-m OPTIONS]
+            [-c OPTIONS] [-d OPTIONS] [-w OPTIONS] [--boot OPTIONS | --uefi]
+            [--vinstall-arg ARG]
             [TEMPLATE]
 
 Leverage virt-builder & virt-install to spin up new VMs with ease
@@ -149,17 +152,6 @@ TOTALLY OPTIONAL OS-LEVEL OPTIONS:
                         syntax, i.e., paths rooted in /usr/share/zoneinfo,
                         e.g.: 'Europe/London' or 'America/Los_Angeles' or
                         'Asia/Calcutta')
-  --rhsm-key ORGID:KEY  With or without this option, upvm creates a
-                        /root/register script inside the guest which can be
-                        used to interactively register to RHN Classic or RHSM;
-                        this option edits that script to include an RHSM
-                        activation key created by you at
-                        access.redhat.com/management/activation_keys -- i.e.,
-                        so the script can be run without requiring a
-                        user/password (to use this option, specify both the
-                        organization ID and the activation key, e.g.,
-                        '0123456789:my_activation_key'); IMPORTANT NOTE: this
-                        does not run the /root/register script for you
   --ssh-pubkey FILE     Inject specific ssh pubkeys (option may be used more
                         than once) into the /root/.ssh/authorized_keys file of
                         the guest (default: contents of local
@@ -201,10 +193,18 @@ TOTALLY OPTIONAL OS-LEVEL OPTIONS:
                         in=/localpath:/mnt/remote')
 
 TOTALLY OPTIONAL HARDWARE-LEVEL (VM) OPTIONS:
-  Guest hardware configured by virt-install (after image is built). Each option
-  corresponds with virt-install option of same name. More configurability is
+  Guest hardware configured by virt-install (after image is built). Most options
+  correspond with virt-install options of same name. More configurability is
   possible than described. See virt-install man page.
 
+  --primary-blockdev BLOCKDEV
+                        To use this option, specify a BLOCKDEV like
+                        '/dev/sdb2' or '/dev/vg/logvol' and then in that case,
+                        the image generated by virt-builder [using the above
+                        options] will be written to BLOCKDEV and then deleted;
+                        the first time you try this as a non-root user, upvm
+                        will exit with a warning explaining how to add the
+                        appropriate sudo access to the dd helper-wrapper
   -m, --memory OPTIONS  If this option is omitted, the guest will be defined
                         with a RAM allocation of 1024 MiB; in its simplest
                         form OPTIONS can be the amount of RAM in MiB; more
@@ -262,6 +262,6 @@ ABOUT CONFIG FILES:
   override config file values which override defaults.
 
 VERSION:
-  upvm v0.10.5 last mod 2016/02/14
+  upvm v0.10.6 last mod 2016/02/16
   See <http://github.com/ryran/upvm> to report bugs or RFEs
 ```

dd-helper.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# Copyright 2016 upvm Contributors (see CONTRIBUTORS.md file in source)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#-------------------------------------------------------------------------------
+
+import subprocess
+from sys import argv, exit
+from os import devnull
+outfile, infile = argv[1], argv[2]
+null = open(devnull, 'w')
+rc = subprocess.call(['findmnt', '-no', 'target', outfile], stdout=null, stderr=null)
+if rc == 0:
+    # If findmnt reports outfile is mounted
+    exit(2)
+lsof = subprocess.Popen(['nice', 'lsof', '-tlS', outfile], stdout=subprocess.PIPE, stderr=null)
+output = lsof.communicate()[0]
+if output:
+    # If lsof reports anything using our outfile
+    exit(3)
+null.close()
+dd = ['nice', 'dd', 'of={}'.format(outfile), 'if={}'.format(infile), 'bs=1M']
+exit(subprocess.call(dd))

modules/argparser.py

@@ -190,9 +190,9 @@ def parse():
     grpAA.add_argument(
         '--timezone',
         help="Set system timezone inside the OS (use traditional syntax, i.e., paths rooted in /usr/share/zoneinfo, e.g.: 'Europe/London' or 'America/Los_Angeles' or 'Asia/Calcutta')")
-    grpAA.add_argument(
-        '--rhsm-key', metavar='ORGID:KEY',
-        help="With or without this option, {} creates a /root/register script inside the guest which can be used to interactively register to RHN Classic or RHSM; this option edits that script to include an RHSM activation key created by you at access.redhat.com/management/activation_keys -- i.e., so the script can be run without requiring a user/password (to use this option, specify both the organization ID and the activation key, e.g., '0123456789:my_activation_key'); IMPORTANT NOTE: this does not run the /root/register script for you".format(cfg.prog))
+    # grpAA.add_argument(
+    #     '--rhsm-key', metavar='ORGID:KEY',
+    #     help="With or without this option, {} creates a /root/register script inside the guest which can be used to interactively register to RHN Classic or RHSM; this option edits that script to include an RHSM activation key created by you at access.redhat.com/management/activation_keys -- i.e., so the script can be run without requiring a user/password (to use this option, specify both the organization ID and the activation key, e.g., '0123456789:my_activation_key'); IMPORTANT NOTE: this does not run the /root/register script for you".format(cfg.prog))
     grpAA.add_argument(
         '--ssh-pubkey', metavar='FILE', action='append',
         default=['~/.ssh/authorized_keys', '~/.ssh/id_[dr]sa.pub', '~/.ssh/id_ecdsa.pub', '~/.ssh/id_ed25519.pub'],
@@ -227,7 +227,10 @@ def parse():
     # Optional virt-install options
     grpBB = p.add_argument_group(
         'TOTALLY OPTIONAL HARDWARE-LEVEL (VM) OPTIONS',
-        description="Guest hardware configured by virt-install (after image is built). Each option\ncorresponds with virt-install option of same name. More configurability is\npossible than described. See virt-install man page.")
+        description="Guest hardware configured by virt-install (after image is built). Most options\ncorrespond with virt-install options of same name. More configurability is\npossible than described. See virt-install man page.")
+    grpBB.add_argument(
+        '--primary-blockdev', metavar='BLOCKDEV',
+        help="To use this option, specify a BLOCKDEV like '/dev/sdb2' or '/dev/vg/logvol' and then in that case, the image generated by virt-builder [using the above options] will be written to BLOCKDEV and then deleted; the first time you try this as a non-root user, {} will exit with a warning explaining how to add the appropriate sudo access to the dd helper-wrapper".format(cfg.prog))
     grpBB.add_argument(
         '-m', '--memory', metavar='OPTIONS', default=1024,
         help="If this option is omitted, the guest will be defined with a RAM allocation of 1024 MiB; in its simplest form OPTIONS can be the amount of RAM in MiB; more complicated example: '-m 512,maxmemory=4096' (which would set the default to 512 MiB but allow raising up to 4 GiB without powering off)")

modules/blockdevimager.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 upvm Contributors (see CONTRIBUTORS.md file in source)
+# License: Apache License 2.0 (see LICENSE file in source)
+
+# Modules from standard library
+from __future__ import print_function
+import subprocess
+import os
+from sys import exit
+
+# Custom modules
+from . import string_ops as c
+from . import cfg
+
+def write_and_cleanup_image():
+    blockdev, imgfile = cfg.opts.primary_blockdev, cfg.opts.outFile
+    cmd = ['sudo', cfg.blockdevHelper, blockdev, imgfile]
+    print(c.GREEN("Starting write of image file '{}' to blockdev '{}'".format(imgfile, blockdev)))
+    print(c.green("(This could take a while)"))
+    rc = subprocess.call(cmd)
+    if rc == 0:
+        c.verbose("Successfully wrote intermediate image file to blockdev")
+        cfg.cleanup_imagefile()
+    else:
+        print(c.RED("Error writing image file '{}' to block device '{}'".format(imgfile, blockdev)))
+        cfg.cleanup_imagefile()
+        exit(1)

modules/builder.py

@@ -71,6 +71,7 @@ def build():
     initialize_libvirt_qemu_session()
     o = cfg.opts
     cmd = [
+        'nice',
         'virt-builder', o.templateName,
         '--output', o.outFile
         ]

modules/cfg.py

@@ -13,12 +13,15 @@
 from . import string_ops as c
 
 # Version info
-__version__ = '0.10.5'
-__date__    = '2016/02/14'
+__version__ = '0.10.6'
+__date__    = '2016/02/16'
 
 # All references to program name should use this
 prog = 'upvm'
 
+# Path to helper blockdev script
+blockdevHelper = '/usr/libexec/{}-dd-helper.py'.format(prog)
+
 # Set config file locations used by configargparse
 cfgfileExample = '/usr/share/{}/example.conf'.format(prog)
 cfgfileSystem = '/etc/{}.conf'.format(prog)
@@ -74,3 +77,15 @@ def prompt_for_template_and_exit():
         exit()
     else:
         exit(1)
+
+def cleanup_imagefile():
+    try:
+        imgfile = opts.outFile
+    except:
+        return
+    try:
+        os.remove(imgfile)
+    except:
+        print(c.yellow("[Cleanup] Deleting image file '{}' failed; you can safely delete it yourself".format(imgfile)))
+    else:
+        c.verbose("[Cleanup] Deleted image file '{}'".format(imgfile))

modules/finalprompter.py

@@ -96,7 +96,10 @@ def check_prompt_hostname():
             cfg.opts.hostname = _default_name
 
 def checkset_img_format():
-    if cfg.opts.img_format in 'auto':
+    if cfg.opts.primary_blockdev:
+        c.verbose("Output image format unconditionally set to raw due to --primary-blockdev option")
+        cfg.opts.img_format = 'raw'
+    elif cfg.opts.img_format in 'auto':
         cfg.opts.img_format = cfg.templateInfo.get('format')
         if not cfg.opts.img_format or cfg.opts.img_format == 'qcow2' or cfg.opts.img_format == 'raw':
             c.verbose("Unable to determine native format of chosen template")

modules/installer.py

@@ -6,6 +6,7 @@
 from __future__ import print_function
 import subprocess
 import os
+from sys import exit
 
 # Custom modules
 from . import cfg
@@ -34,10 +35,12 @@ def install():
         str(o.memory),
         '--vcpus',
         str(o.vcpus),
-        '--disk',
-        '{},format={}'.format(o.outFile, o.img_format),
         '--noautoconsole',
         ]
+    if o.primary_blockdev:
+        cmd.extend(['--disk', o.primary_blockdev])
+    else:
+        cmd.extend(['--disk', '{},format={}'.format(o.outFile, o.img_format)])
     if o.os_variant:
         cmd.extend(['--os-variant', o.os_variant])
     if o.disk:
@@ -58,4 +61,8 @@ def install():
             cmd.append(a)
     c.verbose("Starting virt-install")
     c.debug("Executing:\n    {}\n".format(" \ \n    ".join(cmd)))
-    subprocess.check_call(cmd)
+    rc = subprocess.call(cmd)
+    if rc != 0:
+        print(c.RED("Error: virt-install command exited with non-zero status (see above)"))
+        cfg.cleanup_imagefile()
+        exit(rc)

modules/sysvalidator.py

@@ -10,6 +10,7 @@
 from sys import exit
 import pwd, grp
 import json
+from stat import S_ISBLK
 
 # Custom modules
 from . import cfg
@@ -38,6 +39,27 @@ def call(cmd, showStdout=False, showStderr=False, shell=False):
     null.close()
     return ret(rc)
 
+def convert_size_string_to_bytes(size):
+    unit = size[-1:]
+    sz = float(size[:-1])
+    if unit == 'b':
+        return sz
+    elif unit == 'K':
+        return sz * 1024**1
+    elif unit == 'M':
+        return sz * 1024**2
+    elif unit == 'G':
+        return sz * 1024**3
+    elif unit == 'T':
+        return sz * 1024**4
+    elif unit == 'P':
+        return sz * 1024**5
+    elif unit == 'E':
+        return sz * 1024**6
+    else:
+        print(c.RED("Invalid size specified (use e.g., '256M', '10G')"))
+        exit(1)
+
 def print_template_metadata(template):
     print("{0:17}: {1}".format("template", template['os-version']))
     for key in template:
@@ -151,8 +173,57 @@ def try_capture_existing_vm_names():
         print(c.RED("\nUnknown error executing '{}'".format(" ".join(cmd))))
         exit(1)
 
+def validate_image_size():
+    if cfg.opts.img_size:
+        return convert_size_string_to_bytes(cfg.opts.img_size)
+    else:
+        return float(cfg.templateInfo['size'])
+
+def validate_blockdev(imgsize):
+    blockdev = cfg.opts.primary_blockdev
+    try:
+        mode = os.stat(blockdev).st_mode
+    except OSError, e:
+        print(c.RED(e))
+        print("It looks like you passed the wrong thing to the --primary-blockdev option")
+        exit(1)
+    if not S_ISBLK(mode):
+        print(c.RED("File passed as an arg to the --primary-blockdev option is not a block device"))
+        print("This option is meant to be used to create a VM whose primary storage is a block device like /dev/sdb2")
+        exit(1)
+    cmd = ['lsblk', '-no', 'size', blockdev]
+    c.debug("Executing: {}".format(" ".join(cmd)))
+    try:
+        blksize = subprocess.check_output(cmd).strip()
+    except:
+        print(c.RED("Unexpected error using lsblk to check size of --primary-blockdev device"))
+        print("This shouldn't happen ... unless you somehow don't have util-linux and the lsblk command")
+        exit(1)
+    blksize = convert_size_string_to_bytes(blksize)
+    if imgsize > blksize:
+        print(c.YELLOW("Aborting because block device '{}' is smaller than projected image size".format(blockdev)))
+        print("Block device size: {} MiB".format(int(blksize/1024/1024)))
+        print("Image file size:   {} MiB".format(int(imgsize/1024/1024)))
+        exit(1)
+
+def check_sudo_blockdevHelper_rights():
+    blockdev, imgdir = cfg.opts.primary_blockdev, cfg.opts.img_dir
+    cmd = ['sudo', '-l', cfg.blockdevHelper, blockdev, imgdir]
+    if not call(cmd):
+        print(c.YELLOW("Need root privileges to write image to {}".format(blockdev)))
+        print(c.yellow(
+            "You've requested {} write a VM disk image out to a block device but you\n"
+            "are not root. To do this, give yourself the appropriate sudo access by\n"
+            "running the following command as root:".format(cfg.prog)))
+        print(c.green("  echo '{} ALL=(ALL) NOPASSWD: {}' >>/etc/sudoers.d/{}\n".format(myUser, cfg.blockdevHelper, cfg.prog)))
+        print(c.yellow(
+            "If you want to restrict your user to only have write access to a specific\n"
+            "LVM volume group, you could instead do something like:"))
+        print(c.green("  echo '{} ALL=(ALL) NOPASSWD: {} /dev/volgroup*' >>/etc/sudoers.d/{}\n".format(myUser, cfg.blockdevHelper, cfg.prog)))
+        exit(1)
+
 def check_system_config():
-    check_virtbuilder_version()
+    ##check_virtbuilder_version()
     if_no_template_requested_then_print_vblist_and_quit()
     check_if_requested_template_exists()
     isolate_metadata_for_chosen_vb_template()
@@ -164,3 +235,7 @@ def check_system_config():
     ##check_user_in_libvirt_group()
     check_for_writable_imgdir()
     try_capture_existing_vm_names()
+    imgsize = validate_image_size()
+    if cfg.opts.primary_blockdev:
+        validate_blockdev(imgsize)
+        check_sudo_blockdevHelper_rights()

upvm.py

@@ -87,6 +87,10 @@ def main():
     # Quit if requested
     if cfg.opts.build_image_only:
         exit()
+    # Write image to blockdevice if requested
+    if cfg.opts.primary_blockdev:
+        from modules import blockdevimager
+        blockdevimager.write_and_cleanup_image()
     # Launch virt-install
     from modules import installer
     installer.install()
@@ -101,4 +105,5 @@ def main():
         main()
     except KeyboardInterrupt:
         print("\nReceived KeyboardInterrupt. Exiting.")
+        cfg.cleanup_imagefile()
         exit()