Merge branch 'feature/travis-container' of https://github.com/Alexand…

…erS/bcfg2

.travis.yml

@@ -1,28 +1,82 @@
+sudo: false
+dist: trusty
 language: python
+
+packages: &build_deps_optional
+  - swig
+  - libxml2-utils
+  - yum
+  - libaugeas-dev
+  - augeas-lenses
+  - libacl1-dev
+  - libssl-dev
+
 matrix:
   include:
+    - env: PYTHON=2.4 NOSE_EXCLUDE_DIRS=testsuite/Testsrc/Testlib/TestServer
+      addons:
+        apt:
+          sources: [deadsnakes]
+          packages: [python2.4, python2.4-dev]
+    - env: PYTHON=2.5 NOSE_EXCLUDE_DIRS=testsuite/Testsrc/Testlib/TestServer
+      addons:
+        apt:
+          sources: [deadsnakes]
+          packages: [python2.5, python2.5-dev, libbluetooth-dev]
+
     - python: "2.6"
-      env: WITH_OPTIONAL_DEPS=no TEST_SPHINX=no
     - python: "2.6"
-      env: WITH_OPTIONAL_DEPS=yes TEST_SPHINX=no
-    - python: "2.7_with_system_site_packages"
-      env: WITH_OPTIONAL_DEPS=no TEST_SPHINX=no
-    - python: "2.7_with_system_site_packages"
-      env: WITH_OPTIONAL_DEPS=yes TEST_SPHINX=yes
+      env: WITH_OPTIONAL_DEPS=yes
+      addons:
+        apt:
+          packages: *build_deps_optional
+
+    - python: "2.7"
+    - python: "2.7"
+      env: WITH_OPTIONAL_DEPS=yes WITH_SYSTEM_SITE_PACKAGES=yes TEST_SPHINX=yes
+      addons:
+        apt:
+          packages: [*build_deps_optional, python-gamin, python-selinux]
+
+    - python: "3.5"
+      env: WITH_OPTIONAL_DEPS=yes
+      addons:
+        apt:
+          packages: *build_deps_optional
+
+  allow_failures:
+    - python: "3.5"
+
+  fast_finish: true
+
+before_install:
+  - testsuite/prepare-python.sh
+  - if test -d "$HOME/custom-virtualenv/"; then source "$HOME/custom-virtualenv/bin/activate"; fi
 install:
   - testsuite/install.sh
   - pip install -e .
 script:
-  - nosetests testsuite
+  - testsuite/test.sh
 after_failure:
   - pip freeze
+
 branches:
   except:
     - maint-1.2
     - 1.1.0-stable
+
 notifications:
-  email: [email protected]
   irc:
     channels:
       - "irc.freenode.org#bcfg2"
     use_notice: true
+
+cache:
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/.cache/wheels
+    - $HOME/.cache/xml
+
+before_cache:
+  - rm -f $HOME/.cache/pip/log/debug.log
+  - rm -f $HOME/.cache/xml/catalog.xml

doc/installation/prerequisites.txt

@@ -56,7 +56,8 @@ Bcfg2 Server
 +-------------------------------+----------+--------------------------------+
 | python-gamin or pyinotify     | Any      | gamin or inotify, python       |
 +-------------------------------+----------+--------------------------------+
-| python-ssl (note              | Any      | python, backported ssl module  |
+| python-ssl (this is included  | Any      |                                |
+| in Python2.6 and later)       |          |                                |
 +-------------------------------+----------+--------------------------------+
 | python-setuptools             | Any      |                                |
 +-------------------------------+----------+--------------------------------+

setup.py

@@ -16,11 +16,12 @@
     'lockfile',
     'lxml',
     'python-daemon',
+    'argparse'
 ]
 
-# we only need m2crypto on < python2.6
+# Use the backported ssl module on < python2.6
 if sys.version_info[:2] < (2, 6):
-    inst_reqs.append('M2Crypto')
+    inst_reqs.append('ssl')
 
 setup(name="Bcfg2",
       version=__version__,  # Defined in src/lib/Bcfg2/version.py

src/lib/Bcfg2/Client/Tools/APT.py

@@ -42,10 +42,10 @@ def __init__(self, config):
             if reqdir not in path_entries:
                 os.environ['PATH'] = os.environ['PATH'] + ':' + reqdir
         self.pkgcmd = '%s ' % self.aptget + \
-                      '-o DPkg::Options::=--force-confold ' + \
-                      '-o DPkg::Options::=--force-confmiss ' + \
-                      '--reinstall ' + \
-                      '--force-yes '
+            '-o DPkg::Options::=--force-confold ' + \
+            '-o DPkg::Options::=--force-confmiss ' + \
+            '--reinstall ' + \
+            '--force-yes '
         if not Bcfg2.Options.setup.debug:
             self.pkgcmd += '-q=2 '
         self.pkgcmd += '-y install %s'
@@ -158,8 +158,8 @@ def VerifyPackage(self, entry, modlist, checksums=True):
                              (entry.attrib['name']))
             return False
         pkgname = entry.get('name')
-        if pkgname not in self.pkg_cache or \
-           not self.pkg_cache[pkgname].is_installed:
+        if (pkgname not in self.pkg_cache or
+                not self.pkg_cache[pkgname].is_installed):
             self.logger.info("Package %s not installed" % (entry.get('name')))
             entry.set('current_exists', 'false')
             return False
@@ -183,9 +183,9 @@ def VerifyPackage(self, entry, modlist, checksums=True):
             return False
         else:
             # version matches
-            if not Bcfg2.Options.setup.quick \
-               and entry.get('verify', 'true') == 'true' \
-               and checksums:
+            if (not Bcfg2.Options.setup.quick and
+                    entry.get('verify', 'true') == 'true' and
+                    checksums):
                 pkgsums = self.VerifyDebsums(entry, modlist)
                 return pkgsums
             return True

src/lib/Bcfg2/Client/Tools/BundleDeps.py

@@ -28,7 +28,7 @@ def BundleUpdated(self, entry):
 
         bundle_name = entry.get('name')
         for bundle in self.config.findall('./Bundle/Bundle'):
-            if bundle.get('name') == bundle_name and \
-               bundle not in self.modified:
+            if (bundle.get('name') == bundle_name and
+                    bundle not in self.modified):
                 self.modified.append(bundle)
         return dict()

src/lib/Bcfg2/Client/Tools/Chkconfig.py

@@ -88,8 +88,8 @@ def InstallService(self, entry):
         if bootstatus is not None:
             if bootstatus == 'on':
                 # make sure service is enabled on boot
-                bootcmd = '/sbin/chkconfig %s %s' % \
-                          (entry.get('name'), bootstatus)
+                bootcmd = ('/sbin/chkconfig %s %s' %
+                           (entry.get('name'), bootstatus))
             elif bootstatus == 'off':
                 # make sure service is disabled on boot
                 bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'),

src/lib/Bcfg2/Client/Tools/DebInit.py

@@ -142,8 +142,8 @@ def InstallService(self, entry):
                 # 'disabled' means we don't attempt to modify running svcs
                 return bootcmdrv and seqcmdrv
             buildmode = Bcfg2.Options.setup.service_mode == 'build'
-            if (entry.get('status') == 'on' and not buildmode) and \
-               entry.get('current_status') == 'off':
+            if ((entry.get('status') == 'on' and not buildmode) and
+                    entry.get('current_status') == 'off'):
                 svccmdrv = self.start_service(entry)
             elif (entry.get('status') == 'off' or buildmode) and \
                     entry.get('current_status') == 'on':

src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py

@@ -5,6 +5,7 @@
 from augeas import Augeas
 from Bcfg2.Client.Tools.POSIX.base import POSIXTool
 from Bcfg2.Client.Tools.POSIX.File import POSIXFile
+from Bcfg2.Compat import all  # pylint: disable=W0622
 
 
 class AugeasCommand(object):
@@ -249,8 +250,8 @@ def verify(self, entry, modlist):
         for cmd in self.get_commands(entry):
             try:
                 if not cmd.verify():
-                    err = "Augeas: Command has not been applied to %s: %s" % \
-                          (entry.get("name"), cmd)
+                    err = ("Augeas: Command has not been applied to %s: %s" %
+                           (entry.get("name"), cmd))
                     self.logger.debug(err)
                     entry.set('qtext', "\n".join([entry.get('qtext', ''),
                                                   err]))
@@ -259,8 +260,8 @@ def verify(self, entry, modlist):
                 else:
                     cmd.command.set("verified", "true")
             except:  # pylint: disable=W0702
-                err = "Augeas: Unexpected error verifying %s: %s: %s" % \
-                      (entry.get("name"), cmd, sys.exc_info()[1])
+                err = ("Augeas: Unexpected error verifying %s: %s: %s" %
+                       (entry.get("name"), cmd, sys.exc_info()[1]))
                 self.logger.error(err)
                 entry.set('qtext', "\n".join([entry.get('qtext', ''), err]))
                 rv = False

src/lib/Bcfg2/Client/Tools/RcUpdate.py

@@ -102,8 +102,8 @@ def InstallService(self, entry):
                 # 'disabled' means we don't attempt to modify running svcs
                 return bootcmdrv
             buildmode = Bcfg2.Options.setup.service_mode == 'build'
-            if (entry.get('status') == 'on' and not buildmode) and \
-               entry.get('current_status') == 'off':
+            if ((entry.get('status') == 'on' and not buildmode) and
+                    entry.get('current_status') == 'off'):
                 svccmdrv = self.start_service(entry)
             elif (entry.get('status') == 'off' or buildmode) and \
                     entry.get('current_status') == 'on':

src/lib/Bcfg2/Client/Tools/SYSV.py

@@ -119,8 +119,8 @@ def VerifyPackage(self, entry, modlist):
                 self.logger.debug("Package %s not installed" %
                                   entry.get("name"))
         else:
-            if Bcfg2.Options.setup.quick or \
-               entry.attrib.get('verify', 'true') == 'false':
+            if (Bcfg2.Options.setup.quick or
+                    entry.attrib.get('verify', 'true') == 'false'):
                 return True
             rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name'))
             if rv.success:

src/lib/Bcfg2/Client/XML.py

@@ -2,7 +2,7 @@
 
 # library will use lxml, then builtin xml.etree, then ElementTree
 
-# pylint: disable=E0611,W0611,W0613,C0103
+# pylint: disable=E0611,E1101,W0611,W0613,C0103
 
 try:
     from lxml.etree import Element, SubElement, tostring, XMLParser

src/lib/Bcfg2/Client/__init__.py

@@ -921,8 +921,8 @@ def GenerateStats(self):
         """Generate XML summary of execution statistics."""
         states = {}
         for (item, val) in list(self.states.items()):
-            if not Bcfg2.Options.setup.only_important or \
-               item.get('important', 'false').lower() == 'true':
+            if (not Bcfg2.Options.setup.only_important or
+                    item.get('important', 'false').lower() == 'true'):
                 states[item] = val
 
         feedback = XML.Element("upload-statistics")

src/lib/Bcfg2/DBSettings.py

@@ -21,7 +21,8 @@
 except ImportError:
     HAS_SOUTH = False
 
-settings = dict(  # pylint: disable=C0103
+# pylint: disable=C0103
+settings = dict(
     TIME_ZONE=None,
     TEMPLATE_DEBUG=False,
     DEBUG=False,
@@ -106,8 +107,8 @@ def finalize_django_config(opts=None, silent=False):
             OPTIONS=opts.db_opts,
             SCHEMA=opts.db_schema))
 
-    if hasattr(opts, "reporting_db_engine") and \
-       opts.reporting_db_engine is not None:
+    if (hasattr(opts, "reporting_db_engine") and
+            opts.reporting_db_engine is not None):
         settings['DATABASES']['Reporting'] = dict(
             ENGINE="django.db.backends.%s" % opts.reporting_db_engine,
             NAME=opts.reporting_db_name,
@@ -180,9 +181,9 @@ def upgrade_to_django_migrations(database, logger):
         cursor.cursor.execute('SELECT migration FROM south_migrationhistory')
         applied_migrations = [name for (name,) in cursor.fetchall()]
         last_migration = sorted(applied_migrations).pop()
+    # django.db.DatabaseError is not working here, because we are
+    # using the low level api to interact directly with the database
     except:  # pylint: disable=W0702
-        # django.db.DatabaseError is not working here, because we are
-        # using the low level api to interact directly with the database
         logger.debug("No south migration detected for database: %s." %
                      database)
 

src/lib/Bcfg2/Logger.py

@@ -133,10 +133,11 @@ def emit(self, record):
                                                           logging.WARNING),
                                       self.format(reconn)))
                     self.socket.send(msg)
+
+                # If we still fail then drop it.  Running
+                # bcfg2-server as non-root can trigger permission
+                # denied exceptions.
                 except:  # pylint: disable=W0702
-                    # If we still fail then drop it.  Running
-                    # bcfg2-server as non-root can trigger permission
-                    # denied exceptions.
                     pass
 
 

src/lib/Bcfg2/Options/Parser.py

@@ -12,12 +12,14 @@
            "new_parser"]
 
 
+# pylint: disable=C0103
 #: The repository option.  This is specified here (and imported into
 #: :module:`Bcfg2.Options.Common`) rather than vice-versa due to
 #: circular imports.
-repository = PathOption(  # pylint: disable=C0103
+repository = PathOption(
     '-Q', '--repository', cf=('server', 'repository'),
     default='/var/lib/bcfg2', help="Server repository path")
+# pylint: enable=C0103
 
 
 #: A module-level :class:`argparse.Namespace` object that stores all

src/lib/Bcfg2/Utils.py

@@ -338,7 +338,7 @@ def __init__(self, getter):
         self.getter = getter
 
     def __get__(self, instance, owner):
-        return self.getter(owner)
+        return classmethod(self.getter).__get__(None, owner)()
 
 
 def is_string(strng, encoding):

src/lib/Bcfg2/manage.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-import os
+""" Wrapper for the django manage.py with the Bcfg2 Opitons parsing. """
+
 import sys
 import django
 import Bcfg2.Options
@@ -10,14 +11,21 @@
 except ImportError:
     pass
 
-parser = Bcfg2.Options.get_parser()
-parser.add_options([Bcfg2.Options.PositionalArgument('django_command', nargs='*')])
-parser.parse()
 
-if __name__ == "__main__":
+def main():
+    parser = Bcfg2.Options.get_parser()
+    parser.add_options([
+        Bcfg2.Options.PositionalArgument('django_command', nargs='*')])
+    parser.parse()
+
     if django.VERSION[0] == 1 and django.VERSION[1] >= 6:
         from django.core.management import execute_from_command_line
-        execute_from_command_line(sys.argv[:1] + Bcfg2.Options.setup.django_command)
+        execute_from_command_line(
+            sys.argv[:1] + Bcfg2.Options.setup.django_command)
     else:
         from django.core.management import execute_manager
         execute_manager(Bcfg2.DBSettings.settings)
+
+
+if __name__ == "__main__":
+    main()

testsuite/Testschema/test_schema.py

@@ -34,13 +34,18 @@
 
 class TestSchemas(Bcfg2TestCase):
     schema_url = "http://www.w3.org/2001/XMLSchema.xsd"
+    catalog_file = os.path.expanduser("~/.cache/xml/catalog.xml")
 
     @skipUnless(HAS_XMLLINT, "xmllint not installed")
     def test_valid(self):
+        env = os.environ.copy()
+        if os.path.exists(self.catalog_file):
+            print('Using cached schema files.')
+            env["SGML_CATALOG_FILES"] = self.catalog_file
         schemas = [s for s in glob.glob(os.path.join(srcpath, '*.xsd'))]
-        xmllint = Popen(['xmllint', '--xinclude', '--noout', '--schema',
-                         self.schema_url] + schemas,
-                        stdout=PIPE, stderr=STDOUT)
+        xmllint = Popen(['xmllint', '--xinclude', '--noout', '--catalogs',
+                         '--schema', self.schema_url] + schemas,
+                        stdout=PIPE, stderr=STDOUT, env=env)
         print(xmllint.communicate()[0].decode())
         self.assertEqual(xmllint.wait(), 0)