Bring master up-to-date with a bunch of changes

.gitignore

@@ -42,6 +42,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+.pytest_cache/
 
 # Translations
 *.mo

.travis.yml

@@ -7,10 +7,8 @@ before_install:
 - echo 'America/Los_Angeles' | sudo tee /etc/timezone
 - sudo dpkg-reconfigure --frontend noninteractive tzdata
 install:
-- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh
-  release ; else (exit 0) ; fi
-- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh
-  ; else (exit 0) ; fi
+- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh ; else (exit 0) ; fi
+- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh ; else (exit 0) ; fi
 script:
 - python setup.py test
 env:

examples/init-server.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+#
+# Copyright 2018 MarkLogic Corporation
+#
+
+__author__ = 'ndw'
+
+import argparse
+import logging
+import json
+import logging
+from marklogic import MarkLogic
+
+class InitServer:
+    def __init__(self):
+        pass
+
+#logging.basicConfig(level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--host", action='store', default="localhost",
+                    help="Management API host")
+parser.add_argument("--username", action='store', default="admin",
+                    help="User name")
+parser.add_argument("--password", action='store', default="admin",
+                    help="Password")
+parser.add_argument("--wallet", action='store', default="admin",
+                    help="Wallet password")
+parser.add_argument('--debug', action='store_true',
+                    help='Enable debug logging')
+args = parser.parse_args()
+
+if args.debug:
+    logging.basicConfig(level=logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("marklogic").setLevel(logging.DEBUG)
+
+print("Initialize host {}".format(args.host))
+MarkLogic.instance_init(args.host)
+print("Initialize admin {}".format(args.host))
+MarkLogic.instance_admin(args.host, "public", args.username, args.password, args.wallet)
+
+print("finished")

examples/mldbmirror.py

@@ -67,13 +67,39 @@ def connect(self, args):
         self.path = os.path.abspath(args['path'])
         self.loadconfig(self.path)
 
+        if args['hostname'] is None:
+            if 'host' in self.config:
+                self.hostname = self.config['host']
+                if 'port' in self.config:
+                    self.port = self.config['port']
+                else:
+                    self.port = 8000
+                if 'management-port' in self.config:
+                    self.management_port = self.config['management-port']
+                else:
+                    self.management_port = 8002
+        else:
+            parts = args['hostname'].split(":")
+            self.hostname = parts.pop(0)
+            self.management_port = 8002
+            self.port = 8000
+            if parts:
+                self.management_port = parts.pop(0)
+            if parts:
+                self.port = parts.pop(0)
+
         if args['credentials'] is not None:
             cred = args['credentials']
         else:
             if 'user' in self.config and 'pass' in self.config:
                 cred = self.config['user'] + ":" + self.config['pass']
             else:
                 cred = None
+                key = self.hostname + ":" + str(self.management_port)
+                if key in self.config:
+                    obj = self.config[key]
+                    if 'user' in obj and 'pass' in obj:
+                        cred = obj['user'] + ":" + obj['pass']
 
         try:
             adminuser, adminpass = re.split(":", cred)
@@ -102,27 +128,6 @@ def connect(self, args):
         if self.root.endswith("/"):
             self.root = self.root[0:len(self.root)-1]
 
-        if args['hostname'] is None:
-            if 'host' in self.config:
-                self.hostname = self.config['host']
-                if 'port' in self.config:
-                    self.port = self.config['port']
-                else:
-                    self.port = 8000
-                if 'management-port' in self.config:
-                    self.management_port = self.config['management-port']
-                else:
-                    self.management_port = 8002
-        else:
-            parts = args['hostname'].split(":")
-            self.hostname = parts.pop(0)
-            self.management_port = 8002
-            self.port = 8000
-            if parts:
-                self.management_port = parts.pop(0)
-            if parts:
-                self.port = parts.pop(0)
-
         self.connection \
           = Connection(self.hostname, HTTPDigestAuth(adminuser, adminpass), \
                            port=self.port, management_port=self.management_port)
@@ -486,15 +491,17 @@ def _download_directory(self, trans):
         down_map = {}
         skip_list = []
         for uri in uris:
-            if not self.can_store_on_filesystem(uri):
-                raise RuntimeError("Cannot save URI:", uri)
-
             localfile = self.path + uri
             skip = False
-            if uri in stamps and os.path.exists(localfile):
-                statinfo = os.stat(localfile)
-                stamp = self._convert_timestamp(stamps[uri])
-                skip = statinfo.st_mtime >= stamp.timestamp()
+
+            if not self.can_store_on_filesystem(uri):
+                print("Skipping " + uri + ": cannot store on filesystem")
+                skip = True
+            else:
+                if uri in stamps and os.path.exists(localfile):
+                    statinfo = os.stat(localfile)
+                    stamp = self._convert_timestamp(stamps[uri])
+                    skip = statinfo.st_mtime >= stamp.timestamp()
 
             if skip:
                 skip_list.append(localfile)
@@ -709,7 +716,7 @@ def can_store_on_filesystem(self, filename):
         filesystem because if it's ever uploaded, it'll get a leading /.
         """
         if (not filename.startswith("/")) or ("//" in filename) \
-          or (":" in filename) or ('"' in filename) or ('"' in filename) \
+          or (":" in filename) or ('"' in filename) or ("'" in filename) \
           or ("\\" in filename):
             return False
         else:

examples/read-everything.py

@@ -0,0 +1,111 @@
+#!/usr/bin/python3
+#
+# Copyright 2015 MarkLogic Corporation
+#
+# This script attempts to read all of the resource types on the cluster.
+# The point of this script is to make sure that we catch any new properties
+# that have been added by the server.
+
+__author__ = 'ndw'
+
+import argparse
+import logging
+import json
+import logging
+import sys
+from requests.auth import HTTPDigestAuth
+from marklogic.connection import Connection
+from marklogic.models.cluster import LocalCluster
+from marklogic.models.group import Group
+from marklogic.models.host import Host
+from marklogic.models.database import Database
+from marklogic.models.permission import Permission
+from marklogic.models.privilege import Privilege
+from marklogic.models.role import Role
+from marklogic.models.forest import Forest
+from marklogic.models.server import Server
+from marklogic.models.user import User
+
+class ReadEverything:
+    def __init__(self, connection):
+        self.databases = {}
+        self.forests = {}
+        self.servers = {}
+        self.users = {}
+        self.roles = {}
+        self.privileges = {}
+        self.connection = connection
+        pass
+
+    def readClass(self, kind, klass, max_read=sys.maxsize):
+        names = klass.list(self.connection)
+        for name in names:
+            if max_read > 0:
+                if name.find("|") > 0:
+                    parts = name.split("|")
+                    rsrc = klass.lookup(self.connection, parts[0], parts[1])
+                else:
+                    rsrc = klass.lookup(self.connection, name)
+                max_read = max_read - 1
+        print("{}: {}".format(kind, len(names)))
+
+    def readPrivileges(self):
+        names = Privilege.list(self.connection)
+        max_read = { "execute": 5, "uri": 5 }
+        counts = { "execute": 0, "uri": 0 }
+        for name in names:
+            parts = name.split("|")
+            kind = parts[0]
+            pname = parts[1]
+
+            counts[kind] = counts[kind] + 1
+
+            if max_read[kind] > 0:
+                rsrc = Privilege.lookup(self.connection, pname, kind)
+                max_read[kind] = max_read[kind] - 1
+
+        print("Execute privileges: {}".format(counts["execute"]))
+        print("URI privileges: {}".format(counts["uri"]))
+
+    def read(self):
+        conn = self.connection
+        cluster = LocalCluster(connection=conn).read()
+        print("Read local cluster: {}".format(cluster.cluster_name()))
+
+        self.readClass("Groups", Group, max_read=5)
+        self.readClass("Hosts", Host, max_read=5)
+        self.readClass("Databases", Database, max_read=5)
+        self.readClass("Forests", Forest, max_read=5)
+        self.readClass("Servers", Server)
+        self.readClass("Roles", Role, max_read=5)
+        self.readClass("Users", User, max_read=5)
+        self.readPrivileges()
+
+        return
+
+logging.basicConfig(level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--host", action='store', default="localhost",
+                    help="Management API host")
+parser.add_argument("--username", action='store', default="admin",
+                    help="User name")
+parser.add_argument("--password", action='store', default="admin",
+                    help="Password")
+parser.add_argument('--debug', action='store_true',
+                    help='Enable debug logging')
+args = parser.parse_args()
+
+if args.debug:
+    logging.basicConfig(level=logging.WARNING)
+    logging.getLogger("requests").setLevel(logging.WARNING)
+    logging.getLogger("marklogic").setLevel(logging.DEBUG)
+
+conn = Connection(args.host, HTTPDigestAuth(args.username, args.password))
+read_everything = ReadEverything(conn)
+
+print("Reading all resources from {}".format(args.host))
+
+read_everything.read()
+
+print("Finished")

marklogic/__init__.py

@@ -36,7 +36,7 @@
 from marklogic.models.server import OdbcServer, XdbcServer
 from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse
 
-__version__ = "0.0.14"
+__version__ = "0.0.17"
 
 class MarkLogic:
     """
@@ -446,7 +446,7 @@ def instance_init(cls, host):
         return Host(host)._set_just_initialized()
 
     @classmethod
-    def instance_admin(cls,host,realm,admin,password):
+    def instance_admin(cls,host,realm,admin,password,wallet_password=None):
         """
         Initializes the security database of a newly initialized server.
 
@@ -463,6 +463,9 @@ def instance_admin(cls,host,realm,admin,password):
             'realm': realm
             }
 
+        if wallet_password is not None:
+            payload["wallet-password"] = wallet_password
+
         uri = "{0}://{1}:8001/admin/v1/instance-admin".format(
             conn.protocol, conn.host)
 

marklogic/cli/manager/marklogic.py

@@ -152,14 +152,23 @@ def restart(self, args, config, connection):
                 cluster = LocalCluster(connection=connection).read()
                 print("Restarting cluster...")
                 cluster.restart()
-
+                # Make sure it's back up
+                status = self.status(args,config,connection,internal=True)
+                while status != 'up':
+                    time.sleep(2)
+                    status = self.status(args,config,connection,internal=True)
             else:
                 hostname = connection.host
                 if hostname == 'localhost':
                     hostname = socket.gethostname()
                 host = Host(hostname,connection=connection).read()
                 print("Restarting host...")
                 host.restart()
+                # Make sure it's back up
+                status = self.status(args,config,connection,internal=True)
+                while status != 'up':
+                    time.sleep(2)
+                    status = self.status(args,config,connection,internal=True)
 
     def stop(self, args, config, connection):
         status = self.status(args, config, connection, internal=True)
@@ -174,6 +183,11 @@ def stop(self, args, config, connection):
                 cluster = LocalCluster(connection=connection).read()
                 print("Shutting down cluster...")
                 cluster.shutdown()
+                # Make sure it's all the way down
+                status = self.status(args,config,connection,internal=True)
+                while status != 'down':
+                    time.sleep(2)
+                    status = self.status(args,config,connection,internal=True)
             else:
                 hostname = connection.host
                 if hostname == 'localhost':
@@ -190,6 +204,11 @@ def stop(self, args, config, connection):
 
                 print("Shutting down host: " + host.host_name())
                 host.shutdown()
+                # Make sure it's all the way down
+                status = self.status(args,config,connection,internal=True)
+                while status != 'down':
+                    time.sleep(2)
+                    status = self.status(args,config,connection,internal=True)
 
             status = self.status(args,config,connection,internal=True)
             while status == 'up':

marklogic/cli/template.py

@@ -638,6 +638,8 @@ def _make_parser(self, command, artifact, description=""):
                             help='Host on which to issue the request')
         parser.add_argument('--credentials', default='admin:admin',
                             help='Login credentials for request')
+        parser.add_argument('--https', action='store_true',
+                            help='Enable https')
         parser.add_argument('--debug', action='store_true',
                             help='Enable debug logging')
         return parser