diff --git a/.gitignore b/.gitignore
index a5103ff..e66520c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,6 +42,7 @@ htmlcov/
nosetests.xml
coverage.xml
*,cover
+.pytest_cache/
# Translations
*.mo
diff --git a/.travis.yml b/.travis.yml
index 4af490d..3a94266 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,10 +7,8 @@ before_install:
- echo 'America/Los_Angeles' | sudo tee /etc/timezone
- sudo dpkg-reconfigure --frontend noninteractive tzdata
install:
-- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh
- release ; else (exit 0) ; fi
-- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh
- ; else (exit 0) ; fi
+- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/travis-install-ml.sh ; else (exit 0) ; fi
+- if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then ./shared/dev-tasks/setup-marklogic.sh ; else (exit 0) ; fi
script:
- python setup.py test
env:
diff --git a/examples/init-server.py b/examples/init-server.py
new file mode 100644
index 0000000..0c9b5ac
--- /dev/null
+++ b/examples/init-server.py
@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+#
+# Copyright 2018 MarkLogic Corporation
+#
+
+__author__ = 'ndw'
+
+import argparse
+import logging
+import json
+import logging
+from marklogic import MarkLogic
+
+class InitServer:
+ def __init__(self):
+ pass
+
+#logging.basicConfig(level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--host", action='store', default="localhost",
+ help="Management API host")
+parser.add_argument("--username", action='store', default="admin",
+ help="User name")
+parser.add_argument("--password", action='store', default="admin",
+ help="Password")
+parser.add_argument("--wallet", action='store', default="admin",
+ help="Wallet password")
+parser.add_argument('--debug', action='store_true',
+ help='Enable debug logging')
+args = parser.parse_args()
+
+if args.debug:
+ logging.basicConfig(level=logging.WARNING)
+ logging.getLogger("requests").setLevel(logging.WARNING)
+ logging.getLogger("marklogic").setLevel(logging.DEBUG)
+
+print("Initialize host {}".format(args.host))
+MarkLogic.instance_init(args.host)
+print("Initialize admin {}".format(args.host))
+MarkLogic.instance_admin(args.host, "public", args.username, args.password, args.wallet)
+
+print("finished")
diff --git a/examples/mldbmirror.py b/examples/mldbmirror.py
index fe38c64..6bc79f0 100644
--- a/examples/mldbmirror.py
+++ b/examples/mldbmirror.py
@@ -67,6 +67,27 @@ def connect(self, args):
self.path = os.path.abspath(args['path'])
self.loadconfig(self.path)
+ if args['hostname'] is None:
+ if 'host' in self.config:
+ self.hostname = self.config['host']
+ if 'port' in self.config:
+ self.port = self.config['port']
+ else:
+ self.port = 8000
+ if 'management-port' in self.config:
+ self.management_port = self.config['management-port']
+ else:
+ self.management_port = 8002
+ else:
+ parts = args['hostname'].split(":")
+ self.hostname = parts.pop(0)
+ self.management_port = 8002
+ self.port = 8000
+ if parts:
+ self.management_port = parts.pop(0)
+ if parts:
+ self.port = parts.pop(0)
+
if args['credentials'] is not None:
cred = args['credentials']
else:
@@ -74,6 +95,11 @@ def connect(self, args):
cred = self.config['user'] + ":" + self.config['pass']
else:
cred = None
+ key = self.hostname + ":" + str(self.management_port)
+ if key in self.config:
+ obj = self.config[key]
+ if 'user' in obj and 'pass' in obj:
+ cred = obj['user'] + ":" + obj['pass']
try:
adminuser, adminpass = re.split(":", cred)
@@ -102,27 +128,6 @@ def connect(self, args):
if self.root.endswith("/"):
self.root = self.root[0:len(self.root)-1]
- if args['hostname'] is None:
- if 'host' in self.config:
- self.hostname = self.config['host']
- if 'port' in self.config:
- self.port = self.config['port']
- else:
- self.port = 8000
- if 'management-port' in self.config:
- self.management_port = self.config['management-port']
- else:
- self.management_port = 8002
- else:
- parts = args['hostname'].split(":")
- self.hostname = parts.pop(0)
- self.management_port = 8002
- self.port = 8000
- if parts:
- self.management_port = parts.pop(0)
- if parts:
- self.port = parts.pop(0)
-
self.connection \
= Connection(self.hostname, HTTPDigestAuth(adminuser, adminpass), \
port=self.port, management_port=self.management_port)
@@ -486,15 +491,17 @@ def _download_directory(self, trans):
down_map = {}
skip_list = []
for uri in uris:
- if not self.can_store_on_filesystem(uri):
- raise RuntimeError("Cannot save URI:", uri)
-
localfile = self.path + uri
skip = False
- if uri in stamps and os.path.exists(localfile):
- statinfo = os.stat(localfile)
- stamp = self._convert_timestamp(stamps[uri])
- skip = statinfo.st_mtime >= stamp.timestamp()
+
+ if not self.can_store_on_filesystem(uri):
+ print("Skipping " + uri + ": cannot store on filesystem")
+ skip = True
+ else:
+ if uri in stamps and os.path.exists(localfile):
+ statinfo = os.stat(localfile)
+ stamp = self._convert_timestamp(stamps[uri])
+ skip = statinfo.st_mtime >= stamp.timestamp()
if skip:
skip_list.append(localfile)
@@ -709,7 +716,7 @@ def can_store_on_filesystem(self, filename):
filesystem because if it's ever uploaded, it'll get a leading /.
"""
if (not filename.startswith("/")) or ("//" in filename) \
- or (":" in filename) or ('"' in filename) or ('"' in filename) \
+ or (":" in filename) or ('"' in filename) or ("'" in filename) \
or ("\\" in filename):
return False
else:
diff --git a/examples/read-everything.py b/examples/read-everything.py
new file mode 100644
index 0000000..dfb490b
--- /dev/null
+++ b/examples/read-everything.py
@@ -0,0 +1,111 @@
+#!/usr/bin/python3
+#
+# Copyright 2015 MarkLogic Corporation
+#
+# This script attempts to read all of the resource types on the cluster.
+# The point of this script is to make sure that we catch any new properties
+# that have been added by the server.
+
+__author__ = 'ndw'
+
+import argparse
+import logging
+import json
+import logging
+import sys
+from requests.auth import HTTPDigestAuth
+from marklogic.connection import Connection
+from marklogic.models.cluster import LocalCluster
+from marklogic.models.group import Group
+from marklogic.models.host import Host
+from marklogic.models.database import Database
+from marklogic.models.permission import Permission
+from marklogic.models.privilege import Privilege
+from marklogic.models.role import Role
+from marklogic.models.forest import Forest
+from marklogic.models.server import Server
+from marklogic.models.user import User
+
+class ReadEverything:
+ def __init__(self, connection):
+ self.databases = {}
+ self.forests = {}
+ self.servers = {}
+ self.users = {}
+ self.roles = {}
+ self.privileges = {}
+ self.connection = connection
+ pass
+
+ def readClass(self, kind, klass, max_read=sys.maxsize):
+ names = klass.list(self.connection)
+ for name in names:
+ if max_read > 0:
+ if name.find("|") > 0:
+ parts = name.split("|")
+ rsrc = klass.lookup(self.connection, parts[0], parts[1])
+ else:
+ rsrc = klass.lookup(self.connection, name)
+ max_read = max_read - 1
+ print("{}: {}".format(kind, len(names)))
+
+ def readPrivileges(self):
+ names = Privilege.list(self.connection)
+ max_read = { "execute": 5, "uri": 5 }
+ counts = { "execute": 0, "uri": 0 }
+ for name in names:
+ parts = name.split("|")
+ kind = parts[0]
+ pname = parts[1]
+
+ counts[kind] = counts[kind] + 1
+
+ if max_read[kind] > 0:
+ rsrc = Privilege.lookup(self.connection, pname, kind)
+ max_read[kind] = max_read[kind] - 1
+
+ print("Execute privileges: {}".format(counts["execute"]))
+ print("URI privileges: {}".format(counts["uri"]))
+
+ def read(self):
+ conn = self.connection
+ cluster = LocalCluster(connection=conn).read()
+ print("Read local cluster: {}".format(cluster.cluster_name()))
+
+ self.readClass("Groups", Group, max_read=5)
+ self.readClass("Hosts", Host, max_read=5)
+ self.readClass("Databases", Database, max_read=5)
+ self.readClass("Forests", Forest, max_read=5)
+ self.readClass("Servers", Server)
+ self.readClass("Roles", Role, max_read=5)
+ self.readClass("Users", User, max_read=5)
+ self.readPrivileges()
+
+ return
+
+logging.basicConfig(level=logging.INFO)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--host", action='store', default="localhost",
+ help="Management API host")
+parser.add_argument("--username", action='store', default="admin",
+ help="User name")
+parser.add_argument("--password", action='store', default="admin",
+ help="Password")
+parser.add_argument('--debug', action='store_true',
+ help='Enable debug logging')
+args = parser.parse_args()
+
+if args.debug:
+ logging.basicConfig(level=logging.WARNING)
+ logging.getLogger("requests").setLevel(logging.WARNING)
+ logging.getLogger("marklogic").setLevel(logging.DEBUG)
+
+conn = Connection(args.host, HTTPDigestAuth(args.username, args.password))
+read_everything = ReadEverything(conn)
+
+print("Reading all resources from {}".format(args.host))
+
+read_everything.read()
+
+print("Finished")
diff --git a/marklogic/__init__.py b/marklogic/__init__.py
index ed3de89..740a1ec 100644
--- a/marklogic/__init__.py
+++ b/marklogic/__init__.py
@@ -36,7 +36,7 @@
from marklogic.models.server import OdbcServer, XdbcServer
from marklogic.exceptions import InvalidAPIRequest, UnexpectedManagementAPIResponse
-__version__ = "0.0.14"
+__version__ = "0.0.17"
class MarkLogic:
"""
@@ -446,7 +446,7 @@ def instance_init(cls, host):
return Host(host)._set_just_initialized()
@classmethod
- def instance_admin(cls,host,realm,admin,password):
+ def instance_admin(cls,host,realm,admin,password,wallet_password=None):
"""
Initializes the security database of a newly initialized server.
@@ -463,6 +463,9 @@ def instance_admin(cls,host,realm,admin,password):
'realm': realm
}
+ if wallet_password is not None:
+ payload["wallet-password"] = wallet_password
+
uri = "{0}://{1}:8001/admin/v1/instance-admin".format(
conn.protocol, conn.host)
diff --git a/marklogic/cli/manager/marklogic.py b/marklogic/cli/manager/marklogic.py
index a163fb2..455af67 100644
--- a/marklogic/cli/manager/marklogic.py
+++ b/marklogic/cli/manager/marklogic.py
@@ -152,7 +152,11 @@ def restart(self, args, config, connection):
cluster = LocalCluster(connection=connection).read()
print("Restarting cluster...")
cluster.restart()
-
+ # Make sure it's back up
+ status = self.status(args,config,connection,internal=True)
+ while status != 'up':
+ time.sleep(2)
+ status = self.status(args,config,connection,internal=True)
else:
hostname = connection.host
if hostname == 'localhost':
@@ -160,6 +164,11 @@ def restart(self, args, config, connection):
host = Host(hostname,connection=connection).read()
print("Restarting host...")
host.restart()
+ # Make sure it's back up
+ status = self.status(args,config,connection,internal=True)
+ while status != 'up':
+ time.sleep(2)
+ status = self.status(args,config,connection,internal=True)
def stop(self, args, config, connection):
status = self.status(args, config, connection, internal=True)
@@ -174,6 +183,11 @@ def stop(self, args, config, connection):
cluster = LocalCluster(connection=connection).read()
print("Shutting down cluster...")
cluster.shutdown()
+ # Make sure it's all the way down
+ status = self.status(args,config,connection,internal=True)
+ while status != 'down':
+ time.sleep(2)
+ status = self.status(args,config,connection,internal=True)
else:
hostname = connection.host
if hostname == 'localhost':
@@ -190,6 +204,11 @@ def stop(self, args, config, connection):
print("Shutting down host: " + host.host_name())
host.shutdown()
+ # Make sure it's all the way down
+ status = self.status(args,config,connection,internal=True)
+ while status != 'down':
+ time.sleep(2)
+ status = self.status(args,config,connection,internal=True)
status = self.status(args,config,connection,internal=True)
while status == 'up':
diff --git a/marklogic/cli/template.py b/marklogic/cli/template.py
index 2894e5d..ea647ea 100644
--- a/marklogic/cli/template.py
+++ b/marklogic/cli/template.py
@@ -638,6 +638,8 @@ def _make_parser(self, command, artifact, description=""):
help='Host on which to issue the request')
parser.add_argument('--credentials', default='admin:admin',
help='Login credentials for request')
+ parser.add_argument('--https', action='store_true',
+ help='Enable https')
parser.add_argument('--debug', action='store_true',
help='Enable debug logging')
return parser
diff --git a/marklogic/connection.py b/marklogic/connection.py
index 6b7dca4..e359f30 100644
--- a/marklogic/connection.py
+++ b/marklogic/connection.py
@@ -31,6 +31,7 @@
from requests.exceptions import ReadTimeout
from requests.packages.urllib3.exceptions import ProtocolError
from requests.packages.urllib3.exceptions import ReadTimeoutError
+from requests.packages import urllib3
"""
Connection related classes and method to connect to MarkLogic.
@@ -56,6 +57,9 @@ def __init__(self, host, auth,
self.logger = logging.getLogger("marklogic.connection")
self.payload_logger = logging.getLogger("marklogic.connection.payloads")
+ self.verify = False # Danger, Will Robinson!
+ urllib3.disable_warnings()
+
# You'd expect parameters to be a dictionary, but then it couldn't
# have repeated keys, so it's an array.
def uri(self, relation, name=None,
@@ -104,7 +108,7 @@ def client_uri(self, path, protocol=None, host=None, port=None, version=None):
def head(self, uri, accept="application/json"):
self.logger.debug("HEAD {0}...".format(uri))
- self.response = requests.head(uri, auth=self.auth)
+ self.response = requests.head(uri, auth=self.auth, verify=self.verify)
return self._response()
def get(self, uri, accept="application/json", headers=None):
@@ -117,7 +121,8 @@ def get(self, uri, accept="application/json", headers=None):
self.payload_logger.debug("Headers:")
self.payload_logger.debug(json.dumps(headers, indent=2))
- self.response = requests.get(uri, auth=self.auth, headers=headers)
+ self.response = requests.get(uri, auth=self.auth, headers=headers,
+ verify=self.verify)
return self._response()
def post(self, uri, payload=None, etag=None, headers=None,
@@ -143,14 +148,17 @@ def post(self, uri, payload=None, etag=None, headers=None,
self.payload_logger.debug(payload)
if payload is None:
- self.response = requests.post(uri, auth=self.auth, headers=headers)
+ self.response = requests.post(uri, auth=self.auth, headers=headers,
+ verify=self.verify)
else:
if content_type == "application/json":
self.response = requests.post(uri, json=payload,
- auth=self.auth, headers=headers)
+ auth=self.auth, headers=headers,
+ verify=self.verify)
else:
self.response = requests.post(uri, data=payload,
- auth=self.auth, headers=headers)
+ auth=self.auth, headers=headers,
+ verify=self.verify)
return self._response()
@@ -173,14 +181,17 @@ def put(self, uri, payload=None, etag=None,
self.payload_logger.debug(payload)
if payload is None:
- self.response = requests.put(uri, auth=self.auth, headers=headers)
+ self.response = requests.put(uri, auth=self.auth, headers=headers,
+ verify=self.verify)
else:
if content_type == "application/json":
self.response = requests.put(uri, json=payload,
- auth=self.auth, headers=headers)
+ auth=self.auth, headers=headers,
+ verify=self.verify)
else:
self.response = requests.put(uri, data=payload,
- auth=self.auth, headers=headers)
+ auth=self.auth, headers=headers,
+ verify=self.verify)
return self._response()
@@ -203,10 +214,12 @@ def delete(self, uri, payload=None, etag=None,
self.payload_logger.debug(payload)
if payload is None:
- self.response = requests.delete(uri, auth=self.auth, headers=headers)
+ self.response = requests.delete(uri, auth=self.auth, headers=headers,
+ verify=self.verify)
else:
self.response = requests.delete(uri, json=payload,
- auth=self.auth, headers=headers)
+ auth=self.auth, headers=headers,
+ verify=self.verify)
return self._response()
@@ -250,7 +263,8 @@ def wait_for_restart(self, last_startup, timestamp_uri="/admin/v1/timestamp"):
self.logger.debug("Waiting for restart of {0}"
.format(self.host))
response = requests.get(uri, auth=self.auth,
- headers={'accept': 'application/json'})
+ headers={'accept': 'application/json'},
+ verify=self.verify)
done = (response.status_code == 200
and response.text != last_startup)
except TypeError:
diff --git a/marklogic/mma.py b/marklogic/mma.py
index 8a4bec4..0db3f29 100644
--- a/marklogic/mma.py
+++ b/marklogic/mma.py
@@ -7,6 +7,7 @@
import shlex
import sys
from requests.auth import HTTPDigestAuth
+from requests.auth import HTTPBasicAuth
from marklogic.connection import Connection
from marklogic.cli.template import Template
@@ -57,7 +58,7 @@ def run(self, argv):
optarg = False
elif tok.startswith("-"):
options.append(tok)
- if tok != "--debug":
+ if tok != "--debug" and tok != "--https":
optarg = True
elif "=" in tok:
params.append(tok)
@@ -152,9 +153,16 @@ def run(self, argv):
mgmt_port = args['hostname'].split(":")[1]
except IndexError:
mgmt_port = 8002
- self.connection = Connection(host,
- HTTPDigestAuth(username, password),
- management_port=mgmt_port)
+
+ if args['https']:
+ self.connection = Connection(host,
+ HTTPBasicAuth(username, password),
+ protocol="https",
+ management_port=mgmt_port)
+ else:
+ self.connection = Connection(host,
+ HTTPDigestAuth(username, password),
+ management_port=mgmt_port)
# do it!
if command == 'run':
diff --git a/marklogic/models/database/__init__.py b/marklogic/models/database/__init__.py
index d97d003..689dddd 100644
--- a/marklogic/models/database/__init__.py
+++ b/marklogic/models/database/__init__.py
@@ -3755,7 +3755,7 @@ def unmarshal(cls, config, hostname=None,
olist.append(temp)
result._config['range-path-index'] = olist
else:
- logger.warn("Unexpected database property: " + key)
+ logger.warning("Unexpected database property: " + key)
return result
diff --git a/marklogic/models/forest/__init__.py b/marklogic/models/forest/__init__.py
index f2c1513..f21c6b3 100644
--- a/marklogic/models/forest/__init__.py
+++ b/marklogic/models/forest/__init__.py
@@ -445,7 +445,7 @@ def unmarshal(cls, config, connection=None, save_connection=True):
olist.append(temp)
result._config['forest-replica'] = olist
else:
- logger.warn("Unexpected forest property: " + key)
+ logger.warning("Unexpected forest property: " + key)
return result
diff --git a/marklogic/models/group/__init__.py b/marklogic/models/group/__init__.py
index 1c02a7c..f215e7e 100644
--- a/marklogic/models/group/__init__.py
+++ b/marklogic/models/group/__init__.py
@@ -154,7 +154,9 @@ def unmarshal(cls, config,
'opsdirector-log-level', 'opsdirector-metering',
'opsdirector-session-endpoint', 'telemetry-config',
'telemetry-log-level', 'telemetry-metering',
- 'telemetry-session-endpoint'
+ 'telemetry-session-endpoint',
+ 'xdqp-ssl-disable-sslv3', 'xdqp-ssl-disable-tlsv1',
+ 'xdqp-ssl-disable-tlsv1-1', 'xdqp-ssl-disable-tlsv1-2'
}
for key in result._config:
@@ -188,7 +190,7 @@ def unmarshal(cls, config,
r['audit-restriction-items'])
restrictions.append(rest)
else:
- logger.warn("Unexpected audit property: " + prop)
+ logger.warning("Unexpected audit property: " + prop)
audit = Audit(enabled, keep, rotate, events, restrictions)
result._config[key] = audit
elif key == 'event':
@@ -200,7 +202,7 @@ def unmarshal(cls, config,
schemas.append(schema)
result._config[key] = schemas
else:
- logger.warn("Unexpected group property: " + key)
+ logger.warning("Unexpected group property: " + key)
return result
diff --git a/marklogic/utilities/validators.py b/marklogic/utilities/validators.py
index 1994734..b51c910 100644
--- a/marklogic/utilities/validators.py
+++ b/marklogic/utilities/validators.py
@@ -244,8 +244,8 @@ def validate_collation(index_type, collation):
return
if collation is None or collation == "":
return
- raise ValidationError('Collation cannot be {0} for an index of type {1}' \
- .format(index_type, collation))
+ raise ValidationError('Invalid collation for index of type {0}' \
+ .format(index_type), repr(collation))
def validate_type(raw_val, cls):
"""
diff --git a/setup.py b/setup.py
index 9ee37a6..a37b3a5 100644
--- a/setup.py
+++ b/setup.py
@@ -25,8 +25,8 @@ def read(*filenames, **kwargs):
long_description=read('README.rst'),
packages=find_packages(),
install_requires=[
- 'requests>=2.8.0',
- 'requests_toolbelt>=0.6.0'
+ 'requests>=2.21.0',
+ 'requests_toolbelt>=0.9.1'
],
include_package_data=True,
platforms='any',
diff --git a/shared/dev-tasks/travis-install-ml.sh b/shared/dev-tasks/travis-install-ml.sh
index b008590..bd3fee0 100755
--- a/shared/dev-tasks/travis-install-ml.sh
+++ b/shared/dev-tasks/travis-install-ml.sh
@@ -49,7 +49,7 @@ else
# if the user passed a day string as a param then use it instead
test $1 && day=$1
# make a version number out of the date
- ver="8.0-$day"
+ ver="9.0-$day"
echo "********* Downloading MarkLogic nightly $ver"
@@ -60,7 +60,7 @@ else
suff="_amd64.deb"
fnamedeb=$fnamedeb$suff
- url="https://root.marklogic.com/nightly/builds/linux64/rh6-intel64-80-test-1.marklogic.com/b8_0/pkgs.$day/$fname"
+ url="https://root.marklogic.com/nightly/builds/linux64-rh7/rh7v-intel64-90-test-build.marklogic.com/b9_0/pkgs.20190313/$fname"
status=$(curl -k --anyauth -u $MLBUILD_USER:$MLBUILD_PASSWORD --head --write-out %{http_code} --silent --output /dev/null $url)
if [[ $status = 200 ]]; then
diff --git a/tests/test_group.py b/tests/test_group.py
new file mode 100644
index 0000000..31850f3
--- /dev/null
+++ b/tests/test_group.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2015 MarkLogic Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0#
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# File History
+# ------------
+#
+# Paul Hoehne 03/26/2015 Initial development
+# Norman Walsh 05/02/2018 Adapted from test_host.py
+#
+
+from mlconfig import MLConfig
+from marklogic.models.group import Group
+
+class TestGroup(MLConfig):
+ def group_list(self):
+ return Group.list(self.connection)
+
+ def test_list_groups(self):
+ groups = self.group_list()
+ assert len(groups) > 0
+ assert groups
+
+ def test_load(self):
+ group_name = self.group_list()[0]
+ group = Group(group_name)
+ assert group.read(self.connection) is not None
+ assert group.host_timeout() > 0
+