From 2f7e272745285e7fcc4d9075827a73f6d8aade3c Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Sun, 14 Feb 2016 23:19:58 -0500 Subject: [PATCH 1/8] Merge the SSL tests. --- test/ssl/resources.py | 5 ++++ test/ssl/test_ssl_connection.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 test/ssl/resources.py create mode 100644 test/ssl/test_ssl_connection.py diff --git a/test/ssl/resources.py b/test/ssl/resources.py new file mode 100644 index 0000000..073d0c1 --- /dev/null +++ b/test/ssl/resources.py @@ -0,0 +1,5 @@ +class TestConnection(object): + hostname = "192.168.200.162" + admin = "admin" + password = "admin" + sslport = 8003 \ No newline at end of file diff --git a/test/ssl/test_ssl_connection.py b/test/ssl/test_ssl_connection.py new file mode 100644 index 0000000..0251bc4 --- /dev/null +++ b/test/ssl/test_ssl_connection.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function, absolute_import + +# +# Copyright 2016 MarkLogic Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0# +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File History +# ------------ +# +# Philip Barber 02/14/2016 Initial development +# + +import unittest +from requests.auth import HTTPDigestAuth +from marklogic.models import Database, Server, Forest +from marklogic.connection import Connection +from test.ssl.resources import TestConnection as tc + +class TestSslConnection(unittest.TestCase): + """ + Basic creation test function. + """ + def test_check_ssl_manage_server(self): + conn = Connection(tc.hostname, + HTTPDigestAuth(tc.admin, tc.password), + protocol="https", + verify = "", + management_port=tc.sslport) + server = Server.lookup(conn, "Default|Manage") + self.assertIsNotNone(server) + self.assertEqual('Manage', server.server_name()) + +if __name__ == "__main__": + unittest.main() From 6b76f5007c211b68ec3bf37bd12df0d845ca035d Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Sun, 14 Feb 2016 23:29:34 -0500 Subject: [PATCH 2/8] Merging connection.py --- marklogic/connection.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/marklogic/connection.py b/marklogic/connection.py index 9ca6ce0..d27b74e 100644 --- a/marklogic/connection.py +++ b/marklogic/connection.py @@ -38,7 +38,7 @@ class Connection: """ def __init__(self, host, auth, protocol="http", port=8000, management_port=8002, - root="manage", version="v2", client_version="v1"): + root="manage", version="v2", client_version="v1", verify=False): self.host = host self.auth = auth self.protocol = protocol @@ -47,6 +47,7 @@ def __init__(self, host, auth, self.root = root self.version = version self.client_version = client_version + self.verify = verify self.logger = logging.getLogger("marklogic.connection") self.payload_logger = logging.getLogger("marklogic.connection.payloads") @@ -98,7 +99,7 @@ def client_uri(self, path, protocol=None, host=None, port=None, version=None): def head(self, uri, accept="application/json"): self.logger.debug("HEAD {0}...".format(uri)) - self.response = requests.head(uri, auth=self.auth) + self.response = requests.head(uri, auth=self.auth, verify=verify) return self._response() def get(self, uri, accept="application/json", headers=None): @@ -111,7 +112,7 @@ def get(self, uri, accept="application/json", headers=None): self.payload_logger.debug("Headers:") self.payload_logger.debug(json.dumps(headers, indent=2)) - self.response = requests.get(uri, auth=self.auth, headers=headers) + self.response = requests.get(uri, auth=self.auth, headers=headers, verify=verify) return self._response() def post(self, uri, payload=None, etag=None, @@ -133,14 +134,14 @@ def post(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.post(uri, auth=self.auth, headers=headers) + self.response = requests.post(uri, auth=self.auth, headers=headers, verify=verify) else: if content_type == "application/json": self.response = requests.post(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, verify=verify) else: self.response = requests.post(uri, data=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, verify=verify) return self._response() @@ -163,14 +164,14 @@ def put(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.put(uri, auth=self.auth, headers=headers) + self.response = requests.put(uri, auth=self.auth, headers=headers, verify=verify) else: if content_type == "application/json": self.response = requests.put(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, verify=verify) else: self.response = requests.put(uri, data=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, verify=verify) return self._response() @@ -193,10 +194,10 @@ def delete(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.delete(uri, auth=self.auth, headers=headers) + self.response = requests.delete(uri, auth=self.auth, headers=headers, verify=verify) else: self.response = requests.delete(uri, json=payload, - auth=self.auth, headers=headers) + auth=self.auth, headers=headers, verify=verify) return self._response() @@ -239,7 +240,7 @@ def wait_for_restart(self, last_startup, timestamp_uri="/admin/v1/timestamp"): try: self.logger.debug("Waiting for restart of {0}".format(self.host)) response = requests.get(uri, auth=self.auth, - headers={'accept': 'application/json'}) + headers={'accept': 'application/json'}, verify=verify) done = response.status_code == 200 and response.text != last_startup except TypeError: self.logger.debug("{0}: {1}".format(response.status_code, From 3a7da912cc4430668c3a770cd2a5f3d23c430983 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Sun, 14 Feb 2016 23:33:05 -0500 Subject: [PATCH 3/8] Still merging connection.py --- marklogic/connection.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/marklogic/connection.py b/marklogic/connection.py index d27b74e..8b1bf62 100644 --- a/marklogic/connection.py +++ b/marklogic/connection.py @@ -99,7 +99,7 @@ def client_uri(self, path, protocol=None, host=None, port=None, version=None): def head(self, uri, accept="application/json"): self.logger.debug("HEAD {0}...".format(uri)) - self.response = requests.head(uri, auth=self.auth, verify=verify) + self.response = requests.head(uri, auth=self.auth, verify=self.verify) return self._response() def get(self, uri, accept="application/json", headers=None): @@ -112,7 +112,7 @@ def get(self, uri, accept="application/json", headers=None): self.payload_logger.debug("Headers:") self.payload_logger.debug(json.dumps(headers, indent=2)) - self.response = requests.get(uri, auth=self.auth, headers=headers, verify=verify) + self.response = requests.get(uri, auth=self.auth, headers=headers, verify=self.verify) return self._response() def post(self, uri, payload=None, etag=None, @@ -134,14 +134,14 @@ def post(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.post(uri, auth=self.auth, headers=headers, verify=verify) + self.response = requests.post(uri, auth=self.auth, headers=headers, verify=self.verify) else: if content_type == "application/json": self.response = requests.post(uri, json=payload, - auth=self.auth, headers=headers, verify=verify) + auth=self.auth, headers=headers, verify=self.verify) else: self.response = requests.post(uri, data=payload, - auth=self.auth, headers=headers, verify=verify) + auth=self.auth, headers=headers, verify=self.verify) return self._response() @@ -164,14 +164,14 @@ def put(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.put(uri, auth=self.auth, headers=headers, verify=verify) + self.response = requests.put(uri, auth=self.auth, headers=headers, verify=self.verify) else: if content_type == "application/json": self.response = requests.put(uri, json=payload, - auth=self.auth, headers=headers, verify=verify) + auth=self.auth, headers=headers, verify=self.verify) else: self.response = requests.put(uri, data=payload, - auth=self.auth, headers=headers, verify=verify) + auth=self.auth, headers=headers, verify=self.verify) return self._response() @@ -194,10 +194,10 @@ def delete(self, uri, payload=None, etag=None, self.payload_logger.debug(payload) if payload is None: - self.response = requests.delete(uri, auth=self.auth, headers=headers, verify=verify) + self.response = requests.delete(uri, auth=self.auth, headers=headers, verify=self.verify) else: self.response = requests.delete(uri, json=payload, - auth=self.auth, headers=headers, verify=verify) + auth=self.auth, headers=headers, verify=self.verify) return self._response() @@ -240,7 +240,7 @@ def wait_for_restart(self, last_startup, timestamp_uri="/admin/v1/timestamp"): try: self.logger.debug("Waiting for restart of {0}".format(self.host)) response = requests.get(uri, auth=self.auth, - headers={'accept': 'application/json'}, verify=verify) + headers={'accept': 'application/json'}, verify=self.verify) done = response.status_code == 200 and response.text != last_startup except TypeError: self.logger.debug("{0}: {1}".format(response.status_code, From a790f5bf354153152a6a1f4cd17b22abc4d2b8cd Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 15 Feb 2016 12:52:58 -0500 Subject: [PATCH 4/8] Adding the putFile method. --- marklogic/connection.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/marklogic/connection.py b/marklogic/connection.py index 8b1bf62..ef56f6f 100644 --- a/marklogic/connection.py +++ b/marklogic/connection.py @@ -175,6 +175,22 @@ def put(self, uri, payload=None, etag=None, return self._response() + def putFile(self, uri, data, etag=None, + content_type="application/json", accept="application/json"): + + headers = {'content-type': content_type, + 'accept': accept} + if etag is not None: + headers['if-match'] = etag + + self.logger.debug("PUT {0}...".format(uri)) + self.payload_logger.debug("Headers:") + self.payload_logger.debug(json.dumps(headers, indent=2)) + + self.response = requests.put(uri, data=data, auth=self.auth, headers=headers, verify=self.verify) + + return self._response() + def delete(self, uri, payload=None, etag=None, content_type="application/json", accept="application/json"): From ef74e4125fd2a591591c9347048f912e9a2a3c54 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 15 Feb 2016 12:57:10 -0500 Subject: [PATCH 5/8] Adding an example file of the Connection.putFile method. --- examples/accessControl.sjs | 13 +++++++++++++ examples/props.json | 4 ++++ examples/put-file.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 examples/accessControl.sjs create mode 100644 examples/props.json create mode 100644 examples/put-file.py diff --git a/examples/accessControl.sjs b/examples/accessControl.sjs new file mode 100644 index 0000000..6518f6b --- /dev/null +++ b/examples/accessControl.sjs @@ -0,0 +1,13 @@ +function metadataFilter(context, params, content) { + var mutableDoc = content.toObject(); + xdmp.log("context"); + xdmp.log(context); + xdmp.log("params"); + xdmp.log(params); + xdmp.log("content"); + xdmp.log(content); + content = xdmp.unquote('hello', null, ["repair-none", "default-language=en"]); + return content; +}; + +exports.transform = metadataFilter; \ No newline at end of file diff --git a/examples/props.json b/examples/props.json new file mode 100644 index 0000000..a6420ab --- /dev/null +++ b/examples/props.json @@ -0,0 +1,4 @@ +{ + "document-transform-all":false, + "document-transform-out":"accessControl" +} \ No newline at end of file diff --git a/examples/put-file.py b/examples/put-file.py new file mode 100644 index 0000000..7bd9884 --- /dev/null +++ b/examples/put-file.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 +# +# Copyright 2015 MarkLogic Corporation +# +# This script takes a JSON object that enumerates a set of artifacts and +# applies those changes to the server, creating artifacts if necessary. +# +# See put-json-file.py +# +# For example: +# +# python3 put-json-file +# +# TODO +# +# * Upload a file using a PUT CURL command + +__author__ = 'pmb' + +from requests.auth import HTTPDigestAuth +from marklogic.connection import Connection + +conn = Connection("192.168.200.162", HTTPDigestAuth("admin","admin")) + +uri = "http://192.168.200.162:8004/LATEST/config/properties" +data = open("./props.json", 'rb').read() +response = conn.putFile(uri=uri, data=data) +print(response) + +uri = "http://192.168.200.162:8004/LATEST/config/transforms/accessControl" +data = open("./accessControl.sjs", 'rb').read() +response = conn.putFile(uri=uri, data=data, content_type="application/vnd.marklogic-javascript") +print(response) From 0ece653761b38d46942a528483f0ec419a4434aa Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 15 Feb 2016 13:00:26 -0500 Subject: [PATCH 6/8] Organize and genericize the putFile example. --- examples/data/props.json | 4 ++++ examples/{accessControl.sjs => data/simpleExtension.sjs} | 0 examples/props.json | 4 ---- examples/put-file.py | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 examples/data/props.json rename examples/{accessControl.sjs => data/simpleExtension.sjs} (100%) delete mode 100644 examples/props.json diff --git a/examples/data/props.json b/examples/data/props.json new file mode 100644 index 0000000..02dd711 --- /dev/null +++ b/examples/data/props.json @@ -0,0 +1,4 @@ +{ + "document-transform-all":false, + "document-transform-out":"simpleExtension" +} \ No newline at end of file diff --git a/examples/accessControl.sjs b/examples/data/simpleExtension.sjs similarity index 100% rename from examples/accessControl.sjs rename to examples/data/simpleExtension.sjs diff --git a/examples/props.json b/examples/props.json deleted file mode 100644 index a6420ab..0000000 --- a/examples/props.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "document-transform-all":false, - "document-transform-out":"accessControl" -} \ No newline at end of file diff --git a/examples/put-file.py b/examples/put-file.py index 7bd9884..8a54615 100644 --- a/examples/put-file.py +++ b/examples/put-file.py @@ -23,11 +23,11 @@ conn = Connection("192.168.200.162", HTTPDigestAuth("admin","admin")) uri = "http://192.168.200.162:8004/LATEST/config/properties" -data = open("./props.json", 'rb').read() +data = open("./data/props.json", 'rb').read() response = conn.putFile(uri=uri, data=data) print(response) -uri = "http://192.168.200.162:8004/LATEST/config/transforms/accessControl" -data = open("./accessControl.sjs", 'rb').read() +uri = "http://192.168.200.162:8004/LATEST/config/transforms/simpleExtension" +data = open("./data/simpleExtension.sjs", 'rb').read() response = conn.putFile(uri=uri, data=data, content_type="application/vnd.marklogic-javascript") print(response) From cb6c51876ad5c2caa94afbbb010f7c8a274a745c Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 15 Feb 2016 17:42:08 -0500 Subject: [PATCH 7/8] Adds a test for the Connection.putFile method. --- test/data/simpleExtension.sjs | 13 +++++++++ test/test_put_file.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 test/data/simpleExtension.sjs create mode 100644 test/test_put_file.py diff --git a/test/data/simpleExtension.sjs b/test/data/simpleExtension.sjs new file mode 100644 index 0000000..6518f6b --- /dev/null +++ b/test/data/simpleExtension.sjs @@ -0,0 +1,13 @@ +function metadataFilter(context, params, content) { + var mutableDoc = content.toObject(); + xdmp.log("context"); + xdmp.log(context); + xdmp.log("params"); + xdmp.log(params); + xdmp.log("content"); + xdmp.log(content); + content = xdmp.unquote('hello', null, ["repair-none", "default-language=en"]); + return content; +}; + +exports.transform = metadataFilter; \ No newline at end of file diff --git a/test/test_put_file.py b/test/test_put_file.py new file mode 100644 index 0000000..8a7913a --- /dev/null +++ b/test/test_put_file.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function, absolute_import + +# +# Copyright 2016 MarkLogic Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0# +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# File History +# ------------ +# +# Philip Barber 02/15/2016 Initial development +# + +import unittest +from requests.auth import HTTPDigestAuth +from marklogic.models import Database, Server, Forest +from marklogic.connection import Connection +from test.ssl.resources import TestConnection as tc + +class TestPutFile(unittest.TestCase): + """ + Basic put-file test function. + """ + def test_put_file(self): + conn = Connection(tc.hostname, HTTPDigestAuth(tc.admin, tc.password)) + + response = conn.get("http://192.168.200.162:8004/LATEST/config/transforms/simpleExtension") + self.assertIn("\"status\":\"Not Found\"", response.text) + + data = open("./test/data/simpleExtension.sjs", 'rb').read() + response = conn.putFile(uri="http://192.168.200.162:8004/LATEST/config/transforms/simpleExtension", data=data, content_type="application/vnd.marklogic-javascript") + self.assertEqual(204, response.status_code, "PUTting the file should result in a 204 response code") + + response = conn.get("http://192.168.200.162:8004/LATEST/config/transforms/simpleExtension") + self.assertIn("function metadataFilter", response.text) + + response = conn.delete("http://192.168.200.162:8004/LATEST/config/transforms/simpleExtension") + self.assertEqual(204, response.status_code, "DELETing the file should result in a 204 response code") + +if __name__ == "__main__": + unittest.main() From 971351d9fc7dfe6bd618f4aec8ca72ea7a728a3c Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 15 Feb 2016 19:20:14 -0500 Subject: [PATCH 8/8] Merging the restloader class from the branch as well as the example, deployFile.py. --- examples/deployFile.py | 67 ++++++++++++++++++++++++++++++++++++++++++ examples/restloader.py | 53 +++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 examples/deployFile.py create mode 100644 examples/restloader.py diff --git a/examples/deployFile.py b/examples/deployFile.py new file mode 100644 index 0000000..2dcd672 --- /dev/null +++ b/examples/deployFile.py @@ -0,0 +1,67 @@ +#!/usr/bin/python3 +# +# Copyright 2016 MarkLogic Corporation +# +# This script provides examples of deploying files to a MarkLogic database without using MLCP +# This is useful when the Application Servers within MarkLogic have SSL enabled. +# +# For example: +# +# python3 deployFile + +__author__ = 'pmb' + +import argparse +import os +import requests +from requests.auth import HTTPDigestAuth +from marklogic.connection import Connection +from restloader import RESTLoader + +parser = argparse.ArgumentParser() +parser.add_argument("--host", action='store', default="192.168.200.162", + help="Management API host") +parser.add_argument("--restPort", action='store', default="8000", + help="REST port for modules DB") +parser.add_argument("--protocol", action='store', default="http", + help="http or https") +parser.add_argument("--username", action='store', default="admin", + help="User name") +parser.add_argument("--password", action='store', default="admin", + help="Password") +parser.add_argument("--source", action='store', default="data", + help="file or root directory to deploy") +parser.add_argument("--targetUri", action='store', default="/app", + help="root path for the files in MarkLogic") +parser.add_argument("--certificate", action='store', default=None, + help="root path for the files in MarkLogic") +parser.add_argument("--database", action='store', default=None, + help="root path for the files in MarkLogic") +parser.add_argument("--permissions", action='store', default=True, + help="Permissions for the files in MarkLogic") +args = parser.parse_args() + +source = args.source +targetUri = args.targetUri + +conn = Connection(args.host, HTTPDigestAuth(args.username, args.password), args.protocol, verify=args.certificate) +requests.packages.urllib3.disable_warnings() + +restLoader = RESTLoader(conn, args.restPort, args.database, args.permissions) + +if (source is not None): + if (os.path.isfile(source)): + response = restLoader.load_file(source, targetUri) + if (response.status_code == 201): + print("CREATED: " + targetUri) + elif (response.status_code == 204): + print("UPDATED: " + targetUri) + else: + print("Unexpected Response:"+response) + elif (os.path.isdir(source)): + restLoader.load_directory(source, targetUri) + else: + print("The specified source is either not found or not a file/directory") +else: + print("You must specify the source file/directory") +print("Complete") \ No newline at end of file diff --git a/examples/restloader.py b/examples/restloader.py new file mode 100644 index 0000000..582db86 --- /dev/null +++ b/examples/restloader.py @@ -0,0 +1,53 @@ +import logging +import os +from os import listdir +from os.path import isfile, join +from test.test_threadedtempfile import FILES_PER_THREAD + +class RESTLoader(): + """ + This class will upload files using the REST API. + """ + def __init__(self, conn, restPort, database, permissions): + self.logger = logging.getLogger("marklogic.examples") + self.conn = conn + self.restPort = restPort + self.database = database + self.permissions = permissions + pass + + def build_uri(self, targetUri): + baseUri = self.conn.protocol + "://"+self.conn.host+":" + str(self.restPort) + "/LATEST/documents" + uri = baseUri + "?uri=" + targetUri + if (self.database is not None): + uri += "&database=" + self.database + if (self.permissions is not None): + for perm in self.permissions.split(","): + uri += "&perm:" + perm + return uri + + def load_file(self, sourceFile, targetUri): + self.logger.info("Loading {0} to {1}".format(sourceFile, targetUri)) + uri = self.build_uri(targetUri) + print(uri) + data = open(sourceFile, 'rb').read() + response = self.conn.putFile(uri=uri, data=data, content_type="test/text") + if (response.status_code == 201): + print("CREATED: " + targetUri) + elif (response.status_code == 204): + print("UPDATED: " + targetUri) + else: + print("Unexpected Response:"+str(response.status_code)) + return response + + def load_directory(self, sourceDirectory, targetUri): + self.logger.info("Loading {0} to {1}".format(sourceDirectory, targetUri)) + files = [f for f in listdir(sourceDirectory) if isfile(join(sourceDirectory, f))] + for file in files: + print(file) + response = self.load_file(sourceDirectory+os.path.sep+file, targetUri+"/"+file) + directories = [d for d in listdir(sourceDirectory) if os.path.isdir(sourceDirectory+os.path.sep+d)] + for directory in directories: + print(directory) + response = self.load_directory(sourceDirectory+os.path.sep+directory, targetUri+"/"+directory) + \ No newline at end of file