From e98786a478bd6a8abea257d23fa89542f7d781ef Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Fri, 17 Nov 2023 14:45:40 +0100 Subject: [PATCH 01/41] Get rid of MONETURL constant it has two uses which are going to be changed in different ways --- src/main/java/org/monetdb/jdbc/MonetConnection.java | 2 +- src/main/java/org/monetdb/jdbc/MonetDriver.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java index 5c171bc..3f095d7 100644 --- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -1801,7 +1801,7 @@ boolean mapClobAsVarChar() { */ String getJDBCURL() { final StringBuilder sb = new StringBuilder(128); - sb.append(MonetDriver.MONETURL).append(hostname) + sb.append("jdbc:monetdb://").append(hostname) .append(':').append(port) .append('/').append(database); if (lang == LANG_MAL) diff --git a/src/main/java/org/monetdb/jdbc/MonetDriver.java b/src/main/java/org/monetdb/jdbc/MonetDriver.java index f6f4681..367a2d3 100644 --- a/src/main/java/org/monetdb/jdbc/MonetDriver.java +++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java @@ -42,8 +42,6 @@ public final class MonetDriver implements Driver { // the url kind will be jdbc:monetdb://[:]/ // Chapter 9.2.1 from Sun JDBC 3.0 specification - /** The prefix of a MonetDB url */ - static final String MONETURL = "jdbc:monetdb://"; // initialize this class: register it at the DriverManager // Chapter 9.2 from Sun JDBC 3.0 specification @@ -67,7 +65,7 @@ public final class MonetDriver implements Driver { */ @Override public boolean acceptsURL(final String url) { - return url != null && url.startsWith(MONETURL); + return url != null && url.startsWith("jdbc:monetdb://"); } /** From f1de7cecd443d627f0c001a0544c01f2e1efd030 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Wed, 29 Nov 2023 13:28:52 +0100 Subject: [PATCH 02/41] URL parser passes the tests. Some tests had to change to accomodate Java. --- .../org/monetdb/mcl/net/MonetUrlParser.java | 305 ++++ .../java/org/monetdb/mcl/net/Parameter.java | 81 + .../org/monetdb/mcl/net/ParameterType.java | 68 + src/main/java/org/monetdb/mcl/net/Target.java | 330 ++++ .../org/monetdb/mcl/net/ValidationError.java | 11 + src/main/java/org/monetdb/mcl/net/Verify.java | 8 + tests/UrlTester.java | 373 +++++ tests/tests.md | 1458 +++++++++++++++++ 8 files changed, 2634 insertions(+) create mode 100644 src/main/java/org/monetdb/mcl/net/MonetUrlParser.java create mode 100644 src/main/java/org/monetdb/mcl/net/Parameter.java create mode 100644 src/main/java/org/monetdb/mcl/net/ParameterType.java create mode 100644 src/main/java/org/monetdb/mcl/net/Target.java create mode 100644 src/main/java/org/monetdb/mcl/net/ValidationError.java create mode 100644 src/main/java/org/monetdb/mcl/net/Verify.java create mode 100644 tests/UrlTester.java create mode 100644 tests/tests.md diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java new file mode 100644 index 0000000..da0d53d --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java @@ -0,0 +1,305 @@ +package org.monetdb.mcl.net; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.util.Properties; + +public class MonetUrlParser { + private final Properties props; + private final String urlText; + private final URI url; + boolean userWasSet = false; + boolean passwordWasSet = false; + + public MonetUrlParser(Properties props, String url) throws URISyntaxException { + this.props = props; + this.urlText = url; + this.url = new URI(url); + } + + public static void parse(Properties props, String url) throws URISyntaxException { + boolean modern = true; + if (url.startsWith("mapi:")) { + modern = false; + url = url.substring(5); + if (url.equals("monetdb://")) { + // deal with peculiarity of Java's URI parser + url = "monetdb:///"; + } + + } + try { + MonetUrlParser parser = new MonetUrlParser(props, url); + if (modern) { + parser.parseModern(); + } else { + parser.parseClassic(); + } + if (parser.userWasSet && !parser.passwordWasSet) parser.clear(Parameter.PASSWORD); + } catch (URISyntaxException e) { + int idx = e.getIndex(); + if (idx >= 0 && !modern) { + // "mapi:" + idx += 5; + } + throw new URISyntaxException(e.getInput(), e.getReason(), idx); + } + } + + private static String percentDecode(String context, String text) throws URISyntaxException { + try { + return URLDecoder.decode(text, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("should be unreachable: UTF-8 unknown??", e); + } catch (IllegalArgumentException e) { + throw new URISyntaxException(text, context + ": invalid percent escape"); + } + } + + private void set(Parameter parm, String value) { + parm = keyMagic(parm); + props.setProperty(parm.name, value != null ? value : ""); + } + + private void set(String key, String value) { + Parameter parm = Parameter.forName(key); + if (parm != null) + set(parm, value); + else + props.setProperty(key, value); + } + + private void clear(Parameter parm) { + parm = keyMagic(parm); + String value = parm.type.format(Target.getDefault(parm)); + props.setProperty(parm.name, value); + } + + private Parameter keyMagic(Parameter key) { + switch (key) { + case USER: + userWasSet = true; + break; + case PASSWORD: + passwordWasSet = true; + break; + case FETCHSIZE: + key = Parameter.REPLYSIZE; + break; + default: + break; + } + return key; + } + + private void parseModern() throws URISyntaxException { + clearBasic(); + + String scheme = url.getScheme(); + if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); + switch (scheme) { + case "monetdb": + set(Parameter.TLS, "false"); + break; + case "monetdbs": + set(Parameter.TLS, "true"); + break; + default: + throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); + } + + // The built-in getHost and getPort methods do strange things + // in edge cases such as percent-encoded host names and + // invalid port numbers + String authority = url.getAuthority(); + String host; + String remainder; + int pos; + String raw = url.getRawSchemeSpecificPart(); + if (authority == null) { + if (!url.getRawSchemeSpecificPart().startsWith("//")) { + throw new URISyntaxException(urlText, "expected //"); + } + host = ""; + remainder = ""; + } else if (authority.startsWith("[")) { + // IPv6 + pos = authority.indexOf(']'); + if (pos < 0) + throw new URISyntaxException(urlText, "unmatched '['"); + host = authority.substring(1, pos); + remainder = authority.substring(pos + 1); + } else if ((pos = authority.indexOf(':')) >= 0){ + host = authority.substring(0, pos); + remainder = authority.substring(pos); + } else { + host = authority; + remainder = ""; + } + switch (host) { + case "localhost": + set(Parameter.HOST, ""); + break; + case "localhost.": + set(Parameter.HOST, "localhost"); + break; + default: + set(Parameter.HOST, host); + break; + } + + if (remainder.isEmpty()) { + // do nothing + } else if (remainder.startsWith(":")) { + String portStr = remainder.substring(1); + try { + int port = Integer.parseInt(portStr); + if (port <= 0 || port > 65535) + portStr = null; + } catch (NumberFormatException e) { + portStr = null; + } + if (portStr == null) + throw new URISyntaxException(urlText, "invalid port number"); + set(Parameter.PORT, portStr); + } + + String path = url.getRawPath(); + String[] parts = path.split("/", 5); + // <0: empty before leading slash> / <1: database> / <2: tableschema> / <3: table> / <4: should not exist> + switch (parts.length) { + case 5: + throw new URISyntaxException(urlText, "table name should not contain slashes"); + case 4: + set(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3])); + // fallthrough + case 3: + set(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2])); + // fallthrough + case 2: + set(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1])); + case 1: + case 0: + // fallthrough + break; + } + + final String query = url.getRawQuery(); + if (query != null) { + final String args[] = query.split("&"); + for (int i = 0; i < args.length; i++) { + pos = args[i].indexOf('='); + if (pos <= 0) { + throw new URISyntaxException(args[i], "invalid key=value pair"); + } + String key = args[i].substring(0, pos); + key = percentDecode(key, key); + Parameter parm = Parameter.forName(key); + if (parm != null && parm.isCore) + throw new URISyntaxException(key, key + "= is not allowed as a query parameter"); + + String value = args[i].substring(pos + 1); + set(key, percentDecode(key, value)); + } + } + } + + + private void parseClassic() throws URISyntaxException { + String scheme = url.getScheme(); + if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); + switch (scheme) { + case "monetdb": + clearBasic(); + break; + case "merovingian": + throw new IllegalStateException("mapi:merovingian: not supported yet"); + default: + throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); + } + + if (!url.getRawSchemeSpecificPart().startsWith("//")) { + throw new URISyntaxException(urlText, "expected //"); + } + + String authority = url.getRawAuthority(); + String host; + String portStr; + int pos; + if (authority == null) { + host = ""; + portStr = ""; + } else if (authority.indexOf('@') >= 0) { + throw new URISyntaxException(urlText, "user@host syntax is not allowed"); + } else if ((pos = authority.indexOf(':')) >= 0) { + host = authority.substring(0, pos); + portStr = authority.substring(pos + 1); + } else { + host = authority; + portStr = ""; + } + + if (!portStr.isEmpty()) { + int port; + try { + port = Integer.parseInt(portStr); + } catch (NumberFormatException e) { + port = -1; + } + if (port <= 0) { + throw new URISyntaxException(urlText, "invalid port number"); + } + set(Parameter.PORT, portStr); + } + + String path = url.getRawPath(); + boolean isUnix; + if (host.isEmpty() && portStr.isEmpty()) { + // socket + isUnix = true; + clear(Parameter.HOST); + set(Parameter.SOCK, path != null ? path : ""); + } else { + // tcp + isUnix = false; + clear(Parameter.SOCK); + set(Parameter.HOST, host); + if (path == null || path.isEmpty()) { + // do nothing + } else if (!path.startsWith("/")) { + throw new URISyntaxException(urlText, "expect path to start with /"); + } else { + String database = path.substring(1); + if (database.contains("/")) + throw new URISyntaxException(urlText, "no slashes allowed in database name"); + set(Parameter.DATABASE, database); + } + } + + final String query = url.getRawQuery(); + if (query != null) { + final String args[] = query.split("&"); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("language=")) { + String language = arg.substring(9); + set(Parameter.LANGUAGE, language); + } else if (arg.startsWith("database=")) { + String database = arg.substring(9); + set(Parameter.DATABASE, database); + } else { + // ignore + } + } + } + } + + private void clearBasic() { + clear(Parameter.HOST); + clear(Parameter.PORT); + clear(Parameter.SOCK); + clear(Parameter.DATABASE); + } +} diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java new file mode 100644 index 0000000..c7a70b8 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/Parameter.java @@ -0,0 +1,81 @@ +package org.monetdb.mcl.net; + + +public enum Parameter { + TLS("tls", ParameterType.Bool, false, "secure the connection using TLS", true), + HOST("host", ParameterType.Str, "", "IP number, domain name or one of the special values `localhost` and `localhost.`", true), + PORT("port", ParameterType.Int, -1, "Port to connect to, 1..65535 or -1 for 'not set'", true), + DATABASE("database", ParameterType.Str, "", "name of database to connect to", true), + TABLESCHEMA("tableschema", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true), + TABLE("table", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true), + SOCK("sock", ParameterType.Path, "", "path to Unix domain socket to connect to", false), + SOCKDIR("sockdir", ParameterType.Path, "/tmp", "Directory for implicit Unix domain sockets (.s.monetdb.PORT)", false), + CERT("cert", ParameterType.Path, "", "path to TLS certificate to authenticate server with", false), + CERTHASH("certhash", ParameterType.Str, "", "hash of server TLS certificate must start with these hex digits; overrides cert", false), + CLIENTKEY("clientkey", ParameterType.Path, "", "path to TLS key (+certs) to authenticate with as client", false), + CLIENTCERT("clientcert", ParameterType.Path, "", "path to TLS certs for 'clientkey', if not included there", false), + USER("user", ParameterType.Str, "", "user name to authenticate as", false), + PASSWORD("password", ParameterType.Str, "", "password to authenticate with", false), + LANGUAGE("language", ParameterType.Str, "sql", "for example, \"sql\", \"mal\", \"msql\", \"profiler\"", false), + AUTOCOMMIT("autocommit", ParameterType.Bool, false, "initial value of autocommit", false), + SCHEMA("schema", ParameterType.Str, "", "initial schema", false), + TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false), + BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false), + REPLYSIZE("replysize", ParameterType.Int, 200, "rows beyond this limit are retrieved on demand, <1 means unlimited", false), + FETCHSIZE("fetchsize", ParameterType.Int, null, "alias for replysize, specific to jdbc", false), + HASH("hash", ParameterType.Str, "", "specific to jdbc", false), + DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false), + LOGFILE("logfile", ParameterType.Str, "", "specific to jdbc", false), + + ; + + public final String name; + public final ParameterType type; + public final Object defaultValue; + public final String description; + public final boolean isCore; + + Parameter(String name, ParameterType type, Object defaultValue, String description, boolean isCore) { + this.name = name; + this.type = type; + this.isCore = isCore; + this.defaultValue = defaultValue; + this.description = description; + } + + public static Parameter forName(String name) { + switch (name) { + case "tls": return TLS; + case "host": return HOST; + case "port": return PORT; + case "database": return DATABASE; + case "tableschema": return TABLESCHEMA; + case "table": return TABLE; + case "sock": return SOCK; + case "sockdir": return SOCKDIR; + case "cert": return CERT; + case "certhash": return CERTHASH; + case "clientkey": return CLIENTKEY; + case "clientcert": return CLIENTCERT; + case "user": return USER; + case "password": return PASSWORD; + case "language": return LANGUAGE; + case "autocommit": return AUTOCOMMIT; + case "schema": return SCHEMA; + case "timezone": return TIMEZONE; + case "binary": return BINARY; + case "replysize": return REPLYSIZE; + case "fetchsize": return FETCHSIZE; + case "hash": return HASH; + case "debug": return DEBUG; + case "logfile": return LOGFILE; + default: return null; + } + } + + public static boolean isIgnored(String name) { + if (Parameter.forName(name) != null) + return false; + return name.contains("_"); + } +} diff --git a/src/main/java/org/monetdb/mcl/net/ParameterType.java b/src/main/java/org/monetdb/mcl/net/ParameterType.java new file mode 100644 index 0000000..8b7a57f --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/ParameterType.java @@ -0,0 +1,68 @@ +package org.monetdb.mcl.net; + +public enum ParameterType { + Str, + Int, + Bool, + Path; + + public Object parse(String name, String value) throws ValidationError { + if (value == null) + throw new NullPointerException(); + + try { + switch (this) { + case Bool: + return parseBool(value); + case Int: + return Integer.parseInt(value); + case Str: + case Path: + return value; + default: + throw new IllegalStateException("unreachable"); + } + } catch (IllegalArgumentException e) { + String message = e.toString(); + throw new ValidationError(name, message); + } + } + + public String format(Object value) { + switch (this) { + case Bool: + return (Boolean)value ? "true": "false"; + case Int: + return Integer.toString((Integer)value); + case Str: + case Path: + return (String) value; + default: + throw new IllegalStateException("unreachable"); + } + } + + public static boolean parseBool(String value) { + boolean lowered = false; + String original = value; + while (true) { + switch (value) { + case "true": + case "yes": + case "on": + return true; + case "false": + case "no": + case "off": + return false; + default: + if (!lowered) { + value = value.toLowerCase(); + lowered = true; + continue; + } + throw new IllegalArgumentException("invalid boolean value: " + original); + } + } + } +} diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java new file mode 100644 index 0000000..57d44a8 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -0,0 +1,330 @@ +package org.monetdb.mcl.net; + +import java.util.Calendar; +import java.util.Properties; +import java.util.regex.Pattern; + +public class Target { + private static Pattern namePattern = Pattern.compile("^[a-zA-Z_][-a-zA-Z0-9_.]*$"); + private static Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$"); + private final boolean tls; + private final String host; + private final int port; + private final String database; + private final String tableschema; + private final String table; + private final String sock; + private final String sockdir; + private final String cert; + private final String certhash; + private final String clientkey; + private final String clientcert; + private final String user; + private final String password; + private final String language; + private final boolean autocommit; + private final String schema; + private final int timezone; + private final int binary; + private final int replysize; + private final String hash; + private final boolean debug; + private final String logfile; + + public Target(Properties properties) throws ValidationError { + + // 1. The parameters have the types listed in the table in [Section + // Parameters](#parameters). + tls = validateBoolean(properties, Parameter.TLS); + host = validateString(properties, Parameter.HOST); + port = validateInt(properties, Parameter.PORT); + database = validateString(properties, Parameter.DATABASE); + tableschema = validateString(properties, Parameter.TABLESCHEMA); + table = validateString(properties, Parameter.TABLE); + sock = validateString(properties, Parameter.SOCK); + sockdir = validateString(properties, Parameter.SOCKDIR); + cert = validateString(properties, Parameter.CERT); + certhash = validateString(properties, Parameter.CERTHASH); + clientkey = validateString(properties, Parameter.CLIENTKEY); + clientcert = validateString(properties, Parameter.CLIENTCERT); + user = validateString(properties, Parameter.USER); + password = validateString(properties, Parameter.PASSWORD); + language = validateString(properties, Parameter.LANGUAGE); + autocommit = validateBoolean(properties, Parameter.AUTOCOMMIT); + schema = validateString(properties, Parameter.SCHEMA); + timezone = validateInt(properties, Parameter.TIMEZONE); + replysize = validateInt(properties, Parameter.REPLYSIZE); + hash = validateString(properties, Parameter.HASH); + debug = validateBoolean(properties, Parameter.DEBUG); + logfile = validateString(properties, Parameter.LOGFILE); + + for (String name: properties.stringPropertyNames()) { + if (Parameter.forName(name) != null) + continue; + if (name.contains("_")) + continue; + throw new ValidationError("unknown parameter: " + name); + } + + String binaryString = validateString(properties, Parameter.BINARY); + int binaryInt; + try { + binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString); + } catch (ValidationError e) { + try { + boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString); + binaryInt = b ? 65535 : 0; + } catch (ValidationError ee) { + throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off"); + } + } + if (binaryInt < 0) + throw new ValidationError("binary= cannot be negative"); + binary = binaryInt; + + + // 2. At least one of **sock** and **host** must be empty. + if (!sock.isEmpty() && !host.isEmpty()) + throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host); + + // 3. The string parameter **binary** must either parse as a boolean or as a + // non-negative integer. + // + // (checked above) + + // 4. If **sock** is not empty, **tls** must be 'off'. + if (!sock.isEmpty() && tls) throw new ValidationError("monetdbs:// cannot be combined with sock="); + + // 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits` + // where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons. + // TODO + if (!certhash.isEmpty()) { + if (!certhash.toLowerCase().startsWith("sha256:")) + throw new ValidationError("certificate hash must start with 'sha256:'"); + if (!hashPattern.matcher(certhash).matches()) + throw new ValidationError("invalid certificate hash"); + } + + // 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well. + if (!tls) { + if (!cert.isEmpty() || !certhash.isEmpty()) + throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://"); + } + + // 7. Parameters **database**, **tableschema** and **table** must consist only of + // upper- and lowercase letters, digits, periods, dashes and underscores. They must not + // start with a dash. + // If **table** is not empty, **tableschema** must also not be empty. + // If **tableschema** is not empty, **database** must also not be empty. + if (database.isEmpty() && !tableschema.isEmpty()) + throw new ValidationError("table schema cannot be set without database"); + if (tableschema.isEmpty() && !table.isEmpty()) + throw new ValidationError("table cannot be set without schema"); + if (!database.isEmpty() && !namePattern.matcher(database).matches()) + throw new ValidationError("invalid database name"); + if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches()) + throw new ValidationError("invalid table schema name"); + if (!table.isEmpty() && !namePattern.matcher(table).matches()) + throw new ValidationError("invalid table name"); + + + // 8. Parameter **port** must be -1 or in the range 1-65535. + if (port < -1 || port == 0 || port > 65535) throw new ValidationError("invalid port number " + port); + + // 9. If **clientcert** is set, **clientkey** must also be set. + if (!clientcert.isEmpty() && clientkey.isEmpty()) + throw new ValidationError("clientcert= is only valid in combination with clientkey="); + } + + public static boolean validateBoolean(Properties props, Parameter parm) throws ValidationError { + Object value = props.get(parm.name); + if (value != null) { + return (Boolean) parm.type.parse(parm.name, (String) value); + } else { + return (Boolean) getDefault(parm); + } + } + + public static int validateInt(Properties props, Parameter parm) throws ValidationError { + Object value = props.get(parm.name); + if (value != null) { + return (Integer) parm.type.parse(parm.name, (String) value); + } else { + return (Integer) getDefault(parm); + } + } + + public static String validateString(Properties props, Parameter parm) throws ValidationError { + Object value = props.get(parm.name); + if (value != null) { + return (String) parm.type.parse(parm.name, (String) value); + } else { + return (String) getDefault(parm); + } + } + + private static int timezone() { + Calendar cal = Calendar.getInstance(); + int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); + int offsetSeconds = offsetMillis / 1000; + return offsetSeconds; + } + + public static Object getDefault(Parameter parm) { + if (parm == Parameter.TIMEZONE) return timezone(); + else return parm.defaultValue; + } + + public static Properties defaultProperties() { + Properties props = new Properties(); + return props; + } + + public boolean getTls() { + return tls; + } + + // Getter is private because you probably want connectTcp() instead + private String getHost() { + return host; + } + + // Getter is private because you probably want connectPort() instead + private int getPort() { + return port; + } + + public String getDatabase() { + return database; + } + + public String getTableschema() { + return tableschema; + } + + public String getTable() { + return table; + } + + // Getter is private because you probably want connectUnix() instead + private String getSock() { + return sock; + } + + public String getSockdir() { + return sockdir; + } + + public String getCert() { + return cert; + } + + public String getCerthash() { + return certhash; + } + + public String getClientkey() { + return clientkey; + } + + public String getClientcert() { + return clientcert; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public String getLanguage() { + return language; + } + + public boolean getAutocommit() { + return autocommit; + } + + public String getSchema() { + return schema; + } + + public int getTimezone() { + return timezone; + } + + // Getter is private because you probably want connectBinary() instead + public int getBinary() { + return binary; + } + + public int getReplysize() { + return replysize; + } + + public String getHash() { + return hash; + } + + public boolean getDebug() { + return debug; + } + + public String getLogfile() { + return logfile; + } + + public boolean connectScan() { + if (database.isEmpty()) return false; + if (!sock.isEmpty() || !host.isEmpty() || port != -1) return false; + return !tls; + } + + public int connectPort() { + return port == -1 ? 50000 : port; + } + + public String connectUnix() { + if (!sock.isEmpty()) return sock; + if (tls) return ""; + if (host.isEmpty()) return sockdir + "/.s.monetdb." + connectPort(); + return ""; + } + + public String connectTcp() { + if (!sock.isEmpty()) return ""; + if (host.isEmpty()) return "localhost"; + return host; + } + + public Verify connectVerify() { + if (!tls) return Verify.None; + if (!certhash.isEmpty()) return Verify.Hash; + if (!cert.isEmpty()) return Verify.Cert; + return Verify.System; + } + + public String connectCertHashDigits() { + if (!tls) return null; + StringBuilder builder = new StringBuilder(certhash.length()); + for (int i = "sha256:".length(); i < certhash.length(); i++) { + char c = certhash.charAt(i); + if (Character.digit(c, 16) >= 0) builder.append(Character.toLowerCase(c)); + } + return builder.toString(); + } + + public int connectBinary() { + return binary; + } + + public String connectClientKey() { + return clientkey; + } + + public String connectClientCert() { + return clientcert.isEmpty() ? clientkey : clientcert; + } +} diff --git a/src/main/java/org/monetdb/mcl/net/ValidationError.java b/src/main/java/org/monetdb/mcl/net/ValidationError.java new file mode 100644 index 0000000..bfc60f4 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/ValidationError.java @@ -0,0 +1,11 @@ +package org.monetdb.mcl.net; + +public class ValidationError extends Exception { + public ValidationError(String parameter, String message) { + super(parameter + ": " + message); + } + + public ValidationError(String message) { + super(message); + } +} diff --git a/src/main/java/org/monetdb/mcl/net/Verify.java b/src/main/java/org/monetdb/mcl/net/Verify.java new file mode 100644 index 0000000..457c20e --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/Verify.java @@ -0,0 +1,8 @@ +package org.monetdb.mcl.net; + +public enum Verify { + None, + Cert, + Hash, + System; +} diff --git a/tests/UrlTester.java b/tests/UrlTester.java new file mode 100644 index 0000000..2b79b88 --- /dev/null +++ b/tests/UrlTester.java @@ -0,0 +1,373 @@ +import org.monetdb.mcl.net.*; + +import java.io.*; +import java.net.URISyntaxException; +import java.util.Properties; + +public class UrlTester { + String filename = null; + int verbose = 0; + BufferedReader reader = null; + int lineno = 0; + int testCount = 0; + Properties props = null; + Target validated = null; + + public UrlTester() { + } + + public UrlTester(String filename) { + this.filename = filename; + } + + public static void main(String[] args) throws Exception { + int exitcode; + UrlTester tester = new UrlTester(); + exitcode = tester.parseArgs(args); + if (exitcode == 0) + exitcode = tester.run(); + System.exit(exitcode); + } + + private int parseArgs(String[] args) { + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("-")) { + int result = handleFlags(arg); + if (result != 0) + return result; + } else if (filename == null) { + filename = arg; + } else { + System.err.println("Unexpected argument: " + arg); + return 1; + } + } + return 0; + } + + private int run() throws IOException { + if (filename != null) { + reader = new BufferedReader(new FileReader(filename)); + } else { + String resourceName = "/tests.md"; + InputStream stream = this.getClass().getResourceAsStream(resourceName); + if (stream == null) { + System.err.println("Resource " + resourceName + " not found"); + return 1; + } + reader = new BufferedReader(new InputStreamReader(stream)); + filename = "tests/tests.md"; + } + + try { + processFile(); + } catch (Failure e) { + System.err.println("" + filename + ":" + lineno + ": " + e.getMessage()); + return 1; + } + return 0; + } + + private int handleFlags(String arg) { + if (!arg.startsWith("-") || arg.equals("-")) { + System.err.println("Invalid flag: " + arg); + } + String a = arg.substring(1); + + while (!a.isEmpty()) { + char letter = a.charAt(0); + a = a.substring(1); + switch (letter) { + case 'v': + verbose++; + break; + default: + System.err.println("Unexpected flag " + letter + " in " + arg); + return -1; + } + } + + return 0; + } + + private void processFile() throws IOException, Failure { + while (true) { + String line = reader.readLine(); + if (line == null) + break; + lineno++; + processLine(line); + } + if (verbose >= 1) { + System.out.println(); + System.out.println("Ran " + testCount + " tests in " + lineno + " lines"); + } + } + + private void processLine(String line) throws Failure { + line = line.replaceFirst("\\s+$", ""); // remove trailing + if (props == null && line.equals("```test")) { + if (verbose >= 2) { + if (testCount > 0) { + System.out.println(); + } + System.out.println("\u25B6 " + filename + ":" + lineno); + } + props = Target.defaultProperties(); + testCount++; + return; + } + if (props != null) { + if (line.equals("```")) { + stopProcessing(); + return; + } + handleCommand(line); + } + } + + private void stopProcessing() { + props = null; + validated = null; + } + + private void handleCommand(String line) throws Failure { + if (verbose >= 3) { + System.out.println(line); + } + if (line.isEmpty()) + return; + + String[] parts = line.split("\\s+", 2); + String command = parts[0]; + switch (command.toUpperCase()) { + case "ONLY": + handleOnly(true, parts[1]); + return; + case "NOT": + handleOnly(false, parts[1]); + return; + case "PARSE": + handleParse(parts[1], null); + return; + case "ACCEPT": + handleParse(parts[1], true); + return; + case "REJECT": + handleParse(parts[1], false); + return; + case "SET": + handleSet(parts[1]); + return; + case "EXPECT": + handleExpect(parts[1]); + return; + default: + throw new Failure("Unexpected command: " + command); + } + + } + + private void handleOnly(boolean mustBePresent, String rest) throws Failure { + boolean found = false; + for (String part: rest.split("\\s+")) { + if (part.equals("jdbc")) { + found = true; + break; + } + } + if (found != mustBePresent) { + // do not further process this block + stopProcessing(); + } + } + + private int findEqualSign(String rest) throws Failure { + int index = rest.indexOf('='); + if (index < -1) + throw new Failure("Expected to find a '='"); + return index; + } + + private String splitKey(String rest) throws Failure { + int index = findEqualSign(rest); + return rest.substring(0, index); + } + + private String splitValue(String rest) throws Failure { + int index = findEqualSign(rest); + return rest.substring(index + 1); + } + + private void handleSet(String rest) throws Failure { + validated = null; + String key = splitKey(rest); + String value = splitValue(rest); + + props.put(key, value); + } + + private void handleParse(String rest, Boolean shouldSucceed) throws Failure { + URISyntaxException parseError = null; + ValidationError validationError = null; + + validated = null; + try { + MonetUrlParser.parse(props, rest); + } catch (URISyntaxException e) { + parseError = e; + } + + if (parseError == null) { + try { + validated = new Target(props); + } catch (ValidationError e) { + validationError = e; + } + } + + if (shouldSucceed == Boolean.FALSE) { + if (parseError != null || validationError != null) + return; // happy + else + throw new Failure("URL unexpectedly parsed and validated"); + } + + if (parseError != null) + throw new Failure("Parse error: " + parseError); + if (validationError != null && shouldSucceed == Boolean.TRUE) + throw new Failure("Validation error: " + validationError); + } + + private void handleExpect(String rest) throws Failure { + String key = splitKey(rest); + String expectedString = splitValue(rest); + + Object actual = null; + try { + actual = extract(key); + } catch (ValidationError e) { + throw new Failure(e.getMessage()); + } + + Object expected; + try { + if (actual instanceof Boolean) + expected = ParameterType.Bool.parse(key, expectedString); + else if (actual instanceof Integer) + expected = ParameterType.Int.parse(key, expectedString); + else + expected = expectedString; + } catch (ValidationError e) { + String typ = actual.getClass().getName(); + throw new Failure("Cannot convert expected value <" + expectedString + "> to " + typ + ": " + e.getMessage()); + } + + if (actual.equals(expected)) + return; + throw new Failure("Expected " + key + "=<" + expectedString + ">, found <" + actual + ">"); + } + + private Target tryValidate() throws ValidationError { + if (validated == null) + validated = new Target(props); + return validated; + } + + private Object extract(String key) throws ValidationError, Failure { + switch (key) { + case "tls": + return Target.validateBoolean(props, Parameter.TLS); + case "host": + return Target.validateString(props, Parameter.HOST); + case "port": + return Target.validateInt(props, Parameter.PORT); + case "database": + return Target.validateString(props, Parameter.DATABASE); + case "tableschema": + return Target.validateString(props, Parameter.TABLESCHEMA); + case "table": + return Target.validateString(props, Parameter.TABLE); + case "sock": + return Target.validateString(props, Parameter.SOCK); + case "sockdir": + return Target.validateString(props, Parameter.SOCKDIR); + case "cert": + return Target.validateString(props, Parameter.CERT); + case "certhash": + return Target.validateString(props, Parameter.CERTHASH); + case "clientkey": + return Target.validateString(props, Parameter.CLIENTKEY); + case "clientcert": + return Target.validateString(props, Parameter.CLIENTCERT); + case "user": + return Target.validateString(props, Parameter.USER); + case "password": + return Target.validateString(props, Parameter.PASSWORD); + case "language": + return Target.validateString(props, Parameter.LANGUAGE); + case "autocommit": + return Target.validateBoolean(props, Parameter.AUTOCOMMIT); + case "schema": + return Target.validateString(props, Parameter.SCHEMA); + case "timezone": + return Target.validateInt(props, Parameter.TIMEZONE); + case "binary": + return Target.validateString(props, Parameter.BINARY); + case "replysize": + return Target.validateInt(props, Parameter.REPLYSIZE); + case "fetchsize": + return Target.validateInt(props, Parameter.FETCHSIZE); + case "hash": + return Target.validateString(props, Parameter.HASH); + case "debug": + return Target.validateBoolean(props, Parameter.DEBUG); + case "logfile": + return Target.validateString(props, Parameter.LOGFILE); + + case "valid": + try { + tryValidate(); + } catch (ValidationError e) { + return Boolean.FALSE; + } + return Boolean.TRUE; + + case "connect_scan": + return tryValidate().connectScan(); + case "connect_port": + return tryValidate().connectPort(); + case "connect_unix": + return tryValidate().connectUnix(); + case "connect_tcp": + return tryValidate().connectTcp(); + case "connect_tls_verify": + switch (tryValidate().connectVerify()) { + case None: return ""; + case Cert: return "cert"; + case Hash: return "hash"; + case System: return "system"; + default: + throw new IllegalStateException("unreachable"); + } + case "connect_certhash_digits": + return tryValidate().connectCertHashDigits(); + case "connect_binary": + return tryValidate().connectBinary(); + case "connect_clientkey": + return tryValidate().connectClientKey(); + case "connect_clientcert": + return tryValidate().connectClientCert(); + + default: + throw new Failure("Unknown attribute: " + key); + } + } + + private class Failure extends Exception { + public Failure(String message) { + super(message); + } + } +} diff --git a/tests/tests.md b/tests/tests.md new file mode 100644 index 0000000..3509323 --- /dev/null +++ b/tests/tests.md @@ -0,0 +1,1458 @@ +# Tests + +This document contains a large number of test cases. +They are embedded in the Markdown source, in ```test +. . .``` blocks. + + +The tests are written in a mini language with the following +keywords: + +* `PARSE url`: parse the URL, this should succeed. The validity checks need + not be satisfied. + +* `ACCEPT url`: parse the URL, this should succeed. The validity checks + should pass. + +* `REJECT url`: parse the URL, it should be rejected either in the parsing stage + or by the validity checks. + +* `SET key=value`: modify a parameter, can occur before or after parsing the URL. + Used to model command line parameters, Java Properties objects, etc. + +* `EXPECT key=value`: verify that the given parameter now has the given + value. Fail the test case if the value is different. + +* `ONLY pymonetdb`: only process the rest of the block when testing + pymonetdb, ignore it for other implementations. + +* `NOT pymonetdb`: ignore the rest of the block when testing pymonetdb, + do process it for other implementations. + +At the start of each block the parameters are reset to their default values. + +The EXPECT clause can verify all parameters listen in the Parameters section of +the spec, all 'virtual parameters' and also the special case `valid` which is a +boolean indicating whether all validity rules in section 'Interpreting the +parameters' hold. + +Note: an `EXPECT` of the virtual parameters implies `EXPECT valid=true`. + +TODO before 1.0 does the above explanation make sense? + + +## Tests from the examples + +```test +ACCEPT monetdb:///demo +EXPECT database=demo +EXPECT connect_scan=true +``` + +```test +ACCEPT monetdb://localhost/demo +EXPECT connect_scan=true +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost./demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost.:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix=/tmp/.s.monetdb.12345 +EXPECT connect_tcp=localhost +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb:///demo?user=monetdb&password=monetdb +EXPECT connect_scan=true +EXPECT database=demo +EXPECT user=monetdb +EXPECT password=monetdb +``` + +```test +ACCEPT monetdb://mdb.example.com:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://192.168.13.4:12345/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=192.168.13.4 +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:12345/demo +EXPECT host=2001:0db8:85a3:0000:0000:8a2e:0370:7334 +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=2001:0db8:85a3:0000:0000:8a2e:0370:7334 +EXPECT connect_port=12345 +EXPECT tls=off +EXPECT database=demo +``` + +```test +ACCEPT monetdb://localhost/ +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_scan=false +EXPECT connect_tcp=localhost +EXPECT tls=off +EXPECT database= +``` + +```test +ACCEPT monetdb://localhost +EXPECT connect_scan=false +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +EXPECT tls=off +EXPECT database= +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=system +EXPECT database=demo +``` + +```test +ACCEPT monetdbs:///demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=system +EXPECT database=demo +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo?cert=/home/user/server.crt +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=cert +EXPECT cert=/home/user/server.crt +EXPECT database=demo +``` + +```test +ACCEPT monetdbs://mdb.example.com/demo?certhash=sha256:fb:67:20:aa:00:9f:33:4c +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=mdb.example.com +EXPECT connect_port=50000 +EXPECT tls=on +EXPECT connect_tls_verify=hash +EXPECT certhash=sha256:fb:67:20:aa:00:9f:33:4c +EXPECT connect_certhash_digits=fb6720aa009f334c +EXPECT database=demo +``` + +```test +ACCEPT monetdb:///demo?sock=/var/monetdb/_sock&user=dbuser +EXPECT connect_scan=false +EXPECT connect_unix=/var/monetdb/_sock +EXPECT connect_tcp= +EXPECT tls=off +EXPECT database=demo +EXPECT user=dbuser +EXPECT password= +``` + +```test +ACCEPT monetdb://localhost/demo?sock=/var/monetdb/_sock&user=dbuser +EXPECT connect_scan=false +EXPECT connect_unix=/var/monetdb/_sock +EXPECT connect_tcp= +EXPECT tls=off +EXPECT database=demo +EXPECT user=dbuser +EXPECT password= +``` + + +## Parameter tests + +Tests derived from the parameter section. Test data types and defaults. + +Everything can be SET and EXPECTed + +```test +SET tls=on +EXPECT tls=on +SET host=bananahost +EXPECT host=bananahost +SET port=123 +EXPECT port=123 +SET database=bananadatabase +EXPECT database=bananadatabase +SET tableschema=bananatableschema +EXPECT tableschema=bananatableschema +SET table=bananatable +EXPECT table=bananatable +SET sock=c:\foo.txt +EXPECT sock=c:\foo.txt +SET cert=c:\foo.txt +EXPECT cert=c:\foo.txt +SET certhash=bananacerthash +EXPECT certhash=bananacerthash +SET clientkey=c:\foo.txt +EXPECT clientkey=c:\foo.txt +SET clientcert=c:\foo.txt +EXPECT clientcert=c:\foo.txt +SET user=bananauser +EXPECT user=bananauser +SET password=bananapassword +EXPECT password=bananapassword +SET language=bananalanguage +EXPECT language=bananalanguage +SET autocommit=on +EXPECT autocommit=on +SET schema=bananaschema +EXPECT schema=bananaschema +SET timezone=123 +EXPECT timezone=123 +SET binary=bananabinary +EXPECT binary=bananabinary +SET replysize=123 +EXPECT replysize=123 +SET fetchsize=123 +EXPECT fetchsize=123 +``` + +### core defaults + +```test +EXPECT tls=false +EXPECT host= +EXPECT port=-1 +EXPECT database= +EXPECT tableschema= +EXPECT table= +EXPECT binary=on +``` + +### sock + +Not supported on Windows, but they should still parse. + +```test +EXPECT sock= +ACCEPT monetdb:///?sock=/tmp/sock +EXPECT sock=/tmp/sock +ACCEPT monetdb:///?sock=C:/TEMP/sock +EXPECT sock=C:/TEMP/sock +NOT jdbc +ACCEPT monetdb:///?sock=C:\TEMP\sock +EXPECT sock=C:\TEMP\sock +``` + +### sockdir + +```test +EXPECT sockdir=/tmp +ACCEPT monetdb:///demo?sockdir=/tmp/nonstandard +EXPECT sockdir=/tmp/nonstandard +EXPECT connect_unix=/tmp/nonstandard/.s.monetdb.50000 +``` + +### cert + +```test +EXPECT cert= +ACCEPT monetdbs:///?cert=/tmp/cert.pem +EXPECT cert=/tmp/cert.pem +ACCEPT monetdbs:///?cert=C:/TEMP/cert.pem +EXPECT cert=C:/TEMP/cert.pem +NOT jdbc +ACCEPT monetdbs:///?cert=C:\TEMP\cert.pem +EXPECT cert=C:\TEMP\cert.pem +``` + +### certhash + +```test +EXPECT certhash= +ACCEPT monetdbs:///?certhash=sha256:001122ff +ACCEPT monetdbs:///?certhash=sha256:00:11:22:ff +ACCEPT monetdbs:///?certhash=sha256:::::aa::ff::::: +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +This string of hexdigits is longer than the length of a SHA-256 digest. +It still parses, it will just never match. + +```test +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b8550 +ACCEPT monetdbs:///?certhash=sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855000000000000000000000000000000000000000001 +``` + +```test +REJECT monetdbs:///?certhash=001122ff +REJECT monetdbs:///?certhash=Sha256:001122ff +REJECT monetdbs:///?certhash=sha256:001122gg +REJECT monetdbs:///?certhash=sha +REJECT monetdbs:///?certhash=sha1:aabbcc +REJECT monetdbs:///?certhash=sha1: +REJECT monetdbs:///?certhash=sha1:X +REJECT monetdbs:///?certhash=sha99:aabbcc +REJECT monetdbs:///?certhash=sha99: +REJECT monetdbs:///?certhash=sha99:X +``` + +### clientkey, clientcert + +```test +EXPECT clientkey= +EXPECT clientcert= +ACCEPT monetdbs:///?clientkey=/tmp/clientkey.pem +EXPECT clientkey=/tmp/clientkey.pem +ACCEPT monetdbs:///?clientkey=C:/TEMP/clientkey.pem +EXPECT clientkey=C:/TEMP/clientkey.pem +NOT jdbc +ACCEPT monetdbs:///?clientkey=C:\TEMP\clientkey.pem +EXPECT clientkey=C:\TEMP\clientkey.pem +``` + +```test +EXPECT connect_clientkey= +EXPECT connect_clientcert= +``` + +```test +SET clientkey=/tmp/key.pem +SET clientcert=/tmp/cert.pem +EXPECT valid=true +EXPECT connect_clientkey=/tmp/key.pem +EXPECT connect_clientcert=/tmp/cert.pem +``` + +```test +SET clientkey=/tmp/key.pem +EXPECT valid=true +EXPECT connect_clientkey=/tmp/key.pem +EXPECT connect_clientcert=/tmp/key.pem +``` + +```test +SET clientcert=/tmp/cert.pem +EXPECT valid=false +``` + +```test +SET clientkey=dummy +EXPECT clientcert= +ACCEPT monetdbs:///?clientcert=/tmp/clientcert.pem +EXPECT clientcert=/tmp/clientcert.pem +ACCEPT monetdbs:///?clientcert=C:/TEMP/clientcert.pem +EXPECT clientcert=C:/TEMP/clientcert.pem +NOT jdbc +ACCEPT monetdbs:///?clientcert=C:\TEMP\clientcert.pem +EXPECT clientcert=C:\TEMP\clientcert.pem +``` + +### user, password + +Not testing the default because they are (unfortunately) +implementation specific. + +```test +ACCEPT monetdb:///?user=monetdb +EXPECT user=monetdb +ACCEPT monetdb:///?user=me&password=? +EXPECT user=me +EXPECT password=? +``` + +### language + +```test +EXPECT language=sql +ACCEPT monetdb:///?language=msql +EXPECT language=msql +ACCEPT monetdb:///?language=sql +EXPECT language=sql +``` + +### autocommit + +```test +ACCEPT monetdb:///?autocommit=true +EXPECT autocommit=true +ACCEPT monetdb:///?autocommit=on +EXPECT autocommit=true +ACCEPT monetdb:///?autocommit=yes +EXPECT autocommit=true +``` + +```test +ACCEPT monetdb:///?autocommit=false +EXPECT autocommit=false +ACCEPT monetdb:///?autocommit=off +EXPECT autocommit=false +ACCEPT monetdb:///?autocommit=no +EXPECT autocommit=false +``` + +```test +REJECT monetdb:///?autocommit= +REJECT monetdb:///?autocommit=banana +REJECT monetdb:///?autocommit=0 +REJECT monetdb:///?autocommit=1 +``` + +### schema, timezone + +Must be accepted, no constraints on content + +```test +EXPECT schema= +ACCEPT monetdb:///?schema=foo +EXPECT schema=foo +ACCEPT monetdb:///?schema= +EXPECT schema= +ACCEPT monetdb:///?schema=foo +``` + +```test +ACCEPT monetdb:///?timezone=0 +EXPECT timezone=0 +ACCEPT monetdb:///?timezone=120 +EXPECT timezone=120 +ACCEPT monetdb:///?timezone=-120 +EXPECT timezone=-120 +REJECT monetdb:///?timezone=banana +REJECT monetdb:///?timezone= +``` + +### replysize and fetchsize + +Note we never check `EXPECT fetchsize=`, it doesn't exist. + +```test +ACCEPT monetdb:///?replysize=150 +EXPECT replysize=150 +ACCEPT monetdb:///?fetchsize=150 +EXPECT replysize=150 +ACCEPT monetdb:///?fetchsize=100&replysize=200 +EXPECT replysize=200 +ACCEPT monetdb:///?replysize=100&fetchsize=200 +EXPECT replysize=200 +REJECT monetdb:///?replysize= +REJECT monetdb:///?fetchsize= +``` + +### binary + +```test +EXPECT binary=on +EXPECT connect_binary=65535 +``` + +```test +ACCEPT monetdb:///?binary=on +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=yes +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=true +EXPECT connect_binary=65535 + +ACCEPT monetdb:///?binary=yEs +EXPECT connect_binary=65535 +``` + +```test +ACCEPT monetdb:///?binary=off +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=no +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=false +EXPECT connect_binary=0 +``` + +```test +ACCEPT monetdb:///?binary=0 +EXPECT connect_binary=0 + +ACCEPT monetdb:///?binary=5 +EXPECT connect_binary=5 + +ACCEPT monetdb:///?binary=0100 +EXPECT connect_binary=100 +``` + +```test +REJECT monetdb:///?binary= +REJECT monetdb:///?binary=-1 +REJECT monetdb:///?binary=1.0 +REJECT monetdb:///?binary=banana +``` + +### unknown parameters + +```test +REJECT monetdb:///?banana=bla +``` + +```test +ACCEPT monetdb:///?ban_ana=bla +ACCEPT monetdb:///?hash=sha1 +ACCEPT monetdb:///?debug=true +ACCEPT monetdb:///?logfile=banana +``` + +Unfortunately we can't easily test that it won't allow us +to SET banana. + +```test +SET ban_ana=bla +SET hash=sha1 +SET debug=true +SET logfile=banana +``` + +## Combining sources + +The defaults have been tested in the previous section. + +Rule: If there is overlap, later sources take precedence. + +```test +SET schema=a +ACCEPT monetdb:///db1?schema=b +EXPECT schema=b +EXPECT database=db1 +EXPECT tls=off +ACCEPT monetdbs:///db2?schema=c +EXPECT tls=on +EXPECT database=db2 +EXPECT schema=c +``` + +Rule: a source that sets user must set password or clear. + +```skiptest +ACCEPT monetdb:///?user=foo +EXPECT user=foo +EXPECT password= +SET password=banana +EXPECT user=foo +EXPECT password=banana +SET user=bar +EXPECT password= +``` + +Rule: fetchsize is an alias for replysize, last occurrence counts + +```test +SET replysize=200 +ACCEPT monetdb:///?fetchsize=400 +EXPECT replysize=400 +ACCEPT monetdb:///?replysize=500&fetchsize=600 +EXPECT replysize=600 +``` + +```test +NOT jdbc +SET replysize=200 +SET fetchsize=300 +EXPECT replysize=300 +``` + + + +Rule: parsing a URL sets all of tls, host, port and database +even if left out of the URL + +```test +SET tls=on +SET host=banana +SET port=12345 +SET database=foo +SET timezone=120 +ACCEPT monetdb:/// +EXPECT tls=off +EXPECT host= +EXPECT port=-1 +EXPECT database= +``` + +```test +SET tls=on +SET host=banana +SET port=12345 +SET database=foo +SET timezone=120 +ACCEPT monetdb://dbhost/dbdb +EXPECT tls=off +EXPECT host=dbhost +EXPECT port=-1 +EXPECT database=dbdb +``` + +Careful around passwords + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:/// +EXPECT user=alan +EXPECT password=turing +``` + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:///?user=mathison +EXPECT user=mathison +EXPECT password= +``` + +The rule is, "if **user** is set", not "if **user** is changed". + +```test +SET user=alan +SET password=turing +ACCEPT monetdbs:///?user=alan +EXPECT user=alan +EXPECT password= +``` + +## URL syntax + +General form + +```test +REJECT monetdb: +REJECT monetdbs: +REJECT monetdb:/ +REJECT monetdbs:/ +REJECT monetdb:// +REJECT monetdbs:// +ACCEPT monetdb:/// +ACCEPT monetdbs:/// +``` + + +```test +ACCEPT monetdb://host:12345/db1/schema2/table3?user=mr&password=bean +EXPECT tls=off +EXPECT host=host +EXPECT port=12345 +EXPECT database=db1 +EXPECT tableschema=schema2 +EXPECT table=table3 +EXPECT user=mr +EXPECT password=bean +``` + +Also, TLS and percent-escapes + +```test +ACCEPT monetdbs://h%6Fst:12345/db%31/schema%32/table%33?user=%6Dr&p%61ssword=bean +EXPECT tls=on +EXPECT host=host +EXPECT port=12345 +EXPECT database=db1 +EXPECT tableschema=schema2 +EXPECT table=table3 +EXPECT user=mr +EXPECT password=bean +``` + +Port number + +```test +REJECT monetdb://banana:0/ +REJECT monetdb://banana:-1/ +REJECT monetdb://banana:65536/ +REJECT monetdb://banana:100000/ +``` + +Trailing slash can be left off + +```test +ACCEPT monetdb://host?user=claude&password=m%26ms +EXPECT host=host +EXPECT user=claude +EXPECT password=m&ms +``` + +Error to set tls, host, port, database, tableschema and table as query parameters. + +```test +REJECT monetdb://foo:1/bar?tls=off +REJECT monetdb://foo:1/bar?host=localhost +REJECT monetdb://foo:1/bar?port=12345 +REJECT monetdb://foo:1/bar?database=allmydata +REJECT monetdb://foo:1/bar?tableschema=banana +REJECT monetdb://foo:1/bar?table=tabularity +``` + +Last wins, already tested elsewhere but for completeness + +```test +ACCEPT monetdbs:///?timezone=10&timezone=20 +EXPECT timezone=20 +``` + +Interesting case: setting user must clear the password but does +that also happen with repetitions within a URL? +Not sure. For the time being, no. This makes it easier for +situations where for example the query parameters come in +alphabetical order + +```test +ACCEPT monetdb:///?user=foo&password=banana&user=bar +EXPECT user=bar +EXPECT password=banana +``` + +Similar but even simpler: user comes after password but does not +clear it. + +```test +ACCEPT monetdb:///?password=pw&user=foo +EXPECT user=foo +EXPECT password=pw +``` + +Ways of writing booleans and the binary property have already been tested above. + +Ip numbers: + +```test +ACCEPT monetdb://192.168.1.1:12345/foo +EXPECT connect_unix= +EXPECT connect_tcp=192.168.1.1 +EXPECT database=foo +``` + +```test +ACCEPT monetdb://[::1]:12345/foo +EXPECT connect_unix= +EXPECT connect_tcp=::1 +EXPECT database=foo +``` + +Bad percent escapes: + +```test +REJECT monetdb:///m%xxbad +``` + + +## Interpreting + +Testing the validity constraints. +They apply both when parsing a URL and with ad-hoc settings. + +Rule 1, the type constraints, has already been tested in [Section Parameter +tests](#parameter-tests). + +Rule 2, interaction between **sock** and **host** is tested below in +[the next subsection](#interaction-between-tls-host-sock-and-database). + +Rule 3, about **binary**, is tested in [Subsection Binary](#binary). + +Rule 4, **sock** vs **tls** is tested below in [the next +subsection](#interaction-between-tls-host-sock-and-database). + +Rule 5, **certhash** syntax, is tested in [Subsection Certhash](#certhash). + +Rule 6, **tls** **cert** **certhash** interaction, is tested +in [Subsection Interaction between tls, cert and certhash](#interaction-between-tls-cert-and-certhash). + +Rule 7, **database**, **tableschema**, **table** is tested in [Subsection +Database, schema, table name +constraints](#database-schema-table-name-constraints). + +Here are some tests for Rule 8, **port**. + +```test +SET port=1 +EXPECT valid=true +SET port=10 +EXPECT valid=true +SET port=000010 +EXPECT valid=true +SET port=65535 +EXPECT valid=true +SET port=-1 +EXPECT valid=true +SET port=0 +EXPECT valid=false +SET port=-2 +EXPECT valid=false +SET port=65536 +EXPECT valid=false +``` + +### Database, schema, table name constraints + +```test +SET database= +EXPECT valid=yes +SET database=banana +EXPECT valid=yes +SET database=UPPERCASE +EXPECT valid=yes +SET database=_under_score_ +EXPECT valid=yes +SET database=with-dashes +EXPECT valid=yes +``` + +```test +SET database=with/slash +EXPECT valid=no +SET database=-flag +EXPECT valid=no +SET database=with space +EXPECT valid=no +SET database=with.period +EXPECT valid=yes +SET database=with%percent +EXPECT valid=no +SET database=with!exclamation +EXPECT valid=no +SET database=with?questionmark +EXPECT valid=no +``` + +```test +SET database=demo +SET tableschema= +EXPECT valid=yes +SET tableschema=banana +EXPECT valid=yes +SET tableschema=UPPERCASE +EXPECT valid=yes +SET tableschema=_under_score_ +EXPECT valid=yes +SET tableschema=with-dashes +EXPECT valid=yes +``` + +```test +SET database=demo +SET tableschema=with/slash +EXPECT valid=no +SET tableschema=-flag +EXPECT valid=no +SET tableschema=with space +EXPECT valid=no +SET tableschema=with.period +EXPECT valid=yes +SET tableschema=with%percent +EXPECT valid=no +SET tableschema=with!exclamation +EXPECT valid=no +SET tableschema=with?questionmark +EXPECT valid=no +``` + +```test +SET database=demo +SET tableschema=sys +SET table= +EXPECT valid=yes +SET table=banana +EXPECT valid=yes +SET table=UPPERCASE +EXPECT valid=yes +SET table=_under_score_ +EXPECT valid=yes +SET table=with-dashes +EXPECT valid=yes +``` + +```test +SET database=demo +SET tableschema=sys +SET table=with/slash +EXPECT valid=no +SET table=-flag +EXPECT valid=no +SET table=with space +EXPECT valid=no +SET table=with.period +EXPECT valid=yes +SET table=with%percent +EXPECT valid=no +SET table=with!exclamation +EXPECT valid=no +SET table=with?questionmark +EXPECT valid=no +``` + +### Interaction between tls, cert and certhash + +```test +ACCEPT monetdbs:///?cert=/a/path +EXPECT connect_tls_verify=cert +ACCEPT monetdbs:///?certhash=sha256:aa +EXPECT connect_tls_verify=hash +ACCEPT monetdbs:///?cert=/a/path&certhash=sha256:aa +EXPECT connect_tls_verify=hash +REJECT monetdb:///?cert=/a/path +REJECT monetdb:///?certhash=sha256:aa +``` + +```test +SET tls=off +SET cert= +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify= +``` + +```test +SET tls=off +SET cert= +SET certhash=sha256:abcdef +EXPECT valid=no +``` + +```test +SET tls=off +SET cert=/foo +SET certhash= +EXPECT valid=no +``` + +```test +SET tls=off +SET cert=/foo +SET certhash=sha256:abcdef +EXPECT valid=no +``` + +```test +SET tls=on +SET cert= +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify=system +``` + +```test +SET tls=on +SET cert= +SET certhash=sha256:abcdef +EXPECT valid=yes +EXPECT connect_tls_verify=hash +``` + +```test +SET tls=on +SET cert=/foo +SET certhash= +EXPECT valid=yes +EXPECT connect_tls_verify=cert +``` + +```test +SET tls=on +SET cert=/foo +SET certhash=sha256:abcdef +EXPECT valid=yes +EXPECT connect_tls_verify=hash +``` + + +### Interaction between tls, host, sock and database + +The following tests should exhaustively test all variants. + +```test +ACCEPT monetdb:/// +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT monetdb:///?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost/ +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.50000 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT monetdb://localhost/?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost./ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdb://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdb://not.localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdb://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs:/// +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs:///?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost./ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdbs://not.localhost/ +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdbs://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdb:///demo +EXPECT connect_scan=on +``` + +```test +ACCEPT monetdb:///demo?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost/demo +EXPECT connect_scan=on +``` + +```test +ACCEPT monetdb://localhost/demo?sock=/a/path +EXPECT connect_scan=off +EXPECT connect_unix=/a/path +EXPECT connect_tcp= +``` + +```test +ACCEPT monetdb://localhost./demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdb://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdb://not.localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdb://not.localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs:///demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs:///?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost/?sock=/a/path +``` + +```test +ACCEPT monetdbs://localhost./demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +REJECT monetdbs://localhost./?sock=/a/path +``` + +```test +ACCEPT monetdbs://not.localhost/demo +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=not.localhost +``` + +```test +REJECT monetdbs://not.localhost/?sock=/a/path +``` + +### sock and sockdir + +Sockdir only applies to implicit Unix domain sockets, +not to ones that are given explicitly + +```test +EXPECT sockdir=/tmp +EXPECT port=-1 +EXPECT host= +EXPECT connect_unix=/tmp/.s.monetdb.50000 +SET sockdir=/somewhere/else +EXPECT connect_unix=/somewhere/else/.s.monetdb.50000 +SET port=12345 +EXPECT connect_unix=/somewhere/else/.s.monetdb.12345 +``` + +## Legacy URL's + +```test +REJECT mapi: +REJECT mapi:monetdb +REJECT mapi:monetdb: +REJECT mapi:monetdb:/ +``` + +```test +ACCEPT mapi:monetdb://monet.db:12345/demo +EXPECT host=monet.db +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=monet.db +``` + +This one is the golden standard: + +```test +ACCEPT mapi:monetdb://localhost:12345/demo +EXPECT host=localhost +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost:12345/ +EXPECT host=localhost +EXPECT port=12345 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost:12345 +EXPECT host=localhost +EXPECT port=12345 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://localhost/demo +EXPECT connect_scan=false +EXPECT connect_unix= +EXPECT connect_tcp=localhost +EXPECT connect_port=50000 +``` + +```test +ACCEPT mapi:monetdb://:12345/demo +EXPECT host= +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix=/tmp/.s.monetdb.12345 +EXPECT connect_tcp=localhost +``` + +```test +ACCEPT mapi:monetdb://127.0.0.1:12345/demo +EXPECT host=127.0.0.1 +EXPECT port=12345 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_scan=off +EXPECT connect_unix= +EXPECT connect_tcp=127.0.0.1 +``` + +Database parameter allowed, overrides path + +```test +ACCEPT mapi:monetdb://localhost:12345/demo?database=foo +EXPECT database=foo +``` + +User, username and password parameters are ignored: + +```test +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://localhost:12345/demo?user=foo +EXPECT user=alan +EXPECT password=turing +ACCEPT mapi:monetdb://localhost:12345/demo?password=foo +EXPECT user=alan +EXPECT password=turing +``` + +Pymonetdb used to accept user name and password before +the host name and should continue to do so. + + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo:bar@localhost:12345/demo +EXPECT user=foo +EXPECT password=bar +ACCEPT mapi:monetdb://banana@localhost:12345/demo +EXPECT user=banana +EXPECT password= +``` + +```test +NOT pymonetdb +SET user=alan +SET password=turing +REJECT mapi:monetdb://foo:bar@localhost:12345/demo +REJECT mapi:monetdb://banana@localhost:12345/demo +``` + +Unix domain sockets + +```test +ACCEPT mapi:monetdb:///path/to/sock?database=demo +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT port=-1 +EXPECT database=demo +EXPECT tls=off +EXPECT language=sql +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +```test +ACCEPT mapi:monetdb:///path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT port=-1 +EXPECT database= +EXPECT tls=off +EXPECT language=sql +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +Corner case: both libmapi and pymonetdb interpret this as an attempt +to connect to socket '/'. This will fail of course but the URL does parse + +```test +ACCEPT mapi:monetdb:/// +EXPECT host= +EXPECT sock=/ +EXPECT connect_unix=/ +EXPECT connect_tcp= +``` + +```test +NOT pymonetdb +PARSE mapi:monetdb:///foo:bar@path/to/sock +EXPECT sock=/foo:bar@path/to/sock +REJECT mapi:monetdb://foo:bar@/path/to/sock +``` + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo:bar@/path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT user=foo +EXPECT password=bar +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +```test +ONLY pymonetdb +SET user=alan +SET password=turing +ACCEPT mapi:monetdb://foo@/path/to/sock +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT user=foo +EXPECT password= +EXPECT connect_unix=/path/to/sock +EXPECT connect_tcp= +``` + +Language is supported + +```test +SET language=sql +ACCEPT mapi:monetdb://localhost:12345?language=mal +EXPECT host=localhost +EXPECT sock= +EXPECT language=mal +SET language=sql +ACCEPT mapi:monetdb:///path/to/sock?language=mal +EXPECT host= +EXPECT sock=/path/to/sock +EXPECT language=mal +``` + +No percent decoding is performed + +```test +REJECT mapi:monetdb://localhost:1234%35/demo +PARSE mapi:monetdb://loc%61lhost:12345/d%61tabase +EXPECT host=loc%61lhost +EXPECT database=d%61tabase +EXPECT valid=no +``` + +```test +PARSE mapi:monetdb://localhost:12345/db?database=b%61r&language=m%61l +EXPECT database=b%61r +EXPECT language=m%61l +EXPECT valid=no +``` + +l%61nguage is an unknown parameter, thus ignored not rejected + +```test +SET language=sql +ACCEPT mapi:monetdb://localhost:12345/db?l%61nguage=mal +EXPECT language=sql +ACCEPT mapi:monetdb://localhost:12345/db?_l%61nguage=mal +``` + From 6116b897b3987c53c77bbdcc5af56c6a0c012fd5 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 30 Nov 2023 14:21:46 +0100 Subject: [PATCH 03/41] Adjustments after applying changes to libmapi --- .../org/monetdb/mcl/net/MonetUrlParser.java | 76 +++++++++++++------ src/main/java/org/monetdb/mcl/net/Target.java | 2 +- tests/tests.md | 4 +- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java index da0d53d..4d582f5 100644 --- a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java +++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java @@ -16,6 +16,16 @@ public class MonetUrlParser { public MonetUrlParser(Properties props, String url) throws URISyntaxException { this.props = props; this.urlText = url; + // we want to accept monetdb:// but the Java URI parser rejects that. + switch (url) { + case "monetdb:-": + case "monetdbs:-": + throw new URISyntaxException(url, "invalid MonetDB URL"); + case "monetdb://": + case "monetdbs://": + url += "-"; + break; + } this.url = new URI(url); } @@ -124,31 +134,27 @@ private void parseModern() throws URISyntaxException { } host = ""; remainder = ""; - } else if (authority.startsWith("[")) { - // IPv6 - pos = authority.indexOf(']'); - if (pos < 0) - throw new URISyntaxException(urlText, "unmatched '['"); - host = authority.substring(1, pos); - remainder = authority.substring(pos + 1); - } else if ((pos = authority.indexOf(':')) >= 0){ - host = authority.substring(0, pos); - remainder = authority.substring(pos); - } else { - host = authority; + } else if (authority.equals("-")) { + host = ""; remainder = ""; + } else { + if (authority.startsWith("[")) { + // IPv6 + pos = authority.indexOf(']'); + if (pos < 0) + throw new URISyntaxException(urlText, "unmatched '['"); + host = authority.substring(1, pos); + remainder = authority.substring(pos + 1); + } else if ((pos = authority.indexOf(':')) >= 0) { + host = authority.substring(0, pos); + remainder = authority.substring(pos); + } else { + host = authority; + remainder = ""; + } } - switch (host) { - case "localhost": - set(Parameter.HOST, ""); - break; - case "localhost.": - set(Parameter.HOST, "localhost"); - break; - default: - set(Parameter.HOST, host); - break; - } + host = unwrapLocalhost(host); + set(Parameter.HOST, host); if (remainder.isEmpty()) { // do nothing @@ -206,6 +212,30 @@ private void parseModern() throws URISyntaxException { } } + public static String wrapLocalhost(String host) { + switch (host) { + case "localhost": + host = "localhost."; + break; + case "": + host = "localhost"; + break; + } + return host; + } + + public static String unwrapLocalhost(String host) { + switch (host) { + case "localhost": + host = ""; + break; + case "localhost.": + host = "localhost"; + break; + } + return host; + } + private void parseClassic() throws URISyntaxException { String scheme = url.getScheme(); diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index 57d44a8..cc6a8c4 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -5,7 +5,7 @@ import java.util.regex.Pattern; public class Target { - private static Pattern namePattern = Pattern.compile("^[a-zA-Z_][-a-zA-Z0-9_.]*$"); + private static Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$"); private static Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$"); private final boolean tls; private final String host; diff --git a/tests/tests.md b/tests/tests.md index 3509323..5fb5a2a 100644 --- a/tests/tests.md +++ b/tests/tests.md @@ -672,8 +672,8 @@ REJECT monetdb: REJECT monetdbs: REJECT monetdb:/ REJECT monetdbs:/ -REJECT monetdb:// -REJECT monetdbs:// +ACCEPT monetdb:// +ACCEPT monetdbs:// ACCEPT monetdb:/// ACCEPT monetdbs:/// ``` From 4ee7a73d9be33f2331ffac09739991afae0a832a Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Fri, 1 Dec 2023 14:18:01 +0100 Subject: [PATCH 04/41] Refactor --- .../org/monetdb/mcl/net/MonetUrlParser.java | 135 +--- .../java/org/monetdb/mcl/net/Parameter.java | 18 +- .../org/monetdb/mcl/net/ParameterType.java | 2 +- src/main/java/org/monetdb/mcl/net/Target.java | 654 ++++++++++++------ tests/UrlTester.java | 104 ++- tests/tests.md | 7 +- 6 files changed, 545 insertions(+), 375 deletions(-) diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java index 4d582f5..0117134 100644 --- a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java +++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java @@ -4,17 +4,14 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; -import java.util.Properties; public class MonetUrlParser { - private final Properties props; + private final Target target; private final String urlText; private final URI url; - boolean userWasSet = false; - boolean passwordWasSet = false; - public MonetUrlParser(Properties props, String url) throws URISyntaxException { - this.props = props; + public MonetUrlParser(Target target, String url) throws URISyntaxException { + this.target = target; this.urlText = url; // we want to accept monetdb:// but the Java URI parser rejects that. switch (url) { @@ -29,7 +26,7 @@ public MonetUrlParser(Properties props, String url) throws URISyntaxException { this.url = new URI(url); } - public static void parse(Properties props, String url) throws URISyntaxException { + public static void parse(Target target, String url) throws URISyntaxException, ValidationError { boolean modern = true; if (url.startsWith("mapi:")) { modern = false; @@ -38,16 +35,16 @@ public static void parse(Properties props, String url) throws URISyntaxException // deal with peculiarity of Java's URI parser url = "monetdb:///"; } - } + + target.barrier(); try { - MonetUrlParser parser = new MonetUrlParser(props, url); + MonetUrlParser parser = new MonetUrlParser(target, url); if (modern) { parser.parseModern(); } else { parser.parseClassic(); } - if (parser.userWasSet && !parser.passwordWasSet) parser.clear(Parameter.PASSWORD); } catch (URISyntaxException e) { int idx = e.getIndex(); if (idx >= 0 && !modern) { @@ -56,6 +53,7 @@ public static void parse(Properties props, String url) throws URISyntaxException } throw new URISyntaxException(e.getInput(), e.getReason(), idx); } + target.barrier(); } private static String percentDecode(String context, String text) throws URISyntaxException { @@ -68,53 +66,17 @@ private static String percentDecode(String context, String text) throws URISynta } } - private void set(Parameter parm, String value) { - parm = keyMagic(parm); - props.setProperty(parm.name, value != null ? value : ""); - } - - private void set(String key, String value) { - Parameter parm = Parameter.forName(key); - if (parm != null) - set(parm, value); - else - props.setProperty(key, value); - } - - private void clear(Parameter parm) { - parm = keyMagic(parm); - String value = parm.type.format(Target.getDefault(parm)); - props.setProperty(parm.name, value); - } - - private Parameter keyMagic(Parameter key) { - switch (key) { - case USER: - userWasSet = true; - break; - case PASSWORD: - passwordWasSet = true; - break; - case FETCHSIZE: - key = Parameter.REPLYSIZE; - break; - default: - break; - } - return key; - } - - private void parseModern() throws URISyntaxException { + private void parseModern() throws URISyntaxException, ValidationError { clearBasic(); String scheme = url.getScheme(); if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); switch (scheme) { case "monetdb": - set(Parameter.TLS, "false"); + target.setTls(false); break; case "monetdbs": - set(Parameter.TLS, "true"); + target.setTls(true); break; default: throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://"); @@ -153,8 +115,8 @@ private void parseModern() throws URISyntaxException { remainder = ""; } } - host = unwrapLocalhost(host); - set(Parameter.HOST, host); + host = Target.unpackHost(host); + target.setHost(host); if (remainder.isEmpty()) { // do nothing @@ -168,24 +130,22 @@ private void parseModern() throws URISyntaxException { portStr = null; } if (portStr == null) - throw new URISyntaxException(urlText, "invalid port number"); - set(Parameter.PORT, portStr); + throw new ValidationError(urlText, "invalid port number"); + target.setString(Parameter.PORT, portStr); } String path = url.getRawPath(); - String[] parts = path.split("/", 5); + String[] parts = path.split("/", 4); // <0: empty before leading slash> / <1: database> / <2: tableschema> / <3: table> / <4: should not exist> switch (parts.length) { - case 5: - throw new URISyntaxException(urlText, "table name should not contain slashes"); case 4: - set(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3])); + target.setString(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3])); // fallthrough case 3: - set(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2])); + target.setString(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2])); // fallthrough case 2: - set(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1])); + target.setString(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1])); case 1: case 0: // fallthrough @@ -207,37 +167,12 @@ private void parseModern() throws URISyntaxException { throw new URISyntaxException(key, key + "= is not allowed as a query parameter"); String value = args[i].substring(pos + 1); - set(key, percentDecode(key, value)); + target.setString(key, percentDecode(key, value)); } } } - public static String wrapLocalhost(String host) { - switch (host) { - case "localhost": - host = "localhost."; - break; - case "": - host = "localhost"; - break; - } - return host; - } - - public static String unwrapLocalhost(String host) { - switch (host) { - case "localhost": - host = ""; - break; - case "localhost.": - host = "localhost"; - break; - } - return host; - } - - - private void parseClassic() throws URISyntaxException { + private void parseClassic() throws URISyntaxException, ValidationError { String scheme = url.getScheme(); if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); switch (scheme) { @@ -279,9 +214,9 @@ private void parseClassic() throws URISyntaxException { port = -1; } if (port <= 0) { - throw new URISyntaxException(urlText, "invalid port number"); + throw new ValidationError(urlText, "invalid port number"); } - set(Parameter.PORT, portStr); + target.setString(Parameter.PORT, portStr); } String path = url.getRawPath(); @@ -289,22 +224,20 @@ private void parseClassic() throws URISyntaxException { if (host.isEmpty() && portStr.isEmpty()) { // socket isUnix = true; - clear(Parameter.HOST); - set(Parameter.SOCK, path != null ? path : ""); + target.clear(Parameter.HOST); + target.setString(Parameter.SOCK, path != null ? path : ""); } else { // tcp isUnix = false; - clear(Parameter.SOCK); - set(Parameter.HOST, host); + target.clear(Parameter.SOCK); + target.setString(Parameter.HOST, host); if (path == null || path.isEmpty()) { // do nothing } else if (!path.startsWith("/")) { throw new URISyntaxException(urlText, "expect path to start with /"); } else { String database = path.substring(1); - if (database.contains("/")) - throw new URISyntaxException(urlText, "no slashes allowed in database name"); - set(Parameter.DATABASE, database); + target.setString(Parameter.DATABASE, database); } } @@ -315,10 +248,10 @@ private void parseClassic() throws URISyntaxException { String arg = args[i]; if (arg.startsWith("language=")) { String language = arg.substring(9); - set(Parameter.LANGUAGE, language); + target.setString(Parameter.LANGUAGE, language); } else if (arg.startsWith("database=")) { String database = arg.substring(9); - set(Parameter.DATABASE, database); + target.setString(Parameter.DATABASE, database); } else { // ignore } @@ -327,9 +260,9 @@ private void parseClassic() throws URISyntaxException { } private void clearBasic() { - clear(Parameter.HOST); - clear(Parameter.PORT); - clear(Parameter.SOCK); - clear(Parameter.DATABASE); + target.clear(Parameter.HOST); + target.clear(Parameter.PORT); + target.clear(Parameter.SOCK); + target.clear(Parameter.DATABASE); } } diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java index c7a70b8..e815ac6 100644 --- a/src/main/java/org/monetdb/mcl/net/Parameter.java +++ b/src/main/java/org/monetdb/mcl/net/Parameter.java @@ -1,6 +1,8 @@ package org.monetdb.mcl.net; +import java.util.Calendar; + public enum Parameter { TLS("tls", ParameterType.Bool, false, "secure the connection using TLS", true), HOST("host", ParameterType.Str, "", "IP number, domain name or one of the special values `localhost` and `localhost.`", true), @@ -17,7 +19,7 @@ public enum Parameter { USER("user", ParameterType.Str, "", "user name to authenticate as", false), PASSWORD("password", ParameterType.Str, "", "password to authenticate with", false), LANGUAGE("language", ParameterType.Str, "sql", "for example, \"sql\", \"mal\", \"msql\", \"profiler\"", false), - AUTOCOMMIT("autocommit", ParameterType.Bool, false, "initial value of autocommit", false), + AUTOCOMMIT("autocommit", ParameterType.Bool, true, "initial value of autocommit", false), SCHEMA("schema", ParameterType.Str, "", "initial schema", false), TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false), BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false), @@ -31,7 +33,7 @@ public enum Parameter { public final String name; public final ParameterType type; - public final Object defaultValue; + private final Object defaultValue; public final String description; public final boolean isCore; @@ -78,4 +80,16 @@ public static boolean isIgnored(String name) { return false; return name.contains("_"); } + + public Object getDefault() { + switch (this) { + case TIMEZONE: + Calendar cal = Calendar.getInstance(); + int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); + int offsetSeconds = offsetMillis / 1000; + return (Integer)offsetSeconds; + default: + return defaultValue; + } + } } diff --git a/src/main/java/org/monetdb/mcl/net/ParameterType.java b/src/main/java/org/monetdb/mcl/net/ParameterType.java index 8b7a57f..15bad0c 100644 --- a/src/main/java/org/monetdb/mcl/net/ParameterType.java +++ b/src/main/java/org/monetdb/mcl/net/ParameterType.java @@ -8,7 +8,7 @@ public enum ParameterType { public Object parse(String name, String value) throws ValidationError { if (value == null) - throw new NullPointerException(); + return null; try { switch (this) { diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index cc6a8c4..67c25cf 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -5,326 +5,570 @@ import java.util.regex.Pattern; public class Target { + private boolean tls = false; + private String host = ""; + private int port = -1; + private String database = ""; + private String tableschema = ""; + private String table = ""; + private String sock = ""; + private String sockdir = "/tmp"; + private String cert = ""; + private String certhash = ""; + private String clientkey = ""; + private String clientcert = ""; + private String user = ""; + private String password = ""; + private String language = "sql"; + private boolean autocommit = true; + private String schema = ""; + private int timezone; + private String binary = "on"; + private int replysize = 200; + private String hash = ""; + private boolean debug = false; + private String logfile = ""; + + private boolean userWasSet = false; + private boolean passwordWasSet = false; + protected static final Target defaults = new Target(); + private static Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$"); private static Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$"); - private final boolean tls; - private final String host; - private final int port; - private final String database; - private final String tableschema; - private final String table; - private final String sock; - private final String sockdir; - private final String cert; - private final String certhash; - private final String clientkey; - private final String clientcert; - private final String user; - private final String password; - private final String language; - private final boolean autocommit; - private final String schema; - private final int timezone; - private final int binary; - private final int replysize; - private final String hash; - private final boolean debug; - private final String logfile; - - public Target(Properties properties) throws ValidationError { - - // 1. The parameters have the types listed in the table in [Section - // Parameters](#parameters). - tls = validateBoolean(properties, Parameter.TLS); - host = validateString(properties, Parameter.HOST); - port = validateInt(properties, Parameter.PORT); - database = validateString(properties, Parameter.DATABASE); - tableschema = validateString(properties, Parameter.TABLESCHEMA); - table = validateString(properties, Parameter.TABLE); - sock = validateString(properties, Parameter.SOCK); - sockdir = validateString(properties, Parameter.SOCKDIR); - cert = validateString(properties, Parameter.CERT); - certhash = validateString(properties, Parameter.CERTHASH); - clientkey = validateString(properties, Parameter.CLIENTKEY); - clientcert = validateString(properties, Parameter.CLIENTCERT); - user = validateString(properties, Parameter.USER); - password = validateString(properties, Parameter.PASSWORD); - language = validateString(properties, Parameter.LANGUAGE); - autocommit = validateBoolean(properties, Parameter.AUTOCOMMIT); - schema = validateString(properties, Parameter.SCHEMA); - timezone = validateInt(properties, Parameter.TIMEZONE); - replysize = validateInt(properties, Parameter.REPLYSIZE); - hash = validateString(properties, Parameter.HASH); - debug = validateBoolean(properties, Parameter.DEBUG); - logfile = validateString(properties, Parameter.LOGFILE); - - for (String name: properties.stringPropertyNames()) { - if (Parameter.forName(name) != null) - continue; - if (name.contains("_")) - continue; - throw new ValidationError("unknown parameter: " + name); - } - - String binaryString = validateString(properties, Parameter.BINARY); - int binaryInt; - try { - binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString); - } catch (ValidationError e) { - try { - boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString); - binaryInt = b ? 65535 : 0; - } catch (ValidationError ee) { - throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off"); - } - } - if (binaryInt < 0) - throw new ValidationError("binary= cannot be negative"); - binary = binaryInt; - - // 2. At least one of **sock** and **host** must be empty. - if (!sock.isEmpty() && !host.isEmpty()) - throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host); - - // 3. The string parameter **binary** must either parse as a boolean or as a - // non-negative integer. - // - // (checked above) - - // 4. If **sock** is not empty, **tls** must be 'off'. - if (!sock.isEmpty() && tls) throw new ValidationError("monetdbs:// cannot be combined with sock="); + public Target() { + this.timezone = (int)Parameter.TIMEZONE.getDefault(); + } - // 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits` - // where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons. - // TODO - if (!certhash.isEmpty()) { - if (!certhash.toLowerCase().startsWith("sha256:")) - throw new ValidationError("certificate hash must start with 'sha256:'"); - if (!hashPattern.matcher(certhash).matches()) - throw new ValidationError("invalid certificate hash"); - } + public void barrier() { + if (userWasSet && !passwordWasSet) + password = ""; + userWasSet = false; + passwordWasSet = false; + } - // 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well. - if (!tls) { - if (!cert.isEmpty() || !certhash.isEmpty()) - throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://"); + public static String packHost(String host) { + switch (host) { + case "localhost": + return "localhost."; + case "": + return "localhost"; + default: + return host; } - - // 7. Parameters **database**, **tableschema** and **table** must consist only of - // upper- and lowercase letters, digits, periods, dashes and underscores. They must not - // start with a dash. - // If **table** is not empty, **tableschema** must also not be empty. - // If **tableschema** is not empty, **database** must also not be empty. - if (database.isEmpty() && !tableschema.isEmpty()) - throw new ValidationError("table schema cannot be set without database"); - if (tableschema.isEmpty() && !table.isEmpty()) - throw new ValidationError("table cannot be set without schema"); - if (!database.isEmpty() && !namePattern.matcher(database).matches()) - throw new ValidationError("invalid database name"); - if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches()) - throw new ValidationError("invalid table schema name"); - if (!table.isEmpty() && !namePattern.matcher(table).matches()) - throw new ValidationError("invalid table name"); - - - // 8. Parameter **port** must be -1 or in the range 1-65535. - if (port < -1 || port == 0 || port > 65535) throw new ValidationError("invalid port number " + port); - - // 9. If **clientcert** is set, **clientkey** must also be set. - if (!clientcert.isEmpty() && clientkey.isEmpty()) - throw new ValidationError("clientcert= is only valid in combination with clientkey="); } - public static boolean validateBoolean(Properties props, Parameter parm) throws ValidationError { - Object value = props.get(parm.name); - if (value != null) { - return (Boolean) parm.type.parse(parm.name, (String) value); - } else { - return (Boolean) getDefault(parm); + public void setString(String key, String value) throws ValidationError { + Parameter parm = Parameter.forName(key); + if (parm != null) + setString(parm, value); + else if (!Parameter.isIgnored(key)) + throw new ValidationError(key, "unknown parameter"); + } + + public void setString(Parameter parm, String value) throws ValidationError { + if (value == null) + throw new NullPointerException("'value' must not be null"); + assign(parm, parm.type.parse(parm.name, value)); + } + + public void clear(Parameter parm) { + assign(parm, parm.getDefault()); + } + + private void assign(Parameter parm, Object value) { + switch (parm) { + case TLS: setTls((boolean)value); break; + case HOST: setHost((String)value); break; + case PORT: setPort((int)value); break; + case DATABASE: setDatabase((String)value); break; + case TABLESCHEMA: setTableschema((String)value); break; + case TABLE: setTable((String)value); break; + case SOCK: setSock((String)value); break; + case SOCKDIR: setSockdir((String)value); break; + case CERT: setCert((String)value); break; + case CERTHASH: setCerthash((String)value); break; + case CLIENTKEY: setClientkey((String)value); break; + case CLIENTCERT: setClientcert((String)value); break; + case USER: setUser((String)value); break; + case PASSWORD: setPassword((String)value); break; + case LANGUAGE: setLanguage((String)value); break; + case AUTOCOMMIT: setAutocommit((boolean)value); break; + case SCHEMA: setSchema((String)value); break; + case TIMEZONE: setTimezone((int)value); break; + case BINARY: setBinary((String)value); break; + case REPLYSIZE: setReplysize((int)value); break; + case FETCHSIZE: setReplysize((int)value); break; + case HASH: setHash((String)value); break; + case DEBUG: setDebug((boolean)value); break; + case LOGFILE: setLogfile((String)value); break; + default: + throw new IllegalStateException("unreachable -- missing case"); } } - public static int validateInt(Properties props, Parameter parm) throws ValidationError { - Object value = props.get(parm.name); - if (value != null) { - return (Integer) parm.type.parse(parm.name, (String) value); - } else { - return (Integer) getDefault(parm); + public String getString(Parameter parm) { + Object value = getObject(parm); + return parm.type.format(value); + } + + public Object getObject(Parameter parm) { + switch (parm) { + case TLS: return tls; + case HOST: return host; + case PORT: return port; + case DATABASE: return database; + case TABLESCHEMA: return tableschema; + case TABLE: return table; + case SOCK: return sock; + case SOCKDIR: return sockdir; + case CERT: return cert; + case CERTHASH: return certhash; + case CLIENTKEY: return clientkey; + case CLIENTCERT: return clientcert; + case USER: return user; + case PASSWORD: return password; + case LANGUAGE: return language; + case AUTOCOMMIT: return autocommit; + case SCHEMA: return schema; + case TIMEZONE: return timezone; + case BINARY: return binary; + case REPLYSIZE: return replysize; + case FETCHSIZE: return replysize; + case HASH: return hash; + case DEBUG: return debug; + case LOGFILE: return logfile; + default: + throw new IllegalStateException("unreachable -- missing case"); } } - public static String validateString(Properties props, Parameter parm) throws ValidationError { - Object value = props.get(parm.name); - if (value != null) { - return (String) parm.type.parse(parm.name, (String) value); - } else { - return (String) getDefault(parm); + public static String unpackHost(String host) { + switch (host) { + case "localhost.": + return "localhost"; + case "localhost": + return ""; + default: + return host; } } - private static int timezone() { - Calendar cal = Calendar.getInstance(); - int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); - int offsetSeconds = offsetMillis / 1000; - return offsetSeconds; + public boolean isTls() { + return tls; } - public static Object getDefault(Parameter parm) { - if (parm == Parameter.TIMEZONE) return timezone(); - else return parm.defaultValue; + public void setTls(boolean tls) { + this.tls = tls; } - public static Properties defaultProperties() { - Properties props = new Properties(); - return props; + public String getHost() { + return host; } - public boolean getTls() { - return tls; + public void setHost(String host) { + this.host = host; } - // Getter is private because you probably want connectTcp() instead - private String getHost() { - return host; + public int getPort() { + return port; } - // Getter is private because you probably want connectPort() instead - private int getPort() { - return port; + public void setPort(int port) { + this.port = port; } public String getDatabase() { return database; } + public void setDatabase(String database) { + this.database = database; + } + public String getTableschema() { return tableschema; } + public void setTableschema(String tableschema) { + this.tableschema = tableschema; + } + public String getTable() { return table; } - // Getter is private because you probably want connectUnix() instead - private String getSock() { + public void setTable(String table) { + this.table = table; + } + + public String getSock() { return sock; } + public void setSock(String sock) { + this.sock = sock; + } + public String getSockdir() { return sockdir; } + public void setSockdir(String sockdir) { + this.sockdir = sockdir; + } + public String getCert() { return cert; } + public void setCert(String cert) { + this.cert = cert; + } + public String getCerthash() { return certhash; } + public void setCerthash(String certhash) { + this.certhash = certhash; + } + public String getClientkey() { return clientkey; } + public void setClientkey(String clientkey) { + this.clientkey = clientkey; + } + public String getClientcert() { return clientcert; } + public void setClientcert(String clientcert) { + this.clientcert = clientcert; + } + public String getUser() { return user; } + public void setUser(String user) { + this.user = user; + this.userWasSet = true; + } + public String getPassword() { return password; } + public void setPassword(String password) { + this.password = password; + this.passwordWasSet = true; + } + public String getLanguage() { return language; } - public boolean getAutocommit() { + public void setLanguage(String language) { + this.language = language; + } + + public boolean isAutocommit() { return autocommit; } + public void setAutocommit(boolean autocommit) { + this.autocommit = autocommit; + } + public String getSchema() { return schema; } + public void setSchema(String schema) { + this.schema = schema; + } + public int getTimezone() { return timezone; } - // Getter is private because you probably want connectBinary() instead - public int getBinary() { + public void setTimezone(int timezone) { + this.timezone = timezone; + } + + public String getBinary() { return binary; } + public void setBinary(String binary) { + this.binary = binary; + } + public int getReplysize() { return replysize; } + public void setReplysize(int replysize) { + this.replysize = replysize; + } + public String getHash() { return hash; } - public boolean getDebug() { + public void setHash(String hash) { + this.hash = hash; + } + + public boolean isDebug() { return debug; } + public void setDebug(boolean debug) { + this.debug = debug; + } + public String getLogfile() { return logfile; } - public boolean connectScan() { - if (database.isEmpty()) return false; - if (!sock.isEmpty() || !host.isEmpty() || port != -1) return false; - return !tls; + public void setLogfile(String logfile) { + this.logfile = logfile; } - public int connectPort() { - return port == -1 ? 50000 : port; + public Validated validate() throws ValidationError { + return new Validated(); } - public String connectUnix() { - if (!sock.isEmpty()) return sock; - if (tls) return ""; - if (host.isEmpty()) return sockdir + "/.s.monetdb." + connectPort(); - return ""; - } + public class Validated { - public String connectTcp() { - if (!sock.isEmpty()) return ""; - if (host.isEmpty()) return "localhost"; - return host; - } + private final int nbinary; - public Verify connectVerify() { - if (!tls) return Verify.None; - if (!certhash.isEmpty()) return Verify.Hash; - if (!cert.isEmpty()) return Verify.Cert; - return Verify.System; - } + Validated() throws ValidationError { + + // 1. The parameters have the types listed in the table in [Section + // Parameters](#parameters). + + String binaryString = binary; + int binaryInt; + try { + binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString); + } catch (ValidationError e) { + try { + boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString); + binaryInt = b ? 65535 : 0; + } catch (ValidationError ee) { + throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off"); + } + } + if (binaryInt < 0) + throw new ValidationError("binary= cannot be negative"); + nbinary = binaryInt; + + + // 2. At least one of **sock** and **host** must be empty. + if (!sock.isEmpty() && !host.isEmpty()) + throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host); + + // 3. The string parameter **binary** must either parse as a boolean or as a + // non-negative integer. + // + // (checked above) + + // 4. If **sock** is not empty, **tls** must be 'off'. + if (!sock.isEmpty() && tls) throw new ValidationError("monetdbs:// cannot be combined with sock="); + + // 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits` + // where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons. + // TODO + if (!certhash.isEmpty()) { + if (!certhash.toLowerCase().startsWith("sha256:")) + throw new ValidationError("certificate hash must start with 'sha256:'"); + if (!hashPattern.matcher(certhash).matches()) + throw new ValidationError("invalid certificate hash"); + } + + // 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well. + if (!tls) { + if (!cert.isEmpty() || !certhash.isEmpty()) + throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://"); + } - public String connectCertHashDigits() { - if (!tls) return null; - StringBuilder builder = new StringBuilder(certhash.length()); - for (int i = "sha256:".length(); i < certhash.length(); i++) { - char c = certhash.charAt(i); - if (Character.digit(c, 16) >= 0) builder.append(Character.toLowerCase(c)); + // 7. Parameters **database**, **tableschema** and **table** must consist only of + // upper- and lowercase letters, digits, periods, dashes and underscores. They must not + // start with a dash. + // If **table** is not empty, **tableschema** must also not be empty. + // If **tableschema** is not empty, **database** must also not be empty. + if (database.isEmpty() && !tableschema.isEmpty()) + throw new ValidationError("table schema cannot be set without database"); + if (tableschema.isEmpty() && !table.isEmpty()) + throw new ValidationError("table cannot be set without schema"); + if (!database.isEmpty() && !namePattern.matcher(database).matches()) + throw new ValidationError("invalid database name"); + if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches()) + throw new ValidationError("invalid table schema name"); + if (!table.isEmpty() && !namePattern.matcher(table).matches()) + throw new ValidationError("invalid table name"); + + + // 8. Parameter **port** must be -1 or in the range 1-65535. + if (port < -1 || port == 0 || port > 65535) throw new ValidationError("invalid port number " + port); + + // 9. If **clientcert** is set, **clientkey** must also be set. + if (!clientcert.isEmpty() && clientkey.isEmpty()) + throw new ValidationError("clientcert= is only valid in combination with clientkey="); } - return builder.toString(); - } - public int connectBinary() { - return binary; - } + public boolean getTls() { + return tls; + } - public String connectClientKey() { - return clientkey; - } + // Getter is private because you probably want connectTcp() instead + private String getHost() { + return host; + } + + // Getter is private because you probably want connectPort() instead + private int getPort() { + return port; + } + + public String getDatabase() { + return database; + } + + public String getTableschema() { + return tableschema; + } + + public String getTable() { + return table; + } + + // Getter is private because you probably want connectUnix() instead + private String getSock() { + return sock; + } + + public String getSockdir() { + return sockdir; + } + + public String getCert() { + return cert; + } + + public String getCerthash() { + return certhash; + } + + public String getClientkey() { + return clientkey; + } + + public String getClientcert() { + return clientcert; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } + + public String getLanguage() { + return language; + } - public String connectClientCert() { - return clientcert.isEmpty() ? clientkey : clientcert; + public boolean getAutocommit() { + return autocommit; + } + + public String getSchema() { + return schema; + } + + public int getTimezone() { + return timezone; + } + + // Getter is private because you probably want connectBinary() instead + public int getBinary() { + return nbinary; + } + + public int getReplysize() { + return replysize; + } + + public String getHash() { + return hash; + } + + public boolean getDebug() { + return debug; + } + + public String getLogfile() { + return logfile; + } + + public boolean connectScan() { + if (database.isEmpty()) return false; + if (!sock.isEmpty() || !host.isEmpty() || port != -1) return false; + return !tls; + } + + public int connectPort() { + return port == -1 ? 50000 : port; + } + + public String connectUnix() { + if (!sock.isEmpty()) return sock; + if (tls) return ""; + if (host.isEmpty()) return sockdir + "/.s.monetdb." + connectPort(); + return ""; + } + + public String connectTcp() { + if (!sock.isEmpty()) return ""; + if (host.isEmpty()) return "localhost"; + return host; + } + + public Verify connectVerify() { + if (!tls) return Verify.None; + if (!certhash.isEmpty()) return Verify.Hash; + if (!cert.isEmpty()) return Verify.Cert; + return Verify.System; + } + + public String connectCertHashDigits() { + if (!tls) return null; + StringBuilder builder = new StringBuilder(certhash.length()); + for (int i = "sha256:".length(); i < certhash.length(); i++) { + char c = certhash.charAt(i); + if (Character.digit(c, 16) >= 0) builder.append(Character.toLowerCase(c)); + } + return builder.toString(); + } + + public int connectBinary() { + return nbinary; + } + + public String connectClientKey() { + return clientkey; + } + + public String connectClientCert() { + return clientcert.isEmpty() ? clientkey : clientcert; + } } } diff --git a/tests/UrlTester.java b/tests/UrlTester.java index 2b79b88..a510a3d 100644 --- a/tests/UrlTester.java +++ b/tests/UrlTester.java @@ -2,7 +2,6 @@ import java.io.*; import java.net.URISyntaxException; -import java.util.Properties; public class UrlTester { String filename = null; @@ -10,8 +9,8 @@ public class UrlTester { BufferedReader reader = null; int lineno = 0; int testCount = 0; - Properties props = null; - Target validated = null; + Target target = null; + Target.Validated validated = null; public UrlTester() { } @@ -21,6 +20,8 @@ public UrlTester(String filename) { } public static void main(String[] args) throws Exception { + checkDefaults(); + int exitcode; UrlTester tester = new UrlTester(); exitcode = tester.parseArgs(args); @@ -29,6 +30,21 @@ public static void main(String[] args) throws Exception { System.exit(exitcode); } + private static void checkDefaults() { + Target target = new Target(); + + for (Parameter parm: Parameter.values()) { + Object expected = parm.getDefault(); + if (expected == null) + continue; + Object actual = target.getObject(parm); + if (!expected.equals(actual)) { + System.err.println("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">"); + System.exit(1); + } + } + } + private int parseArgs(String[] args) { for (int i = 0; i < args.length; i++) { String arg = args[i]; @@ -107,18 +123,18 @@ private void processFile() throws IOException, Failure { private void processLine(String line) throws Failure { line = line.replaceFirst("\\s+$", ""); // remove trailing - if (props == null && line.equals("```test")) { + if (target == null && line.equals("```test")) { if (verbose >= 2) { if (testCount > 0) { System.out.println(); } System.out.println("\u25B6 " + filename + ":" + lineno); } - props = Target.defaultProperties(); + target = new Target(); testCount++; return; } - if (props != null) { + if (target != null) { if (line.equals("```")) { stopProcessing(); return; @@ -128,7 +144,7 @@ private void processLine(String line) throws Failure { } private void stopProcessing() { - props = null; + target = null; validated = null; } @@ -205,7 +221,11 @@ private void handleSet(String rest) throws Failure { String key = splitKey(rest); String value = splitValue(rest); - props.put(key, value); + try { + target.setString(key, value); + } catch (ValidationError e) { + throw new Failure(e.getMessage()); + } } private void handleParse(String rest, Boolean shouldSucceed) throws Failure { @@ -214,14 +234,17 @@ private void handleParse(String rest, Boolean shouldSucceed) throws Failure { validated = null; try { - MonetUrlParser.parse(props, rest); + target.barrier(); + MonetUrlParser.parse(target, rest); } catch (URISyntaxException e) { parseError = e; + } catch (ValidationError e) { + validationError = e; } - if (parseError == null) { + if (parseError == null && validationError == null) { try { - validated = new Target(props); + tryValidate(); } catch (ValidationError e) { validationError = e; } @@ -269,63 +292,14 @@ else if (actual instanceof Integer) throw new Failure("Expected " + key + "=<" + expectedString + ">, found <" + actual + ">"); } - private Target tryValidate() throws ValidationError { + private Target.Validated tryValidate() throws ValidationError { if (validated == null) - validated = new Target(props); + validated = target.validate(); return validated; } private Object extract(String key) throws ValidationError, Failure { switch (key) { - case "tls": - return Target.validateBoolean(props, Parameter.TLS); - case "host": - return Target.validateString(props, Parameter.HOST); - case "port": - return Target.validateInt(props, Parameter.PORT); - case "database": - return Target.validateString(props, Parameter.DATABASE); - case "tableschema": - return Target.validateString(props, Parameter.TABLESCHEMA); - case "table": - return Target.validateString(props, Parameter.TABLE); - case "sock": - return Target.validateString(props, Parameter.SOCK); - case "sockdir": - return Target.validateString(props, Parameter.SOCKDIR); - case "cert": - return Target.validateString(props, Parameter.CERT); - case "certhash": - return Target.validateString(props, Parameter.CERTHASH); - case "clientkey": - return Target.validateString(props, Parameter.CLIENTKEY); - case "clientcert": - return Target.validateString(props, Parameter.CLIENTCERT); - case "user": - return Target.validateString(props, Parameter.USER); - case "password": - return Target.validateString(props, Parameter.PASSWORD); - case "language": - return Target.validateString(props, Parameter.LANGUAGE); - case "autocommit": - return Target.validateBoolean(props, Parameter.AUTOCOMMIT); - case "schema": - return Target.validateString(props, Parameter.SCHEMA); - case "timezone": - return Target.validateInt(props, Parameter.TIMEZONE); - case "binary": - return Target.validateString(props, Parameter.BINARY); - case "replysize": - return Target.validateInt(props, Parameter.REPLYSIZE); - case "fetchsize": - return Target.validateInt(props, Parameter.FETCHSIZE); - case "hash": - return Target.validateString(props, Parameter.HASH); - case "debug": - return Target.validateBoolean(props, Parameter.DEBUG); - case "logfile": - return Target.validateString(props, Parameter.LOGFILE); - case "valid": try { tryValidate(); @@ -361,7 +335,11 @@ private Object extract(String key) throws ValidationError, Failure { return tryValidate().connectClientCert(); default: - throw new Failure("Unknown attribute: " + key); + Parameter parm = Parameter.forName(key); + if (parm != null) + return target.getObject(parm); + else + throw new Failure("Unknown attribute: " + key); } } diff --git a/tests/tests.md b/tests/tests.md index 5fb5a2a..ab37f2a 100644 --- a/tests/tests.md +++ b/tests/tests.md @@ -598,14 +598,11 @@ EXPECT replysize=600 ``` ```test -NOT jdbc SET replysize=200 SET fetchsize=300 EXPECT replysize=300 ``` - - Rule: parsing a URL sets all of tls, host, port and database even if left out of the URL @@ -780,6 +777,10 @@ EXPECT connect_tcp=::1 EXPECT database=foo ``` +```test +REJECT monetdb://[::1]banana/foo +``` + Bad percent escapes: ```test From 16063e44661b1922ca2d22381024e9e3b9f0048a Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Fri, 1 Dec 2023 15:47:20 +0100 Subject: [PATCH 05/41] Understand so_timeout, treat_clob_as_varchar and treat_blob_as_binary --- .../java/org/monetdb/mcl/net/Parameter.java | 6 +++ src/main/java/org/monetdb/mcl/net/Target.java | 50 ++++++++++++++++++- tests/UrlTester.java | 15 +++++- tests/tests.md | 28 +++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java index e815ac6..fb236a4 100644 --- a/src/main/java/org/monetdb/mcl/net/Parameter.java +++ b/src/main/java/org/monetdb/mcl/net/Parameter.java @@ -29,6 +29,9 @@ public enum Parameter { DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false), LOGFILE("logfile", ParameterType.Str, "", "specific to jdbc", false), + SO_TIMEOUT("so_timeout", ParameterType.Int, 0, "abort if network I/O does not complete in this many milliseconds", false), + CLOB_AS_VARCHAR("treat_clob_as_varchar", ParameterType.Bool, true, "return CLOB/TEXT data as type VARCHAR instead of type CLOB", false), + BLOB_AS_BINARY("treat_blob_as_binary", ParameterType.Bool, true, "return BLOB data as type BINARY instead of type BLOB", false), ; public final String name; @@ -71,6 +74,9 @@ public static Parameter forName(String name) { case "hash": return HASH; case "debug": return DEBUG; case "logfile": return LOGFILE; + case "so_timeout": return SO_TIMEOUT; + case "treat_clob_as_varchar": return CLOB_AS_VARCHAR; + case "treat_blob_as_binary": return BLOB_AS_BINARY; default: return null; } } diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index 67c25cf..168535b 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -28,6 +28,9 @@ public class Target { private String hash = ""; private boolean debug = false; private String logfile = ""; + private int soTimeout = 0; + private boolean treatClobAsVarchar = true; + private boolean treatBlobAsBinary = true; private boolean userWasSet = false; private boolean passwordWasSet = false; @@ -102,8 +105,13 @@ private void assign(Parameter parm, Object value) { case HASH: setHash((String)value); break; case DEBUG: setDebug((boolean)value); break; case LOGFILE: setLogfile((String)value); break; + + case SO_TIMEOUT: setSoTimeout((int)value); break; + case CLOB_AS_VARCHAR: setTreatClobAsVarchar((boolean)value); break; + case BLOB_AS_BINARY: setTreatBlobAsBinary((boolean)value); break; + default: - throw new IllegalStateException("unreachable -- missing case"); + throw new IllegalStateException("unreachable -- missing case: " + parm.name); } } @@ -138,6 +146,9 @@ public Object getObject(Parameter parm) { case HASH: return hash; case DEBUG: return debug; case LOGFILE: return logfile; + case SO_TIMEOUT: return soTimeout; + case CLOB_AS_VARCHAR: return treatClobAsVarchar; + case BLOB_AS_BINARY: return treatBlobAsBinary; default: throw new IllegalStateException("unreachable -- missing case"); } @@ -340,6 +351,31 @@ public void setLogfile(String logfile) { this.logfile = logfile; } + public int getSoTimeout() { + return soTimeout; + } + + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + } + + public void setTreatClobAsVarchar(boolean treatClobAsVarchar) { + this.treatClobAsVarchar = treatClobAsVarchar; + } + + public boolean isTreatClobAsVarchar() { + return treatClobAsVarchar; + } + + public boolean isTreatBlobAsBinary() { + return treatBlobAsBinary; + } + + public void setTreatBlobAsBinary(boolean treatBlobAsBinary) { + this.treatBlobAsBinary = treatBlobAsBinary; + } + public Validated validate() throws ValidationError { return new Validated(); } @@ -519,6 +555,18 @@ public String getLogfile() { return logfile; } + public int getSoTimeout() { + return soTimeout; + } + + public boolean isTreatClobAsVarchar() { + return treatClobAsVarchar; + } + + public boolean isTreatBlobAsBinary() { + return treatBlobAsBinary; + } + public boolean connectScan() { if (database.isEmpty()) return false; if (!sock.isEmpty() || !host.isEmpty() || port != -1) return false; diff --git a/tests/UrlTester.java b/tests/UrlTester.java index a510a3d..00093ba 100644 --- a/tests/UrlTester.java +++ b/tests/UrlTester.java @@ -21,6 +21,7 @@ public UrlTester(String filename) { public static void main(String[] args) throws Exception { checkDefaults(); + checkParameters(); int exitcode; UrlTester tester = new UrlTester(); @@ -30,6 +31,7 @@ public static void main(String[] args) throws Exception { System.exit(exitcode); } + private static void checkDefaults() { Target target = new Target(); @@ -39,8 +41,17 @@ private static void checkDefaults() { continue; Object actual = target.getObject(parm); if (!expected.equals(actual)) { - System.err.println("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">"); - System.exit(1); + throw new RuntimeException("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">"); + } + } + } + + private static void checkParameters() { + for (Parameter parm: Parameter.values()) { + Parameter found = Parameter.forName(parm.name); + if (parm != found) { + String foundStr = found != null ? found.name : "null"; + throw new RuntimeException("Looking up <" + parm.name + ">, found <" + foundStr); } } } diff --git a/tests/tests.md b/tests/tests.md index ab37f2a..73148d2 100644 --- a/tests/tests.md +++ b/tests/tests.md @@ -1457,3 +1457,31 @@ EXPECT language=sql ACCEPT mapi:monetdb://localhost:12345/db?_l%61nguage=mal ``` +# lalala Java + +```test +ONLY jdbc +EXPECT so_timeout=0 +SET so_timeout=42 +EXPECT so_timeout=42 +ACCEPT monetdb://?so_timeout=99 +EXPECT so_timeout=99 +``` + +```test +ONLY jdbc +EXPECT treat_clob_as_varchar=true +SET treat_clob_as_varchar=off +EXPECT treat_clob_as_varchar=false +ACCEPT monetdb://?treat_clob_as_varchar=yes +EXPECT treat_clob_as_varchar=on +``` + +```test +ONLY jdbc +EXPECT treat_blob_as_binary=true +SET treat_blob_as_binary=off +EXPECT treat_blob_as_binary=false +ACCEPT monetdb://?treat_blob_as_binary=yes +EXPECT treat_blob_as_binary=on +``` From 3d159139bba0d1d2e9116875e3b4e3b4b026ed78 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Wed, 6 Dec 2023 16:17:13 +0100 Subject: [PATCH 06/41] Use the new url parser --- .../org/monetdb/jdbc/MonetConnection.java | 293 +++---- .../java/org/monetdb/jdbc/MonetDriver.java | 65 +- .../java/org/monetdb/mcl/MCLException.java | 8 +- .../org/monetdb/mcl/io/BufferedMCLReader.java | 8 + .../java/org/monetdb/mcl/net/MapiSocket.java | 713 ++++++++---------- .../org/monetdb/mcl/net/MonetUrlParser.java | 92 ++- src/main/java/org/monetdb/mcl/net/Target.java | 45 +- tests/tests.md | 34 + 8 files changed, 574 insertions(+), 684 deletions(-) diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java index 3f095d7..6e3b916 100644 --- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -25,7 +25,6 @@ import java.sql.Savepoint; import java.sql.Statement; import java.util.ArrayList; -import java.util.Calendar; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -36,8 +35,8 @@ import org.monetdb.mcl.io.BufferedMCLReader; import org.monetdb.mcl.io.BufferedMCLWriter; import org.monetdb.mcl.io.LineType; -import org.monetdb.mcl.net.HandshakeOption; import org.monetdb.mcl.net.MapiSocket; +import org.monetdb.mcl.net.Target; import org.monetdb.mcl.parser.HeaderLineParser; import org.monetdb.mcl.parser.MCLParseException; import org.monetdb.mcl.parser.StartOfHeaderParser; @@ -74,17 +73,8 @@ public class MonetConnection extends MonetWrapper implements Connection, AutoCloseable { - /** The hostname to connect to */ - private final String hostname; - /** The port to connect on the host to */ - private int port; - /** The database to use (currently not used) */ - private final String database; - /** The username to use when authenticating */ - private final String username; - /** The password to use when authenticating */ - private final String password; - + /** All connection parameters */ + Target target; /** A connection to mserver5 using a TCP socket */ private final MapiSocket server; /** The Reader from the server */ @@ -120,6 +110,9 @@ public class MonetConnection /** The number of results we receive from the server at once */ private int curReplySize = 100; // server default + private boolean sizeHeaderEnabled = false; // used during handshake + private boolean timeZoneSet = false; // used during handshake + /** A template to apply to each query (like pre and post fixes), filled in constructor */ // note: it is made public to the package as queryTempl[2] is used from MonetStatement @@ -137,11 +130,6 @@ public class MonetConnection /** The language which is used */ private final int lang; - /** Whether or not BLOB is mapped to Types.VARBINARY instead of Types.BLOB within this connection */ - private boolean treatBlobAsVarBinary = true; // turned on by default for optimal performance (from JDBC Driver release 3.0 onwards) - /** Whether or not CLOB is mapped to Types.VARCHAR instead of Types.CLOB within this connection */ - private boolean treatClobAsVarChar = true; // turned on by default for optimal performance (from JDBC Driver release 3.0 onwards) - /** The last set query timeout on the server as used by Statement, PreparedStatement and CallableStatement */ protected int lastSetQueryTimeout = 0; // 0 means no timeout, which is the default on the server @@ -155,137 +143,23 @@ public class MonetConnection * createStatement() call. This constructor is only accessible to * classes from the jdbc package. * - * @param props a Property hashtable holding the properties needed for connecting + * @param target a Target object holding the connection parameters * @throws SQLException if a database error occurs * @throws IllegalArgumentException is one of the arguments is null or empty */ - MonetConnection(final Properties props) + MonetConnection(Target target) throws SQLException, IllegalArgumentException { - HandshakeOption.AutoCommit autoCommitSetting = new HandshakeOption.AutoCommit(true); - HandshakeOption.ReplySize replySizeSetting = new HandshakeOption.ReplySize(DEF_FETCHSIZE); - HandshakeOption.SizeHeader sizeHeaderSetting = new HandshakeOption.SizeHeader(true); - HandshakeOption.TimeZone timeZoneSetting = new HandshakeOption.TimeZone(0); - - // for debug: System.out.println("New connection object. Received properties are: " + props.toString()); - // get supported property values from the props argument. - this.hostname = props.getProperty("host"); - - final String port_prop = props.getProperty("port"); - if (port_prop != null) { - try { - this.port = Integer.parseInt(port_prop); - } catch (NumberFormatException e) { - addWarning("Unable to parse port number from: " + port_prop, "M1M05"); - } - } - - this.database = props.getProperty("database"); - this.username = props.getProperty("user"); - this.password = props.getProperty("password"); - String language = props.getProperty("language"); - - boolean debug = false; - String debug_prop = props.getProperty("debug"); - if (debug_prop != null) { - debug = Boolean.parseBoolean(debug_prop); - } - - final String hash = props.getProperty("hash"); - - String autocommit_prop = props.getProperty("autocommit"); - if (autocommit_prop != null) { - boolean ac = Boolean.parseBoolean(autocommit_prop); - autoCommitSetting.set(ac); - } - - final String fetchsize_prop = props.getProperty("fetchsize"); - if (fetchsize_prop != null) { - try { - int fetchsize = Integer.parseInt(fetchsize_prop); - if (fetchsize > 0 || fetchsize == -1) { - replySizeSetting.set(fetchsize); - } else { - addWarning("Fetch size must either be positive or -1. Value " + fetchsize + " ignored", "M1M05"); - } - } catch (NumberFormatException e) { - addWarning("Unable to parse fetch size number from: " + fetchsize_prop, "M1M05"); - } - } - - final String treatBlobAsVarBinary_prop = props.getProperty("treat_blob_as_binary"); - if (treatBlobAsVarBinary_prop != null) { - treatBlobAsVarBinary = Boolean.parseBoolean(treatBlobAsVarBinary_prop); - if (treatBlobAsVarBinary) - typeMap.put("blob", Byte[].class); - } - - final String treatClobAsVarChar_prop = props.getProperty("treat_clob_as_varchar"); - if (treatClobAsVarChar_prop != null) { - treatClobAsVarChar = Boolean.parseBoolean(treatClobAsVarChar_prop); - if (treatClobAsVarChar) - typeMap.put("clob", String.class); - } - - int sockTimeout = 0; - final String so_timeout_prop = props.getProperty("so_timeout"); - if (so_timeout_prop != null) { - try { - sockTimeout = Integer.parseInt(so_timeout_prop); - if (sockTimeout < 0) { - addWarning("Negative socket timeout not allowed. Value ignored", "M1M05"); - sockTimeout = 0; - } - } catch (NumberFormatException e) { - addWarning("Unable to parse socket timeout number from: " + so_timeout_prop, "M1M05"); - } - } - - // check mandatory input arguments - if (hostname == null || hostname.isEmpty()) - throw new IllegalArgumentException("Missing or empty host name"); - if (port <= 0 || port > 65535) - throw new IllegalArgumentException("Invalid port number: " + port - + ". It should not be " + (port < 0 ? "negative" : (port > 65535 ? "larger than 65535" : "0"))); - if (username == null || username.isEmpty()) - throw new IllegalArgumentException("Missing or empty user name"); - if (password == null || password.isEmpty()) - throw new IllegalArgumentException("Missing or empty password"); - if (language == null || language.isEmpty()) { - // fallback to default language: sql - language = "sql"; - addWarning("No language specified, defaulting to 'sql'", "M1M05"); - } - - // warn about unrecognized property names - for (Entry e: props.entrySet()) { - checkValidProperty(e.getKey().toString(), "MonetConnection"); - } - + this.target = target; server = new MapiSocket(); - if (hash != null) - server.setHash(hash); - if (database != null) - server.setDatabase(database); - server.setLanguage(language); - - // calculate our time zone offset - final Calendar cal = Calendar.getInstance(); - final int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET); - final int offsetSeconds = offsetMillis / 1000; - timeZoneSetting.set(offsetSeconds); - - server.setHandshakeOptions(new HandshakeOption[] { - autoCommitSetting, - replySizeSetting, - sizeHeaderSetting, - timeZoneSetting, - }); // we're debugging here... uhm, should be off in real life - if (debug) { + if (target.isDebug()) { try { - final String fname = props.getProperty("logfile", "monet_" + System.currentTimeMillis() + ".log"); + String fname = target.getLogfile(); + if (fname == null) + fname = "monet_" + System.currentTimeMillis() + ".log"; + File f = new File(fname); int ext = fname.lastIndexOf('.'); @@ -304,26 +178,45 @@ public class MonetConnection } } + SqlOptionsCallback callback = null; + switch (target.getLanguage()) { + case "sql": + lang = LANG_SQL; + queryTempl[0] = "s"; // pre + queryTempl[1] = "\n;"; // post + queryTempl[2] = "\n;\n"; // separator + commandTempl[0] = "X"; // pre + commandTempl[1] = ""; // post + callback = new SqlOptionsCallback(); + break; + case "mal": + lang = LANG_MAL; + queryTempl[0] = ""; // pre + queryTempl[1] = ";\n"; // post + queryTempl[2] = ";\n"; // separator + commandTempl[0] = ""; // pre + commandTempl[1] = ""; // post + break; + default: + lang = LANG_UNKNOWN; + break; + } + try { - final java.util.List warnings = server.connect(hostname, port, username, password); + + final java.util.List warnings = server.connect(target, callback); for (String warning : warnings) { addWarning(warning, "01M02"); } - // apply NetworkTimeout value from legacy (pre 4.1) driver - // so_timeout calls - server.setSoTimeout(sockTimeout); - in = server.getReader(); out = server.getWriter(); final String error = in.discardRemainder(); if (error != null) throw new SQLNonTransientConnectionException((error.length() > 6) ? error.substring(6) : error, "08001"); - } catch (java.net.UnknownHostException e) { - throw new SQLNonTransientConnectionException("Unknown Host (" + hostname + "): " + e.getMessage(), "08006"); } catch (IOException e) { - throw new SQLNonTransientConnectionException("Unable to connect (" + hostname + ":" + port + "): " + e.getMessage(), "08006"); + throw new SQLNonTransientConnectionException("Cannot connect: " + e.getMessage(), "08006"); } catch (MCLParseException e) { throw new SQLNonTransientConnectionException(e.getMessage(), "08001"); } catch (org.monetdb.mcl.MCLException e) { @@ -335,53 +228,17 @@ public class MonetConnection throw sqle; } - // we seem to have managed to log in, let's store the - // language used and language specific query templates - if ("sql".equals(language)) { - lang = LANG_SQL; - - queryTempl[0] = "s"; // pre - queryTempl[1] = "\n;"; // post - queryTempl[2] = "\n;\n"; // separator - - commandTempl[0] = "X"; // pre - commandTempl[1] = ""; // post - //commandTempl[2] = "\nX"; // separator (is not used) - } else if ("mal".equals(language)) { - lang = LANG_MAL; - - queryTempl[0] = ""; // pre - queryTempl[1] = ";\n"; // post - queryTempl[2] = ";\n"; // separator - - commandTempl[0] = ""; // pre - commandTempl[1] = ""; // post - //commandTempl[2] = ""; // separator (is not used) - } else { - lang = LANG_UNKNOWN; - } - // Now take care of any handshake options not handled during the handshake - if (replySizeSetting.isSent()) { - this.curReplySize = replySizeSetting.get(); - } - this.defaultFetchSize = replySizeSetting.get(); + curReplySize = defaultFetchSize; if (lang == LANG_SQL) { - if (autoCommitSetting.mustSend(autoCommit)) { - setAutoCommit(autoCommitSetting.get()); - } else { - // update bookkeeping - autoCommit = autoCommitSetting.get(); + if (autoCommit != target.isAutocommit()) { + setAutoCommit(target.isAutocommit()); } - if (sizeHeaderSetting.mustSend(false)) { + if (!sizeHeaderEnabled) { sendControlCommand("sizeheader 1"); - } else { - // no bookkeeping to update } - if (timeZoneSetting.mustSend(0)) { - setTimezone(timeZoneSetting.get()); - } else { - // no bookkeeping to update + if (!timeZoneSet) { + setTimezone(target.getTimezone()); } } @@ -1780,7 +1637,7 @@ void setQueryTimeout(final int seconds) throws SQLException { * @return whether the JDBC BLOB type should be mapped to VARBINARY type. */ boolean mapBlobAsVarBinary() { - return treatBlobAsVarBinary; + return target.isTreatBlobAsBinary(); } /** @@ -1791,7 +1648,7 @@ boolean mapBlobAsVarBinary() { * @return whether the JDBC CLOB type should be mapped to VARCHAR type. */ boolean mapClobAsVarChar() { - return treatClobAsVarChar; + return target.isTreatClobAsVarchar(); } /** @@ -1800,13 +1657,7 @@ boolean mapClobAsVarChar() { * @return the MonetDB JDBC Connection URL (without user name and password). */ String getJDBCURL() { - final StringBuilder sb = new StringBuilder(128); - sb.append("jdbc:monetdb://").append(hostname) - .append(':').append(port) - .append('/').append(database); - if (lang == LANG_MAL) - sb.append("?language=mal"); - return sb.toString(); + return target.buildUrl(); } /** @@ -3887,4 +3738,48 @@ public void close() throws IOException { super.close(); } } + + public static enum SqlOption { + Autocommit(1, "auto_commit"), + ReplySize(2, "reply_size"), + SizeHeader(3, "size_header"), + // NOTE: 4 has been omitted on purpose + TimeZone(5, "time_zone"), + ; + final int level; + final String field; + + SqlOption(int level, String field) { + this.level = level; + this.field = field; + } + } + + + private class SqlOptionsCallback extends MapiSocket.OptionsCallback { + private int level; + @Override + public void addOptions(String lang, int level) { + if (!lang.equals("sql")) + return; + this.level = level; + + // Try to add options and record that this happened if it succeeds. + if (contribute(SqlOption.Autocommit, target.isAutocommit() ? 1 : 0)) + autoCommit = target.isAutocommit(); + if (contribute(SqlOption.ReplySize, target.getReplysize())) + defaultFetchSize = target.getReplysize(); + if (contribute(SqlOption.SizeHeader, 1)) + sizeHeaderEnabled = true; + if (contribute(SqlOption.TimeZone, target.getTimezone())) + timeZoneSet = true; + } + + private boolean contribute(SqlOption opt, int value) { + if (this.level <= opt.level) + return false; + contribute(opt.field, value); + return true; + } + } } diff --git a/src/main/java/org/monetdb/jdbc/MonetDriver.java b/src/main/java/org/monetdb/jdbc/MonetDriver.java index 367a2d3..0d91466 100644 --- a/src/main/java/org/monetdb/jdbc/MonetDriver.java +++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java @@ -8,7 +8,12 @@ package org.monetdb.jdbc; -import java.net.URI; +import org.monetdb.mcl.net.MonetUrlParser; +import org.monetdb.mcl.net.Parameter; +import org.monetdb.mcl.net.Target; +import org.monetdb.mcl.net.ValidationError; + +import java.net.URISyntaxException; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; @@ -97,56 +102,24 @@ public Connection connect(final String url, Properties info) if (!acceptsURL(url)) return null; - final Properties props = new Properties(); - // set the optional properties and their defaults here - props.put("port", "50000"); - props.put("debug", "false"); - props.put("language", "sql"); // mal, sql, - props.put("so_timeout", "0"); - - if (info != null) - props.putAll(info); - info = props; + Target target = new Target(); - // remove leading "jdbc:" so the rest is a valid hierarchical URI - final URI uri; try { - uri = new URI(url.substring(5)); - } catch (java.net.URISyntaxException e) { - return null; - } - - final String uri_host = uri.getHost(); - if (uri_host == null) - return null; - info.put("host", uri_host); - - int uri_port = uri.getPort(); - if (uri_port > 0) - info.put("port", Integer.toString(uri_port)); - - // check the database - String uri_path = uri.getPath(); - if (uri_path != null && !uri_path.isEmpty()) { - uri_path = uri_path.substring(1).trim(); - if (!uri_path.isEmpty()) - info.put("database", uri_path); - } - - final String uri_query = uri.getQuery(); - if (uri_query != null) { - int pos; - // handle additional connection properties separated by the & character - final String args[] = uri_query.split("&"); - for (int i = 0; i < args.length; i++) { - pos = args[i].indexOf('='); - if (pos > 0) - info.put(args[i].substring(0, pos), args[i].substring(pos + 1)); + if (info != null) { + for (String key : info.stringPropertyNames()) { + String value = info.getProperty(key); + if (key.equals(Parameter.HOST.name)) + value = Target.unpackHost(value); + target.setString(key, value); + } } + MonetUrlParser.parse(target, url.substring(5)); + } catch (ValidationError | URISyntaxException e) { + throw new SQLException(e.getMessage()); } - // finally return the Connection object as requested - return new MonetConnection(info); + // finally return the Connection object as requested + return new MonetConnection(target); } /** diff --git a/src/main/java/org/monetdb/mcl/MCLException.java b/src/main/java/org/monetdb/mcl/MCLException.java index 6be3062..4cfa786 100644 --- a/src/main/java/org/monetdb/mcl/MCLException.java +++ b/src/main/java/org/monetdb/mcl/MCLException.java @@ -15,7 +15,11 @@ public final class MCLException extends Exception { private static final long serialVersionUID = 1L; - public MCLException(String e) { - super(e); + public MCLException(String message) { + super(message); + } + + public MCLException(String message, Exception cause) { + super(message, cause); } } diff --git a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java index 84aeb44..6137851 100644 --- a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java +++ b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java @@ -94,6 +94,14 @@ public String getLine() { return current; } + /** + * Return a substring of the current line, or null if we're at the end or before the beginning. + * @return the current line or null + */ + public String getLine(int start) { + return getLine().substring(start); + } + /** * getLineType returns the type of the current line. * diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java index ef420eb..a8ca99f 100644 --- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java +++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java @@ -17,16 +17,17 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.Writer; -import java.net.Socket; -import java.net.SocketException; -import java.net.UnknownHostException; -import java.net.URI; +import java.net.*; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.monetdb.mcl.MCLException; import org.monetdb.mcl.io.BufferedMCLReader; @@ -85,10 +86,21 @@ * @see org.monetdb.mcl.io.BufferedMCLWriter */ public final class MapiSocket { + private static final String[][] KNOWN_ALGORITHMS = new String[][] { + {"SHA512", "SHA-512"}, + {"SHA384", "SHA-384"}, + {"SHA256", "SHA-256"}, + // should we deprecate this by now? + {"SHA1", "SHA-1"}, + }; + + // MUST be lowercase! + private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray(); + + /** Connection parameters */ + private Target target; /** The TCP Socket to mserver */ private Socket con; - /** The TCP Socket timeout in milliseconds. Default is 0 meaning the timeout is disabled (i.e., timeout of infinity) */ - private int soTimeout = 0; /** Stream from the Socket for reading */ private BlockInputStream fromMonet; /** Stream from the Socket for writing */ @@ -100,36 +112,25 @@ public final class MapiSocket { /** protocol version of the connection */ private int version; - /** The database to connect to */ - private String database = null; - /** The language to connect with */ - private String language = "sql"; - /** The hash methods to use (null = default) */ - private String hash = null; - /** Whether we should follow redirects */ private boolean followRedirects = true; /** How many redirections do we follow until we're fed up with it? */ private int ttl = 10; - /** Whether we are debugging or not */ - private boolean debug = false; /** The Writer for the debug log-file */ private Writer log; /** The blocksize (hardcoded in compliance with MonetDB common/stream/stream.h) */ - public final static int BLOCK = 8 * 1024 - 2; + public final static int BLOCK = 8190; /** A short in two bytes for holding the block size in bytes */ private final byte[] blklen = new byte[2]; - /** Options that can be sent during the auth handshake if the server supports it */ - private HandshakeOption[] handshakeOptions; - /** * Constructs a new MapiSocket. */ public MapiSocket() { + target = new Target(); con = null; } @@ -141,7 +142,7 @@ public MapiSocket() { * @param db the database */ public void setDatabase(final String db) { - this.database = db; + target.setDatabase(db); } /** @@ -150,7 +151,7 @@ public void setDatabase(final String db) { * @param lang the language */ public void setLanguage(final String lang) { - this.language = lang; + target.setLanguage(lang); } /** @@ -163,7 +164,7 @@ public void setLanguage(final String lang) { * @param hash the hash method to use */ public void setHash(final String hash) { - this.hash = hash; + target.setHash(hash); } /** @@ -208,7 +209,7 @@ public void setSoTimeout(final int s) throws SocketException { if (s < 0) { throw new IllegalArgumentException("timeout can't be negative"); } - this.soTimeout = s; + target.setSoTimeout(s); // limit time to wait on blocking operations if (con != null) { con.setSoTimeout(s); @@ -222,10 +223,7 @@ public void setSoTimeout(final int s) throws SocketException { * @throws SocketException Issue with the socket */ public int getSoTimeout() throws SocketException { - if (con != null) { - this.soTimeout = con.getSoTimeout(); - } - return this.soTimeout; + return target.getSoTimeout(); } /** @@ -234,7 +232,7 @@ public int getSoTimeout() throws SocketException { * @param debug Value to set */ public void setDebug(final boolean debug) { - this.debug = debug; + target.setDebug(debug); } /** @@ -257,376 +255,300 @@ public void setDebug(final boolean debug) { public List connect(final String host, final int port, final String user, final String pass) throws IOException, SocketException, UnknownHostException, MCLParseException, MCLException { - // Wrap around the internal connect that needs to know if it - // should really make a TCP connection or not. - return connect(host, port, user, pass, true); + target.setHost(host); + target.setPort(port); + target.setUser(user); + target.setPassword(pass); + return connect(target, null); } - /** - * Connects to the given host and port, logging in as the given - * user. If followRedirect is false, a RedirectionException is - * thrown when a redirect is encountered. - * - * @param host the hostname, or null for the loopback address - * @param port the port number (must be between 0 and 65535, inclusive) - * @param user the username - * @param pass the password - * @param makeConnection whether a new socket connection needs to be created - * @return A List with informational (warning) messages. If this - * list is empty; then there are no warnings. - * @throws IOException if an I/O error occurs when creating the socket - * @throws SocketException - if there is an error in the underlying protocol, such as a TCP error. - * @throws UnknownHostException if the IP address of the host could not be determined - * @throws MCLParseException if bogus data is received - * @throws MCLException if an MCL related error occurs - */ - private List connect(final String host, final int port, final String user, final String pass, final boolean makeConnection) - throws IOException, SocketException, UnknownHostException, MCLParseException, MCLException - { - if (ttl-- <= 0) - throw new MCLException("Maximum number of redirects reached, aborting connection attempt."); - - if (makeConnection) { - con = new Socket(host, port); - con.setSoTimeout(this.soTimeout); - // set nodelay, as it greatly speeds up small messages (like we often do) - con.setTcpNoDelay(true); - con.setKeepAlive(true); - - fromMonet = new BlockInputStream(con.getInputStream()); - toMonet = new BlockOutputStream(con.getOutputStream()); - reader = new BufferedMCLReader(fromMonet, StandardCharsets.UTF_8); - writer = new BufferedMCLWriter(toMonet, StandardCharsets.UTF_8); - writer.registerReader(reader); + public List connect(Target target, OptionsCallback callback) throws MCLException, MCLParseException, IOException { + // get rid of any earlier connection state, including the existing target + close(); + this.target = target; + + Target.Validated validated; + try { + validated = target.validate(); + } catch (ValidationError e) { + throw new MCLException(e.getMessage()); } - reader.advance(); - final String c = reader.getLine(); - reader.discardRemainder(); - writer.writeLine(getChallengeResponse(c, user, pass, language, database, hash)); - - // read monetdb mserver response till prompt - final ArrayList redirects = new ArrayList(); - final List warns = new ArrayList(); - String err = "", tmp; + if (validated.connectScan()) { + return scanUnixSockets(callback); + } + + ArrayList warnings = new ArrayList<>(); + int attempts = 0; do { - reader.advance(); - tmp = reader.getLine(); - if (tmp == null) - throw new IOException("Read from " + - con.getInetAddress().getHostName() + ":" + - con.getPort() + ": End of stream reached"); - if (reader.getLineType() == LineType.ERROR) { - err += "\n" + tmp.substring(7); - } else if (reader.getLineType() == LineType.INFO) { - warns.add(tmp.substring(1)); - } else if (reader.getLineType() == LineType.REDIRECT) { - redirects.add(tmp.substring(1)); + boolean ok = false; + try { + boolean done = tryConnect(callback, warnings); + ok = true; + if (done) { + return warnings; + } + } finally { + if (!ok) + close(); } - } while (reader.getLineType() != LineType.PROMPT); + } while (attempts++ < this.ttl); + throw new MCLException("max redirect count exceeded"); + } - if (err.length() > 0) { + private List scanUnixSockets(OptionsCallback callback) throws MCLException, MCLParseException, IOException { + // Because we do not support Unix Domain sockets, we just go back to connect(). + // target.connectScan() will now return false; + target.setHost("localhost"); + return connect(target, callback); + } + + private boolean tryConnect(OptionsCallback callback, ArrayList warningBuffer) throws MCLException, IOException { + try { + // We need a valid target + Target.Validated validated = target.validate(); + // con will be non-null if the previous attempt ended in a redirect to mapi:monetdb://proxy + if (con == null) + connectSocket(validated); + return handshake(validated, callback, warningBuffer); + } catch (IOException | MCLException e) { close(); - throw new MCLException(err); + throw e; + } catch (ValidationError e) { + close(); + throw new MCLException(e.getMessage()); + } + } + + private void connectSocket(Target.Validated validated) throws MCLException, IOException { + // This method performs steps 2-6 of the procedure outlined in the URL spec + String tcpHost = validated.connectTcp(); + if (tcpHost.isEmpty()) { + throw new MCLException("Unix domain sockets are not supported, only TCP"); } + int port = validated.connectPort(); + Socket sock = new Socket(tcpHost, port); + sock.setSoTimeout(validated.getSoTimeout()); + sock.setTcpNoDelay(true); + sock.setKeepAlive(true); + + sock = wrapTLS(sock, validated); + + fromMonet = new BlockInputStream(sock.getInputStream()); + toMonet = new BlockOutputStream(sock.getOutputStream()); + reader = new BufferedMCLReader(fromMonet, StandardCharsets.UTF_8); + writer = new BufferedMCLWriter(toMonet, StandardCharsets.UTF_8); + writer.registerReader(reader); + reader.advance(); - if (!redirects.isEmpty()) { - if (followRedirects) { - // Ok, server wants us to go somewhere else. The list - // might have multiple clues on where to go. For now we - // don't support anything intelligent but trying the - // first one. URI should be in form of: - // "mapi:monetdb://host:port/database?arg=value&..." - // or - // "mapi:merovingian://proxy?arg=value&..." - // note that the extra arguments must be obeyed in both - // cases - final String suri = redirects.get(0).toString(); - if (!suri.startsWith("mapi:")) - throw new MCLException("unsupported redirect: " + suri); - - final URI u; - try { - u = new URI(suri.substring(5)); - } catch (java.net.URISyntaxException e) { - throw new MCLParseException(e.toString()); - } + // Only assign to sock when everything went ok so far + con = sock; + } - tmp = u.getQuery(); - if (tmp != null) { - final String args[] = tmp.split("&"); - for (int i = 0; i < args.length; i++) { - int pos = args[i].indexOf('='); - if (pos > 0) { - tmp = args[i].substring(0, pos); - switch (tmp) { - case "database": - tmp = args[i].substring(pos + 1); - if (!tmp.equals(database)) { - warns.add("redirect points to different database: " + tmp); - setDatabase(tmp); - } - break; - case "language": - tmp = args[i].substring(pos + 1); - warns.add("redirect specifies use of different language: " + tmp); - setLanguage(tmp); - break; - case "user": - tmp = args[i].substring(pos + 1); - if (!tmp.equals(user)) - warns.add("ignoring different username '" + tmp + "' set by " + - "redirect, what are the security implications?"); - break; - case "password": - warns.add("ignoring different password set by redirect, " + - "what are the security implications?"); - break; - default: - warns.add("ignoring unknown argument '" + tmp + "' from redirect"); - break; - } - } else { - warns.add("ignoring illegal argument from redirect: " + args[i]); - } - } - } + private Socket wrapTLS(Socket sock, Target.Validated validated) throws MCLException { + if (validated.getTls()) + throw new MCLException("TLS connections (monetdbs://) are not supported yet"); + return sock; + } - if (u.getScheme().equals("monetdb")) { - // this is a redirect to another (monetdb) server, - // which means a full reconnect - // avoid the debug log being closed - if (debug) { - debug = false; - close(); - debug = true; - } else { - close(); - } - tmp = u.getPath(); - if (tmp != null && tmp.length() > 0) { - tmp = tmp.substring(1).trim(); - if (!tmp.isEmpty() && !tmp.equals(database)) { - warns.add("redirect points to different database: " + tmp); - setDatabase(tmp); - } - } - final int p = u.getPort(); - warns.addAll(connect(u.getHost(), p == -1 ? port : p, user, pass, true)); - warns.add("Redirect by " + host + ":" + port + " to " + suri); - } else if (u.getScheme().equals("merovingian")) { - // reuse this connection to inline connect to the - // right database that Merovingian proxies for us - reader.resetLineType(); - warns.addAll(connect(host, port, user, pass, false)); - } else { - throw new MCLException("unsupported scheme in redirect: " + suri); - } - } else { - final StringBuilder msg = new StringBuilder("The server sent a redirect for this connection:"); - for (String it : redirects) { - msg.append(" [" + it + "]"); - } - throw new MCLException(msg.toString()); + private boolean handshake(Target.Validated validated, OptionsCallback callback, ArrayList warnings) throws IOException, MCLException { + String challenge = reader.getLine(); + reader.advance(); + if (reader.getLineType() != LineType.PROMPT) + throw new MCLException("Garbage after server challenge: " + reader.getLine()); + String response = challengeResponse(validated, challenge, callback); + writer.writeLine(response); + reader.advance(); + + // Process the response lines. + String redirect = null; + StringBuilder errors = new StringBuilder(); + while (reader.getLineType() != LineType.PROMPT) { + switch (reader.getLineType()) { + case REDIRECT: + if (redirect == null) + redirect = reader.getLine(1); + break; + case ERROR: + if (errors.length() > 0) + errors.append("\n"); + errors.append(reader.getLine(7)); // 7 not 1! + break; + case INFO: + warnings.add(reader.getLine(1)); + break; + default: + // ignore??!! + break; } + reader.advance(); } - return warns; - } + if (errors.length() > 0) + throw new MCLException(errors.toString()); - /** - * A little helper function that processes a challenge string, and - * returns a response string for the server. If the challenge - * string is null, a challengeless response is returned. - * - * @param chalstr the challenge string - * for example: H8sRMhtevGd:mserver:9:PROT10,RIPEMD160,SHA256,SHA1,COMPRESSION_SNAPPY,COMPRESSION_LZ4:LIT:SHA512: - * @param username the username to use - * @param password the password to use - * @param language the language to use - * @param database the database to connect to - * @param hash the hash method(s) to use, or NULL for all supported hashes - * @return the response string for the server - * @throws MCLParseException when parsing failed - * @throws MCLException if an MCL related error occurs - * @throws IOException when IO exception occurred - */ - private String getChallengeResponse( - final String chalstr, - String username, - String password, - final String language, - final String database, - final String hash - ) throws MCLParseException, MCLException, IOException { - // parse the challenge string, split it on ':' - final String[] chaltok = chalstr.split(":"); - if (chaltok.length <= 5) - throw new MCLParseException("Server challenge string unusable! It contains too few (" + chaltok.length + ") tokens: " + chalstr); + if (redirect == null) + return true; // we're happy + // process redirect try { - version = Integer.parseInt(chaltok[2]); // protocol version - } catch (NumberFormatException e) { - throw new MCLParseException("Protocol version (" + chaltok[2] + ") unparseable as integer."); + MonetUrlParser.parse(target, redirect); + } catch (URISyntaxException | ValidationError e) { + throw new MCLException("While processing redirect " + redirect + ": " + e.getMessage(), e); } + if (redirect.startsWith("mapi:merovingian://proxy")) { + // The reader is stuck at LineType.PROMPT but actually the + // next challenge is already there. + reader.resetLineType(); + reader.advance(); + } else { + close(); + } + + return false; // we need another go + } + + private String challengeResponse( + Target.Validated validated, final String challengeLine, + OptionsCallback callback + ) throws MCLException { + // The challengeLine looks like this: + // + // 45IYyVyRnbgEnK92ad:merovingian:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512: + // WgHIibSyH:mserver:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512:sql=6:BINARY=1: + // 0 1 2 3 4 5 6 7 + + String parts[] = challengeLine.split(":"); + if (parts.length < 3) + throw new MCLException("Invalid challenge: expect at least 3 fields"); + String challengePart = parts[0]; + String serverTypePart = parts[1]; + String versionPart = parts[2]; + int version; + if (versionPart.equals("9")) + version = 9; + else + throw new MCLException("Protocol versions other than 9 are note supported: " + versionPart); + if (parts.length < 6) + throw new MCLException("Protocol version " + version + " requires at least 6 fields, found " + parts.length + ": " + challengeLine); + String serverHashesPart = parts[3]; +// String endianPart = parts[4]; + String passwordHashPart = parts[5]; + String optionsPart = parts.length > 6 ? parts[6] : null; +// String binaryPart = parts.length > 7 ? parts[7] : null; + + String userResponse; + String password = target.getPassword(); + if (serverTypePart.equals("merovingian") && !target.getLanguage().equals("control")) { + userResponse = "merovingian"; + password = "merovingian"; + } else { + userResponse = target.getUser(); + } + String passwordResponse = hashPassword(challengePart, password, passwordHashPart, validated.getHash(), serverHashesPart); + + String optionsResponse = handleOptions(callback, optionsPart); + + // Response looks like this: + // + // LIT:monetdb:{RIPEMD160}f2236256e5a9b20a5ecab4396e36c14f66c3e3c5:sql:demo + // :FILETRANS:auto_commit=1,reply_size=1000,size_header=0,columnar_protocol=0,time_zone=3600: + StringBuilder response = new StringBuilder(80); + response.append("BIG:"); + response.append(userResponse).append(":"); + response.append(passwordResponse).append(":"); + response.append(validated.getLanguage()).append(":"); + response.append(validated.getDatabase()).append(":"); + response.append("FILETRANS:"); + response.append(optionsResponse).append(":"); + + return response.toString(); + } - // handle the challenge according to the version it is - switch (version) { - case 9: - // proto 9 is like 8, but uses a hash instead of the plain password - // the server tells us (in 6th token) which hash in the - // challenge after the byte-order token - - String algo; - String pwhash = chaltok[5]; - /* NOTE: Java doesn't support RIPEMD160 :( */ - /* see: https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#MessageDigest */ - switch (pwhash) { - case "SHA512": - algo = "SHA-512"; - break; - case "SHA384": - algo = "SHA-384"; - break; - case "SHA256": - algo = "SHA-256"; - /* NOTE: Java 7 doesn't support SHA-224. Java 8 does but we have not tested it. It is also not requested yet. */ - break; - case "SHA1": - algo = "SHA-1"; - break; - default: - /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ - throw new MCLException("Unsupported password hash: " + pwhash); - } - try { - final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes(StandardCharsets.UTF_8)); - password = toHex(md.digest()); - } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); - } - - // proto 7 (finally) used the challenge and works with a - // password hash. The supported implementations come - // from the server challenge. We chose the best hash - // we can find, in the order SHA512, SHA1, MD5, plain. - // Also the byte-order is reported in the challenge string, - // which makes sense, since only blockmode is supported. - // proto 8 made this obsolete, but retained the - // byte-order report for future "binary" transports. - // In proto 8, the byte-order of the blocks is always little - // endian because most machines today are. - final String hashes = (hash == null || hash.isEmpty()) ? chaltok[3] : hash; - final HashSet hashesSet = new HashSet(java.util.Arrays.asList(hashes.toUpperCase().split("[, ]"))); // split on comma or space - - // if we deal with merovingian, mask our credentials - if (chaltok[1].equals("merovingian") && !language.equals("control")) { - username = "merovingian"; - password = "merovingian"; - } + // challengePart, passwordHashPart, supportedHashesPart, target.getPassword() + private String hashPassword(String challenge, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException { + int maxHashLength = 512; - // reuse variables algo and pwhash - algo = null; - pwhash = null; - if (hashesSet.contains("SHA512")) { - algo = "SHA-512"; - pwhash = "{SHA512}"; - } else if (hashesSet.contains("SHA384")) { - algo = "SHA-384"; - pwhash = "{SHA384}"; - } else if (hashesSet.contains("SHA256")) { - algo = "SHA-256"; - pwhash = "{SHA256}"; - } else if (hashesSet.contains("SHA1")) { - algo = "SHA-1"; - pwhash = "{SHA1}"; - } else { - /* Note: MD5 has been deprecated by security experts and support is removed from Oct 2020 release */ - throw new MCLException("no supported hash algorithms found in " + hashes); - } - try { - final MessageDigest md = MessageDigest.getInstance(algo); - md.update(password.getBytes(StandardCharsets.UTF_8)); - md.update(chaltok[0].getBytes(StandardCharsets.UTF_8)); // salt/key - pwhash += toHex(md.digest()); - } catch (NoSuchAlgorithmException e) { - throw new MCLException("This JVM does not support password hash: " + pwhash + "\n" + e); - } + StringBuilder output = new StringBuilder(10 + maxHashLength / 4); + MessageDigest passwordDigest = pickBestAlgorithm(Collections.singleton(passwordAlgo), output); - // TODO: some day when we need this, we should store this - if (chaltok[4].equals("BIG")) { - // byte-order of server is big-endian - } else if (chaltok[4].equals("LIT")) { - // byte-order of server is little-endian - } else { - throw new MCLParseException("Invalid byte-order: " + chaltok[4]); - } - - // compose and return response - String response = "BIG:" // JVM byte-order is big-endian - + username + ":" - + pwhash + ":" - + language + ":" - + (database == null ? "" : database) + ":" - + "FILETRANS:"; // this capability is added in monetdb-jdbc-3.2.jre8.jar - if (chaltok.length > 6) { - // if supported, send handshake options - for (String part : chaltok[6].split(",")) { - if (part.startsWith("sql=") && handshakeOptions != null) { - int level; - try { - level = Integer.parseInt(chaltok[6].substring(4)); - } catch (NumberFormatException e) { - throw new MCLParseException("Invalid handshake level: " + chaltok[6]); - } - boolean first = true; - for (HandshakeOption opt: handshakeOptions) { - if (opt.getLevel() < level) { - // server supports it - if (first) { - first = false; - } else { - response += ","; - } - response += opt.getFieldName() + "=" + opt.numericValue(); - opt.setSent(true); - } - } - break; - } - } - // this ':' delimits the handshake options field. - response += ":"; - } - return response; - default: - throw new MCLException("Unsupported protocol version: " + version); + Set algoSet = new HashSet(Arrays.asList(serverSupportedAlgos.split(","))); + if (!configuredHashes.isEmpty()) { + Set keep = new HashSet(Arrays.asList(configuredHashes.toUpperCase().split("[, ]"))); + algoSet.retainAll(keep); + if (algoSet.isEmpty()) { + throw new MCLException("None of the hash algorithms <" + configuredHashes + "> are supported, server only supports <" + serverSupportedAlgos + ">"); + } } + MessageDigest challengeDigest = pickBestAlgorithm(algoSet, null); + + // First we use the password algo to hash the password. + // Then we use the challenge algo to hash the combination of the resulting hash digits and the challenge. + StringBuilder intermediate = new StringBuilder(maxHashLength / 4 + challenge.length()); + hexhash(intermediate, passwordDigest, password); + intermediate.append(challenge); + hexhash(output, challengeDigest, intermediate.toString()); + return output.toString(); } - /** - * Small helper method to convert a byte string to a hexadecimal - * string representation. - * - * @param digest the byte array to convert - * @return the byte array as hexadecimal string - */ - private final static String toHex(final byte[] digest) { - final char[] result = new char[digest.length * 2]; - int pos = 0; - for (int i = 0; i < digest.length; i++) { - result[pos++] = hexChar((digest[i] & 0xf0) >> 4); - result[pos++] = hexChar(digest[i] & 0x0f); - } - return new String(result); + private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendPrefixHere) throws MCLException { + for (String[] choice: KNOWN_ALGORITHMS) { + String mapiName = choice[0]; + String algoName = choice[1]; + MessageDigest digest; + if (!algos.contains(mapiName)) + continue; + try { + digest = MessageDigest.getInstance(algoName); + } catch (NoSuchAlgorithmException e) { + continue; + } + // we found a match + if (appendPrefixHere != null) { + appendPrefixHere.append('{'); + appendPrefixHere.append(mapiName); + appendPrefixHere.append('}'); + } + return digest; + } + String algoNames = algos.stream().collect(Collectors.joining()); + throw new MCLException("No supported hash algorithm: " + algoNames); } - private final static char hexChar(final int n) { - return (n > 9) - ? (char) ('a' + (n - 10)) - : (char) ('0' + n); + private void hexhash(StringBuilder buffer, MessageDigest digest, String text) { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + digest.update(bytes); + byte[] output = digest.digest(); + for (byte b: output) { + int hi = (b & 0xF0) >> 4; + int lo = b & 0x0F; + buffer.append(HEXDIGITS[hi]); + buffer.append(HEXDIGITS[lo]); + } } + private String handleOptions(OptionsCallback callback, String optionsPart) throws MCLException { + if (callback == null || optionsPart == null || optionsPart.isEmpty()) + return ""; + + StringBuilder buffer = new StringBuilder(); + callback.setBuffer(buffer); + for (String optlevel: optionsPart.split(",")) { + int eqindex = optlevel.indexOf('='); + if (eqindex < 0) + throw new MCLException("Invalid options part in server challenge: " + optionsPart); + String lang = optlevel.substring(0, eqindex); + int level; + try { + level = Integer.parseInt(optlevel.substring(eqindex + 1)); + } catch (NumberFormatException e) { + throw new MCLException("Invalid option level in server challenge: " + optlevel); + } + callback.addOptions(lang, level); + } + + return buffer.toString(); + } + /** * Returns an InputStream that reads from this open connection on * the MapiSocket. @@ -716,7 +638,7 @@ public void debug(final String filename) throws IOException { */ public void debug(final Writer out) { log = out; - debug = true; + setDebug(true); } /** @@ -747,15 +669,6 @@ private final void log(final String type, final String message, final boolean fl log.flush(); } - /** - * Set the HandshakeOptions - * - * @param handshakeOptions the options array - */ - public void setHandshakeOptions(HandshakeOption[] handshakeOptions) { - this.handshakeOptions = handshakeOptions; - } - /** * For internal use * @@ -766,6 +679,10 @@ public boolean setInsertFakePrompts(boolean b) { return fromMonet.setInsertFakePrompts(b); } + public boolean isDebug() { + return target.isDebug(); + } + /** * Inner class that is used to write data on a normal stream as a @@ -805,7 +722,7 @@ public void flush() throws IOException { // it's a bit nasty if an exception is thrown from the log, // but ignoring it can be nasty as well, so it is decided to // let it go so there is feedback about something going wrong - if (debug) { + if (isDebug()) { log.flush(); } } @@ -839,7 +756,7 @@ public void writeBlock(final boolean last) throws IOException { // write the actual block out.write(block, 0, writePos); - if (debug) { + if (isDebug()) { if (last) { log("TD ", "write final block: " + writePos + " bytes", false); } else { @@ -964,7 +881,7 @@ private boolean _read(final byte[] b, int len) throws IOException { // if we have read something before, we should have been // able to read the whole, so make this fatal if (off > 0) { - if (debug) { + if (isDebug()) { log("RD ", "the following incomplete block was received:", false); log("RX ", new String(b, 0, off, StandardCharsets.UTF_8), true); } @@ -972,7 +889,7 @@ private boolean _read(final byte[] b, int len) throws IOException { con.getInetAddress().getHostName() + ":" + con.getPort() + ": Incomplete block read from stream"); } - if (debug) + if (isDebug()) log("RD ", "server closed the connection (EOF)", true); return false; } @@ -1023,7 +940,7 @@ private int readBlock() throws IOException { readPos = 0; - if (debug) { + if (isDebug()) { if (wasEndBlock) { log("RD ", "read final block: " + blockLen + " bytes", false); } else { @@ -1039,7 +956,7 @@ private int readBlock() throws IOException { if (!_read(block, blockLen)) return -1; - if (debug) + if (isDebug()) log("RX ", new String(block, 0, blockLen, StandardCharsets.UTF_8), true); // if this is the last block, make it end with a newline and prompt @@ -1054,7 +971,7 @@ private int readBlock() throws IOException { block[blockLen++] = b; } block[blockLen++] = '\n'; - if (debug) { + if (isDebug()) { log("RD ", "inserting prompt", true); } } @@ -1070,7 +987,7 @@ public int read() throws IOException { return -1; } - if (debug) + if (isDebug()) log("RX ", new String(block, readPos, 1, StandardCharsets.UTF_8), true); return (int)block[readPos++]; @@ -1209,7 +1126,7 @@ public synchronized void close() { con = null; } catch (IOException e) { /* ignore it */ } } - if (debug && log != null && log instanceof FileWriter) { + if (isDebug() && log != null && log instanceof FileWriter) { try { log.close(); log = null; @@ -1520,4 +1437,22 @@ public int read(final byte[] dest, int off, int len) throws IOException { return off - origOff; } } + + public static abstract class OptionsCallback { + private StringBuilder buffer; + + protected void contribute(String field, int value) { + if (buffer.length() > 0) + buffer.append(','); + buffer.append(field); + buffer.append('='); + buffer.append(value); + } + + public abstract void addOptions(String lang, int level); + + void setBuffer(StringBuilder buf) { + buffer = buf; + } + } } diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java index 0117134..036fae1 100644 --- a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java +++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java @@ -27,31 +27,24 @@ public MonetUrlParser(Target target, String url) throws URISyntaxException { } public static void parse(Target target, String url) throws URISyntaxException, ValidationError { - boolean modern = true; - if (url.startsWith("mapi:")) { - modern = false; - url = url.substring(5); - if (url.equals("monetdb://")) { - // deal with peculiarity of Java's URI parser - url = "monetdb:///"; - } + if (url.equals("monetdb://")) { + // deal with peculiarity of Java's URI parser + url = "monetdb:///"; } target.barrier(); - try { - MonetUrlParser parser = new MonetUrlParser(target, url); - if (modern) { - parser.parseModern(); - } else { + if (url.startsWith("mapi:")) { + try { + MonetUrlParser parser = new MonetUrlParser(target, url.substring(5)); parser.parseClassic(); + } catch (URISyntaxException e) { + URISyntaxException exc = new URISyntaxException(e.getInput(), e.getReason(), -1); + exc.setStackTrace(e.getStackTrace()); + throw exc; } - } catch (URISyntaxException e) { - int idx = e.getIndex(); - if (idx >= 0 && !modern) { - // "mapi:" - idx += 5; - } - throw new URISyntaxException(e.getInput(), e.getReason(), idx); + } else { + MonetUrlParser parser = new MonetUrlParser(target, url); + parser.parseModern(); } target.barrier(); } @@ -89,7 +82,6 @@ private void parseModern() throws URISyntaxException, ValidationError { String host; String remainder; int pos; - String raw = url.getRawSchemeSpecificPart(); if (authority == null) { if (!url.getRawSchemeSpecificPart().startsWith("//")) { throw new URISyntaxException(urlText, "expected //"); @@ -173,22 +165,48 @@ private void parseModern() throws URISyntaxException, ValidationError { } private void parseClassic() throws URISyntaxException, ValidationError { + if (!url.getRawSchemeSpecificPart().startsWith("//")) { + throw new URISyntaxException(urlText, "expected //"); + } + String scheme = url.getScheme(); - if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); + if (scheme == null) + scheme = ""; switch (scheme) { case "monetdb": - clearBasic(); + parseClassicAuthorityAndPath(); break; case "merovingian": - throw new IllegalStateException("mapi:merovingian: not supported yet"); + String authority = url.getRawAuthority(); + // authority must be "proxy" ignore authority and path + boolean valid = urlText.startsWith("merovingian://proxy?") || urlText.equals("merovingian://proxy"); + if (!valid) + throw new URISyntaxException(urlText, "with mapi:merovingian:, only //proxy is supported"); + break; default: throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://"); } - if (!url.getRawSchemeSpecificPart().startsWith("//")) { - throw new URISyntaxException(urlText, "expected //"); + final String query = url.getRawQuery(); + if (query != null) { + final String args[] = query.split("&"); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("language=")) { + String language = arg.substring(9); + target.setString(Parameter.LANGUAGE, language); + } else if (arg.startsWith("database=")) { + String database = arg.substring(9); + target.setString(Parameter.DATABASE, database); + } else { + // ignore + } + } } + } + private void parseClassicAuthorityAndPath() throws URISyntaxException, ValidationError { + clearBasic(); String authority = url.getRawAuthority(); String host; String portStr; @@ -220,15 +238,12 @@ private void parseClassic() throws URISyntaxException, ValidationError { } String path = url.getRawPath(); - boolean isUnix; if (host.isEmpty() && portStr.isEmpty()) { // socket - isUnix = true; target.clear(Parameter.HOST); target.setString(Parameter.SOCK, path != null ? path : ""); } else { // tcp - isUnix = false; target.clear(Parameter.SOCK); target.setString(Parameter.HOST, host); if (path == null || path.isEmpty()) { @@ -240,29 +255,12 @@ private void parseClassic() throws URISyntaxException, ValidationError { target.setString(Parameter.DATABASE, database); } } - - final String query = url.getRawQuery(); - if (query != null) { - final String args[] = query.split("&"); - for (int i = 0; i < args.length; i++) { - String arg = args[i]; - if (arg.startsWith("language=")) { - String language = arg.substring(9); - target.setString(Parameter.LANGUAGE, language); - } else if (arg.startsWith("database=")) { - String database = arg.substring(9); - target.setString(Parameter.DATABASE, database); - } else { - // ignore - } - } - } } private void clearBasic() { + target.clear(Parameter.TLS); target.clear(Parameter.HOST); target.clear(Parameter.PORT); - target.clear(Parameter.SOCK); target.clear(Parameter.DATABASE); } } diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index 168535b..d2628dd 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -35,6 +35,7 @@ public class Target { private boolean userWasSet = false; private boolean passwordWasSet = false; protected static final Target defaults = new Target(); + private Validated validated = null; private static Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$"); private static Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$"); @@ -171,6 +172,7 @@ public boolean isTls() { public void setTls(boolean tls) { this.tls = tls; + validated = null; } public String getHost() { @@ -179,6 +181,7 @@ public String getHost() { public void setHost(String host) { this.host = host; + validated = null; } public int getPort() { @@ -187,6 +190,7 @@ public int getPort() { public void setPort(int port) { this.port = port; + validated = null; } public String getDatabase() { @@ -195,6 +199,7 @@ public String getDatabase() { public void setDatabase(String database) { this.database = database; + validated = null; } public String getTableschema() { @@ -203,6 +208,7 @@ public String getTableschema() { public void setTableschema(String tableschema) { this.tableschema = tableschema; + validated = null; } public String getTable() { @@ -211,6 +217,7 @@ public String getTable() { public void setTable(String table) { this.table = table; + validated = null; } public String getSock() { @@ -219,6 +226,7 @@ public String getSock() { public void setSock(String sock) { this.sock = sock; + validated = null; } public String getSockdir() { @@ -227,6 +235,7 @@ public String getSockdir() { public void setSockdir(String sockdir) { this.sockdir = sockdir; + validated = null; } public String getCert() { @@ -235,6 +244,7 @@ public String getCert() { public void setCert(String cert) { this.cert = cert; + validated = null; } public String getCerthash() { @@ -243,6 +253,7 @@ public String getCerthash() { public void setCerthash(String certhash) { this.certhash = certhash; + validated = null; } public String getClientkey() { @@ -251,6 +262,7 @@ public String getClientkey() { public void setClientkey(String clientkey) { this.clientkey = clientkey; + validated = null; } public String getClientcert() { @@ -259,6 +271,7 @@ public String getClientcert() { public void setClientcert(String clientcert) { this.clientcert = clientcert; + validated = null; } public String getUser() { @@ -268,6 +281,7 @@ public String getUser() { public void setUser(String user) { this.user = user; this.userWasSet = true; + validated = null; } public String getPassword() { @@ -277,6 +291,7 @@ public String getPassword() { public void setPassword(String password) { this.password = password; this.passwordWasSet = true; + validated = null; } public String getLanguage() { @@ -285,6 +300,7 @@ public String getLanguage() { public void setLanguage(String language) { this.language = language; + validated = null; } public boolean isAutocommit() { @@ -293,6 +309,7 @@ public boolean isAutocommit() { public void setAutocommit(boolean autocommit) { this.autocommit = autocommit; + validated = null; } public String getSchema() { @@ -301,6 +318,7 @@ public String getSchema() { public void setSchema(String schema) { this.schema = schema; + validated = null; } public int getTimezone() { @@ -309,6 +327,7 @@ public int getTimezone() { public void setTimezone(int timezone) { this.timezone = timezone; + validated = null; } public String getBinary() { @@ -317,6 +336,7 @@ public String getBinary() { public void setBinary(String binary) { this.binary = binary; + validated = null; } public int getReplysize() { @@ -325,6 +345,7 @@ public int getReplysize() { public void setReplysize(int replysize) { this.replysize = replysize; + validated = null; } public String getHash() { @@ -333,6 +354,7 @@ public String getHash() { public void setHash(String hash) { this.hash = hash; + validated = null; } public boolean isDebug() { @@ -341,6 +363,7 @@ public boolean isDebug() { public void setDebug(boolean debug) { this.debug = debug; + validated = null; } public String getLogfile() { @@ -349,6 +372,7 @@ public String getLogfile() { public void setLogfile(String logfile) { this.logfile = logfile; + validated = null; } public int getSoTimeout() { @@ -358,10 +382,12 @@ public int getSoTimeout() { public void setSoTimeout(int soTimeout) { this.soTimeout = soTimeout; + validated = null; } public void setTreatClobAsVarchar(boolean treatClobAsVarchar) { this.treatClobAsVarchar = treatClobAsVarchar; + validated = null; } public boolean isTreatClobAsVarchar() { @@ -374,10 +400,23 @@ public boolean isTreatBlobAsBinary() { public void setTreatBlobAsBinary(boolean treatBlobAsBinary) { this.treatBlobAsBinary = treatBlobAsBinary; + validated = null; } public Validated validate() throws ValidationError { - return new Validated(); + if (validated == null) + validated = new Validated(); + return validated; + } + + public String buildUrl() { + final StringBuilder sb = new StringBuilder(128); + sb.append("jdbc:monetdb://").append(host) + .append(':').append(port) + .append('/').append(database); + if (!language.equals("sql")) + sb.append("?language=").append(language); + return sb.toString(); } public class Validated { @@ -457,6 +496,10 @@ public class Validated { // 9. If **clientcert** is set, **clientkey** must also be set. if (!clientcert.isEmpty() && clientkey.isEmpty()) throw new ValidationError("clientcert= is only valid in combination with clientkey="); + + // JDBC specific + if (soTimeout < 0) + throw new ValidationError("so_timeout= must not be negative"); } public boolean getTls() { diff --git a/tests/tests.md b/tests/tests.md index 73148d2..42ee9b4 100644 --- a/tests/tests.md +++ b/tests/tests.md @@ -1457,6 +1457,40 @@ EXPECT language=sql ACCEPT mapi:monetdb://localhost:12345/db?_l%61nguage=mal ``` + +## Merovingian URLs + +These occur in redirects. +Leave host, port and database are not cleared the way the are +for `monetdb:`, `monetdbs:` and `mapi:monetdb:` URLs. + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy +PARSE mapi:merovingian://proxy +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=dummy +PARSE mapi:merovingian://proxy? +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=dummy +PARSE mapi:merovingian://proxy?database=yeah&unknown=unknown +EXPECT host=banana +EXPECT port=123 +EXPECT tls=on +EXPECT sock=/tmp/sock +EXPECT database=yeah +``` + + # lalala Java ```test From 3ebc5b691e87a63f016cfeb90d15d50484aad02f Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Wed, 6 Dec 2023 16:50:37 +0100 Subject: [PATCH 07/41] Suppress jdbcclient warnings about unknown .monetdb settings --- src/main/java/org/monetdb/client/JdbcClient.java | 15 ++++++++++----- src/main/java/org/monetdb/util/CmdLineOpts.java | 13 ++++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/monetdb/client/JdbcClient.java b/src/main/java/org/monetdb/client/JdbcClient.java index d73077e..5870461 100644 --- a/src/main/java/org/monetdb/client/JdbcClient.java +++ b/src/main/java/org/monetdb/client/JdbcClient.java @@ -190,6 +190,13 @@ public final static void main(String[] args) throws Exception { "statements read. Batching can greatly speedup the " + "process of restoring a database dump."); +// This file can contain defaults for the flags user, password, language, +// database, save_history, format, host, port, and width. For example, an + + copts.addIgnored("save_history"); + copts.addIgnored("format"); + copts.addIgnored("width"); + // we store user and password in separate variables in order to // be able to properly act on them like forgetting the password // from the user's file if the user supplies a username on the @@ -325,11 +332,9 @@ public final static void main(String[] args) throws Exception { // make sure the driver class is loaded (and thus register itself with the DriverManager) Class.forName("org.monetdb.jdbc.MonetDriver"); - con = DriverManager.getConnection( - "jdbc:monetdb://" + host + "/" + database + attr, - user, - pass - ); + String url = "jdbc:monetdb://" + host + "/" + database + attr; + System.err.println(url); + con = DriverManager.getConnection(url, user, pass); SQLWarning warn = con.getWarnings(); while (warn != null) { System.err.println("Connection warning: " + warn.getMessage()); diff --git a/src/main/java/org/monetdb/util/CmdLineOpts.java b/src/main/java/org/monetdb/util/CmdLineOpts.java index 5318f52..3216223 100644 --- a/src/main/java/org/monetdb/util/CmdLineOpts.java +++ b/src/main/java/org/monetdb/util/CmdLineOpts.java @@ -10,11 +10,13 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Properties; public final class CmdLineOpts { /** the arguments we handle */ private HashMap opts = new HashMap(); + private final HashSet ignoredInFile = new HashSet<>(); /** the options themself */ private ArrayList options = new ArrayList(); @@ -53,6 +55,10 @@ public void addOption( opts.put(longa, oc); } + public void addIgnored(String name) { + ignoredInFile.add(name); + } + public void removeOption(final String name) { final OptionContainer oc = opts.get(name); if (oc != null) { @@ -88,10 +94,11 @@ public void processFile(final java.io.File file) throws OptionsException { if (option != null) { option.resetArguments(); option.addArgument(prop.getProperty(key)); - } else + } else if (!ignoredInFile.contains(key)) { // ignore unknown options (it used to throw an OptionsException) - System.out.println("Info: Ignoring unknown/unsupported option (in " + file.getAbsolutePath() + "): " + key); - } + System.out.println("Info: Ignoring unknown/unsupported option (in " + file.getAbsolutePath() + "): " + key); + } + } } catch (java.io.IOException e) { throw new OptionsException("File IO Exception: " + e); } From 9929dfe18fd6a17d52586bd117769d26231640a3 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 08:54:34 +0100 Subject: [PATCH 08/41] MonetDriver: Use only the properties if URL is exactly "jdbc:monetdb:" Normally, parsing the URL clears host, port, etc. That means that an application that already has everything as Properties has to extract those from the Properties and construct a URL, which is inconvenient and error-prone. --- src/main/java/org/monetdb/jdbc/MonetDriver.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/monetdb/jdbc/MonetDriver.java b/src/main/java/org/monetdb/jdbc/MonetDriver.java index 0d91466..0ad1551 100644 --- a/src/main/java/org/monetdb/jdbc/MonetDriver.java +++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java @@ -70,7 +70,11 @@ public final class MonetDriver implements Driver { */ @Override public boolean acceptsURL(final String url) { - return url != null && url.startsWith("jdbc:monetdb://"); + if (url == null) + return false; + if (url.startsWith("jdbc:monetdb:") || url.startsWith("jdbc:monetdbs:")) + return true; + return false; } /** @@ -105,6 +109,7 @@ public Connection connect(final String url, Properties info) Target target = new Target(); try { + // If properties are given, add those first if (info != null) { for (String key : info.stringPropertyNames()) { String value = info.getProperty(key); @@ -113,7 +118,13 @@ public Connection connect(final String url, Properties info) target.setString(key, value); } } - MonetUrlParser.parse(target, url.substring(5)); + + // If url is exactly "jdbc:monetdb:", use just the properties. + // This is different from, say, jdbc:monetdb://, because the + // latter will clear preexisting host, port, TLS and database settings. + // Useful in combination with Target.toProperties(). + if (!url.equals("jdbc:monetdb:")) + MonetUrlParser.parse(target, url.substring(5)); } catch (ValidationError | URISyntaxException e) { throw new SQLException(e.getMessage()); } From 443167da4e609aff0dcf618ec15640011ff1c995 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 09:00:00 +0100 Subject: [PATCH 09/41] Allow to pass whole URL to jdbcclient not just separate parameters Pass all information using Properties rather than building a URL. If the 'database' parameter starts with jdbc:, use that as the URL. Otherwise, use a stub --- .../java/org/monetdb/client/JdbcClient.java | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/monetdb/client/JdbcClient.java b/src/main/java/org/monetdb/client/JdbcClient.java index 5870461..8722577 100644 --- a/src/main/java/org/monetdb/client/JdbcClient.java +++ b/src/main/java/org/monetdb/client/JdbcClient.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Properties; /** * This program acts like an extended client program for MonetDB. Its @@ -123,6 +124,7 @@ public final class JdbcClient { * @throws Exception if uncaught exception is thrown */ public final static void main(String[] args) throws Exception { + final Properties props = new Properties(); final CmdLineOpts copts = new CmdLineOpts(); // arguments which take exactly one argument @@ -290,19 +292,24 @@ public final static void main(String[] args) throws Exception { user = copts.getOption("user").getArgument(); - // build the hostname + // extract hostname and port String host = copts.getOption("host").getArgument(); - if (host.indexOf(':') == -1) { - host = host + ":" + copts.getOption("port").getArgument(); + String port = copts.getOption("port").getArgument(); + int hostColon = host.indexOf(':'); + if (hostColon > 0) { + port = host.substring(hostColon + 1); + host = host.substring(0, hostColon); } + props.setProperty("host", host); + props.setProperty("port", port); - // build the extra arguments of the JDBC connect string // increase the fetchsize from the default 250 to 10000 - String attr = "?fetchsize=10000&"; + props.setProperty("fetchsize", "10000"); + CmdLineOpts.OptionContainer oc = copts.getOption("language"); final String lang = oc.getArgument(); if (oc.isPresent()) - attr += "language=" + lang + "&"; + props.setProperty("language", lang); /* Xquery is no longer functional or supported // set some behaviour based on the language XQuery @@ -314,13 +321,13 @@ public final static void main(String[] args) throws Exception { */ oc = copts.getOption("Xdebug"); if (oc.isPresent()) { - attr += "debug=true&"; + props.setProperty("debug", "true"); if (oc.getArgumentCount() == 1) - attr += "logfile=" + oc.getArgument() + "&"; + props.setProperty("logfile", "logfile=" + oc.getArgument()); } oc = copts.getOption("Xhash"); if (oc.isPresent()) - attr += "hash=" + oc.getArgument() + "&"; + props.setProperty("hash", oc.getArgument()); // request a connection suitable for MonetDB from the driver // manager note that the database specifier is only used when @@ -332,9 +339,18 @@ public final static void main(String[] args) throws Exception { // make sure the driver class is loaded (and thus register itself with the DriverManager) Class.forName("org.monetdb.jdbc.MonetDriver"); - String url = "jdbc:monetdb://" + host + "/" + database + attr; - System.err.println(url); - con = DriverManager.getConnection(url, user, pass); + // If the database name is a full url, use that. + // Otherwise, construct something. + String url; + if (database.startsWith("jdbc:")) { + url = database; + } else { + url = "jdbc:monetdb:"; // special case + props.setProperty("database", database); + } + props.setProperty("user", user); + props.setProperty("password", pass); + con = DriverManager.getConnection(url, props); SQLWarning warn = con.getWarnings(); while (warn != null) { System.err.println("Connection warning: " + warn.getMessage()); From 6631ddfca39fd2e9666c7a9bf7e869d1afe5f35a Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 10:25:16 +0100 Subject: [PATCH 10/41] Arrange testing --- tests/JDBC_API_Tester.java | 3 + tests/UrlTester.java | 210 +++++++++++++++++++++++-------------- tests/build.xml | 6 ++ tests/javaspecific.md | 31 ++++++ tests/tests.md | 46 +++----- 5 files changed, 187 insertions(+), 109 deletions(-) create mode 100644 tests/javaspecific.md diff --git a/tests/JDBC_API_Tester.java b/tests/JDBC_API_Tester.java index 04eb53c..75041dd 100644 --- a/tests/JDBC_API_Tester.java +++ b/tests/JDBC_API_Tester.java @@ -48,6 +48,9 @@ final public class JDBC_API_Tester { public static void main(String[] args) throws Exception { String con_URL = args[0]; + // Test this before trying to connect + UrlTester.runAllTests(); + JDBC_API_Tester jt = new JDBC_API_Tester(); jt.sb = new StringBuilder(sbInitLen); jt.con = DriverManager.getConnection(con_URL); diff --git a/tests/UrlTester.java b/tests/UrlTester.java index 00093ba..3be4f4f 100644 --- a/tests/UrlTester.java +++ b/tests/UrlTester.java @@ -2,40 +2,107 @@ import java.io.*; import java.net.URISyntaxException; +import java.util.ArrayList; public class UrlTester { - String filename = null; - int verbose = 0; - BufferedReader reader = null; + final String filename; + final int verbose; + final BufferedReader reader; int lineno = 0; int testCount = 0; Target target = null; Target.Validated validated = null; - public UrlTester() { + public UrlTester(String filename, BufferedReader reader, int verbose) { + this.filename = filename; + this.verbose = verbose; + this.reader = reader; } - public UrlTester(String filename) { + public UrlTester(String filename, int verbose) throws IOException { this.filename = filename; + this.verbose = verbose; + this.reader = new BufferedReader(new FileReader(filename)); + } + + public static void main(String[] args) throws IOException { + ArrayList filenames = new ArrayList<>(); + int verbose = 0; + for (String arg : args) { + switch (arg) { + case "-vvv": + verbose++; + case "-vv": + verbose++; + case "-v": + verbose++; + break; + case "-h": + case "--help": + exitUsage(null); + break; + default: + if (!arg.startsWith("-")) { + filenames.add(arg); + } else { + exitUsage("Unexpected argument: " + arg); + } + break; + } + } + + runUnitTests(); + + try { + if (filenames.isEmpty()) { + runAllTests(); + } else { + for (String filename : filenames) { + new UrlTester(filename, verbose).run(); + } + } + } catch (Failure e) { + System.err.println("Test failed: " + e.getMessage()); + System.exit(1); + } + } + + private static void exitUsage(String message) { + if (message != null) { + System.err.println(message); + } + System.err.println("Usage: UrlTester OPTIONS [FILENAME..]"); + System.err.println("Options:"); + System.err.println(" -v Be more verbose"); + System.err.println(" -h --help Show this help"); + int status = message == null ? 0 : 1; + System.exit(status); } - public static void main(String[] args) throws Exception { - checkDefaults(); - checkParameters(); + public static UrlTester forResource(String resourceName, int verbose) throws FileNotFoundException { + InputStream stream = UrlTester.class.getResourceAsStream(resourceName); + if (stream == null) { + throw new FileNotFoundException("Resource " + resourceName); + } + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + return new UrlTester(resourceName, reader, verbose); + } - int exitcode; - UrlTester tester = new UrlTester(); - exitcode = tester.parseArgs(args); - if (exitcode == 0) - exitcode = tester.run(); - System.exit(exitcode); + public static void runAllTests() throws IOException, Failure { + runUnitTests(); + UrlTester.forResource("/tests.md", 0).run(); + UrlTester.forResource("/javaspecific.md", 0).run(); } + public static void runUnitTests() { + testDefaults(); + testParameters(); + } - private static void checkDefaults() { + private static void testDefaults() { Target target = new Target(); - for (Parameter parm: Parameter.values()) { + for (Parameter parm : Parameter.values()) { Object expected = parm.getDefault(); if (expected == null) continue; @@ -46,8 +113,8 @@ private static void checkDefaults() { } } - private static void checkParameters() { - for (Parameter parm: Parameter.values()) { + private static void testParameters() { + for (Parameter parm : Parameter.values()) { Parameter found = Parameter.forName(parm.name); if (parm != found) { String foundStr = found != null ? found.name : "null"; @@ -56,66 +123,16 @@ private static void checkParameters() { } } - private int parseArgs(String[] args) { - for (int i = 0; i < args.length; i++) { - String arg = args[i]; - if (arg.startsWith("-")) { - int result = handleFlags(arg); - if (result != 0) - return result; - } else if (filename == null) { - filename = arg; - } else { - System.err.println("Unexpected argument: " + arg); - return 1; - } - } - return 0; - } - - private int run() throws IOException { - if (filename != null) { - reader = new BufferedReader(new FileReader(filename)); - } else { - String resourceName = "/tests.md"; - InputStream stream = this.getClass().getResourceAsStream(resourceName); - if (stream == null) { - System.err.println("Resource " + resourceName + " not found"); - return 1; - } - reader = new BufferedReader(new InputStreamReader(stream)); - filename = "tests/tests.md"; - } - + public void run() throws Failure, IOException { try { processFile(); } catch (Failure e) { - System.err.println("" + filename + ":" + lineno + ": " + e.getMessage()); - return 1; - } - return 0; - } - - private int handleFlags(String arg) { - if (!arg.startsWith("-") || arg.equals("-")) { - System.err.println("Invalid flag: " + arg); - } - String a = arg.substring(1); - - while (!a.isEmpty()) { - char letter = a.charAt(0); - a = a.substring(1); - switch (letter) { - case 'v': - verbose++; - break; - default: - System.err.println("Unexpected flag " + letter + " in " + arg); - return -1; + if (e.getFilename() == null) { + e.setFilename(filename); + e.setLineno(lineno); + throw e; } } - - return 0; } private void processFile() throws IOException, Failure { @@ -198,7 +215,7 @@ private void handleCommand(String line) throws Failure { private void handleOnly(boolean mustBePresent, String rest) throws Failure { boolean found = false; - for (String part: rest.split("\\s+")) { + for (String part : rest.split("\\s+")) { if (part.equals("jdbc")) { found = true; break; @@ -329,10 +346,14 @@ private Object extract(String key) throws ValidationError, Failure { return tryValidate().connectTcp(); case "connect_tls_verify": switch (tryValidate().connectVerify()) { - case None: return ""; - case Cert: return "cert"; - case Hash: return "hash"; - case System: return "system"; + case None: + return ""; + case Cert: + return "cert"; + case Hash: + return "hash"; + case System: + return "system"; default: throw new IllegalStateException("unreachable"); } @@ -354,9 +375,40 @@ private Object extract(String key) throws ValidationError, Failure { } } - private class Failure extends Exception { + public static class Failure extends Exception { + private String filename = null; + private int lineno = -1; + public Failure(String message) { super(message); } + + @Override + public String getMessage() { + StringBuilder buffer = new StringBuilder(); + if (filename != null) { + buffer.append(filename).append(":"); + if (lineno >= 0) + buffer.append(lineno).append(":"); + } + buffer.append(super.getMessage()); + return buffer.toString(); + } + + public String getFilename() { + return filename; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public int getLineno() { + return lineno; + } + + public void setLineno(int lineno) { + this.lineno = lineno; + } } } diff --git a/tests/build.xml b/tests/build.xml index 68bd4bb..798eccb 100644 --- a/tests/build.xml +++ b/tests/build.xml @@ -72,11 +72,17 @@ Copyright 1997 - July 2008 CWI, August 2008 - 2023 MonetDB B.V. + + + + + + diff --git a/tests/javaspecific.md b/tests/javaspecific.md new file mode 100644 index 0000000..f1739fc --- /dev/null +++ b/tests/javaspecific.md @@ -0,0 +1,31 @@ + +# Java-specific tests + +Test settings that are only in monetdb-java. + +```test +ONLY jdbc +EXPECT so_timeout=0 +SET so_timeout=42 +EXPECT so_timeout=42 +ACCEPT monetdb://?so_timeout=99 +EXPECT so_timeout=99 +``` + +```test +ONLY jdbc +EXPECT treat_clob_as_varchar=true +SET treat_clob_as_varchar=off +EXPECT treat_clob_as_varchar=false +ACCEPT monetdb://?treat_clob_as_varchar=yes +EXPECT treat_clob_as_varchar=on +``` + +```test +ONLY jdbc +EXPECT treat_blob_as_binary=true +SET treat_blob_as_binary=off +EXPECT treat_blob_as_binary=false +ACCEPT monetdb://?treat_blob_as_binary=yes +EXPECT treat_blob_as_binary=on +``` diff --git a/tests/tests.md b/tests/tests.md index 42ee9b4..d0d13f3 100644 --- a/tests/tests.md +++ b/tests/tests.md @@ -1476,12 +1476,28 @@ EXPECT port=123 EXPECT tls=on EXPECT sock=/tmp/sock EXPECT database=dummy +``` + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy PARSE mapi:merovingian://proxy? EXPECT host=banana EXPECT port=123 EXPECT tls=on EXPECT sock=/tmp/sock EXPECT database=dummy +``` + +```test +SET host=banana +SET port=123 +SET tls=on +SET sock=/tmp/sock +SET database=dummy PARSE mapi:merovingian://proxy?database=yeah&unknown=unknown EXPECT host=banana EXPECT port=123 @@ -1489,33 +1505,3 @@ EXPECT tls=on EXPECT sock=/tmp/sock EXPECT database=yeah ``` - - -# lalala Java - -```test -ONLY jdbc -EXPECT so_timeout=0 -SET so_timeout=42 -EXPECT so_timeout=42 -ACCEPT monetdb://?so_timeout=99 -EXPECT so_timeout=99 -``` - -```test -ONLY jdbc -EXPECT treat_clob_as_varchar=true -SET treat_clob_as_varchar=off -EXPECT treat_clob_as_varchar=false -ACCEPT monetdb://?treat_clob_as_varchar=yes -EXPECT treat_clob_as_varchar=on -``` - -```test -ONLY jdbc -EXPECT treat_blob_as_binary=true -SET treat_blob_as_binary=off -EXPECT treat_blob_as_binary=false -ACCEPT monetdb://?treat_blob_as_binary=yes -EXPECT treat_blob_as_binary=on -``` From b2f06dbccd68972d2b10b19a08980ec47bdc707e Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 10:44:38 +0100 Subject: [PATCH 11/41] replysize -> replySize --- .../org/monetdb/jdbc/MonetConnection.java | 4 ++-- src/main/java/org/monetdb/mcl/net/Target.java | 24 +++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java index 6e3b916..9772188 100644 --- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -3767,8 +3767,8 @@ public void addOptions(String lang, int level) { // Try to add options and record that this happened if it succeeds. if (contribute(SqlOption.Autocommit, target.isAutocommit() ? 1 : 0)) autoCommit = target.isAutocommit(); - if (contribute(SqlOption.ReplySize, target.getReplysize())) - defaultFetchSize = target.getReplysize(); + if (contribute(SqlOption.ReplySize, target.getReplySize())) + defaultFetchSize = target.getReplySize(); if (contribute(SqlOption.SizeHeader, 1)) sizeHeaderEnabled = true; if (contribute(SqlOption.TimeZone, target.getTimezone())) diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index d2628dd..f08f7c1 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -1,7 +1,5 @@ package org.monetdb.mcl.net; -import java.util.Calendar; -import java.util.Properties; import java.util.regex.Pattern; public class Target { @@ -24,7 +22,7 @@ public class Target { private String schema = ""; private int timezone; private String binary = "on"; - private int replysize = 200; + private int replySize = 200; private String hash = ""; private boolean debug = false; private String logfile = ""; @@ -101,8 +99,8 @@ private void assign(Parameter parm, Object value) { case SCHEMA: setSchema((String)value); break; case TIMEZONE: setTimezone((int)value); break; case BINARY: setBinary((String)value); break; - case REPLYSIZE: setReplysize((int)value); break; - case FETCHSIZE: setReplysize((int)value); break; + case REPLYSIZE: setReplySize((int)value); break; + case FETCHSIZE: setReplySize((int)value); break; case HASH: setHash((String)value); break; case DEBUG: setDebug((boolean)value); break; case LOGFILE: setLogfile((String)value); break; @@ -142,8 +140,8 @@ public Object getObject(Parameter parm) { case SCHEMA: return schema; case TIMEZONE: return timezone; case BINARY: return binary; - case REPLYSIZE: return replysize; - case FETCHSIZE: return replysize; + case REPLYSIZE: return replySize; + case FETCHSIZE: return replySize; case HASH: return hash; case DEBUG: return debug; case LOGFILE: return logfile; @@ -339,12 +337,12 @@ public void setBinary(String binary) { validated = null; } - public int getReplysize() { - return replysize; + public int getReplySize() { + return replySize; } - public void setReplysize(int replysize) { - this.replysize = replysize; + public void setReplySize(int replySize) { + this.replySize = replySize; validated = null; } @@ -582,8 +580,8 @@ public int getBinary() { return nbinary; } - public int getReplysize() { - return replysize; + public int getReplySize() { + return replySize; } public String getHash() { From 91a7aeb04d0830b53a02ad944e200472e1a3b045 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 10:53:50 +0100 Subject: [PATCH 12/41] Prepare for implementing TLS --- src/main/java/org/monetdb/mcl/net/MapiSocket.java | 2 +- src/main/java/org/monetdb/mcl/net/SecureSocket.java | 9 +++++++++ src/main/java/org/monetdb/mcl/net/Target.java | 7 +++++++ src/main/java/org/monetdb/mcl/net/Verify.java | 8 -------- 4 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/monetdb/mcl/net/SecureSocket.java delete mode 100644 src/main/java/org/monetdb/mcl/net/Verify.java diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java index a8ca99f..d5a92d7 100644 --- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java +++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java @@ -347,7 +347,7 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx private Socket wrapTLS(Socket sock, Target.Validated validated) throws MCLException { if (validated.getTls()) - throw new MCLException("TLS connections (monetdbs://) are not supported yet"); + return SecureSocket.wrap(validated, sock); return sock; } diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java new file mode 100644 index 0000000..3214422 --- /dev/null +++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java @@ -0,0 +1,9 @@ +package org.monetdb.mcl.net; + +import java.net.Socket; + +public class SecureSocket { + public static Socket wrap(Target.Validated validated, Socket sock) { + throw new MCLException("TLS connections (monetdbs://) are not supported yet"); + } +} diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java index f08f7c1..8b20b2a 100644 --- a/src/main/java/org/monetdb/mcl/net/Target.java +++ b/src/main/java/org/monetdb/mcl/net/Target.java @@ -660,4 +660,11 @@ public String connectClientCert() { return clientcert.isEmpty() ? clientkey : clientcert; } } + + public enum Verify { + None, + Cert, + Hash, + System; + } } diff --git a/src/main/java/org/monetdb/mcl/net/Verify.java b/src/main/java/org/monetdb/mcl/net/Verify.java deleted file mode 100644 index 457c20e..0000000 --- a/src/main/java/org/monetdb/mcl/net/Verify.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.monetdb.mcl.net; - -public enum Verify { - None, - Cert, - Hash, - System; -} From d3814c3386d84d277468de643ead8580dca21af7 Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Thu, 7 Dec 2023 11:21:39 +0100 Subject: [PATCH 13/41] TLS support in its most basic form --- .../java/org/monetdb/mcl/net/MapiSocket.java | 2 +- .../java/org/monetdb/mcl/net/SecureSocket.java | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java index d5a92d7..7360751 100644 --- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java +++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java @@ -345,7 +345,7 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx con = sock; } - private Socket wrapTLS(Socket sock, Target.Validated validated) throws MCLException { + private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException { if (validated.getTls()) return SecureSocket.wrap(validated, sock); return sock; diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java index 3214422..1cdf54c 100644 --- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java +++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java @@ -1,9 +1,22 @@ package org.monetdb.mcl.net; +import org.monetdb.mcl.MCLException; + +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; import java.net.Socket; public class SecureSocket { - public static Socket wrap(Target.Validated validated, Socket sock) { - throw new MCLException("TLS connections (monetdbs://) are not supported yet"); + public static Socket wrap(Target.Validated validated, Socket inner) throws IOException { + SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + String host = validated.connectTcp(); + int port = validated.connectPort(); + boolean autoclose = true; + SSLSocket sock = (SSLSocket) factory.createSocket(inner, host, port, autoclose); + sock.setUseClientMode(true); + + sock.startHandshake(); + return sock; } } From de54ef5ee237f58a5fce94a2a7a9ad31e173e08a Mon Sep 17 00:00:00 2001 From: Joeri van Ruth Date: Fri, 8 Dec 2023 15:17:50 +0100 Subject: [PATCH 14/41] TLS seems to work Need to add tests though --- .../org/monetdb/jdbc/MonetConnection.java | 6 +- .../java/org/monetdb/mcl/net/MapiSocket.java | 9 +- .../org/monetdb/mcl/net/SecureSocket.java | 168 +++++++++++++++++- 3 files changed, 169 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java index 9772188..52fd1cf 100644 --- a/src/main/java/org/monetdb/jdbc/MonetConnection.java +++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java @@ -41,6 +41,8 @@ import org.monetdb.mcl.parser.MCLParseException; import org.monetdb.mcl.parser.StartOfHeaderParser; +import javax.net.ssl.SSLException; + /** *
  * A {@link Connection} suitable for the MonetDB database.
@@ -215,8 +217,10 @@ public class MonetConnection
 			final String error = in.discardRemainder();
 			if (error != null)
 				throw new SQLNonTransientConnectionException((error.length() > 6) ? error.substring(6) : error, "08001");
+		} catch (SSLException e) {
+			throw new SQLNonTransientConnectionException("Cannot establish secure connection: " + e.getMessage(), e);
 		} catch (IOException e) {
-			throw new SQLNonTransientConnectionException("Cannot connect: " + e.getMessage(), "08006");
+			throw new SQLNonTransientConnectionException("Cannot connect: " + e.getMessage(), "08006", e);
 		} catch (MCLParseException e) {
 			throw new SQLNonTransientConnectionException(e.getMessage(), "08001");
 		} catch (org.monetdb.mcl.MCLException e) {
diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 7360751..60e090a 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -346,10 +346,11 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx
 	}
 
 	private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException {
-		if (validated.getTls())
-			return SecureSocket.wrap(validated, sock);
-		return sock;
-	}
+        if (validated.getTls())
+            return SecureSocket.wrap(validated, sock);
+        else
+            return sock;
+    }
 
 	private boolean handshake(Target.Validated validated, OptionsCallback callback, ArrayList warnings) throws IOException, MCLException {
 		String challenge = reader.getLine();
diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index 1cdf54c..d3a2093 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -1,22 +1,172 @@
 package org.monetdb.mcl.net;
 
-import org.monetdb.mcl.MCLException;
-
-import javax.net.ssl.SSLSocket;
-import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.*;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.net.Socket;
+import java.security.*;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
 
 public class SecureSocket {
+    private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
+    final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
+
     public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
-        SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
-        String host = validated.connectTcp();
-        int port = validated.connectPort();
-        boolean autoclose = true;
-        SSLSocket sock = (SSLSocket) factory.createSocket(inner, host, port, autoclose);
+        Target.Verify verify = validated.connectVerify();
+        SSLSocketFactory socketFactory;
+        try {
+            switch (verify) {
+                case System:
+                    socketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
+                    return wrapSocket(inner, validated, socketFactory, true);
+                case Cert:
+                        KeyStore keyStore = keyStoreForCert(validated.getCert());
+                        socketFactory = certBasedSocketFactory(keyStore);
+                    return wrapSocket(inner, validated, socketFactory, true);
+                case Hash:
+                    return wrapHash(validated, inner);
+                default:
+                    throw new RuntimeException("unreachable: unexpected verification strategy " + verify.name());
+            }
+        } catch (CertificateException e) {
+            throw new SSLException(e.getMessage(), e);
+        }
+    }
+
+    private static Socket wrapHash(Target.Validated validated, Socket inner) throws IOException, CertificateException {
+        SSLSocketFactory socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits());
+        SSLSocket sock = wrapSocket(inner, validated, socketFactory, false);
+
+        return sock;
+    }
+
+    private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException {
+        SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true);
+
         sock.setUseClientMode(true);
+        sock.setEnabledProtocols(ENABLED_PROTOCOLS);
 
+        if (checkName) {
+            SSLParameters parameters = sock.getSSLParameters();
+            parameters.setEndpointIdentificationAlgorithm("HTTPS");
+            sock.setSSLParameters(parameters);
+        }
         sock.startHandshake();
         return sock;
     }
+
+    private static X509Certificate loadCertificate(String path) throws CertificateException, IOException {
+        CertificateFactory factory = CertificateFactory.getInstance("X509");
+        try (FileInputStream s = new FileInputStream(path)) {
+            return (X509Certificate) factory.generateCertificate(s);
+        }
+    }
+
+    private static SSLSocketFactory certBasedSocketFactory(KeyStore store) throws IOException, CertificateException {
+        TrustManagerFactory trustManagerFactory;
+        try {
+            trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            trustManagerFactory.init(store);
+        } catch (NoSuchAlgorithmException | KeyStoreException e) {
+            throw new RuntimeException("Could not create TrustManagerFactory", e);
+        }
+
+        SSLContext context;
+        try {
+            context = SSLContext.getInstance("TLS");
+            context.init(null, trustManagerFactory.getTrustManagers(), null);
+        } catch (NoSuchAlgorithmException | KeyManagementException e) {
+            throw new RuntimeException("Could not create SSLContext", e);
+        }
+
+        return context.getSocketFactory();
+    }
+
+    private static KeyStore keyStoreForCert(String path) throws IOException, CertificateException {
+        try {
+            X509Certificate cert = loadCertificate(path);
+            KeyStore store = emptyKeyStore();
+            store.setCertificateEntry("root", cert);
+            return store;
+        } catch (KeyStoreException e) {
+            throw new RuntimeException("Could not create KeyStore for certificate", e);
+        }
+    }
+
+    private static KeyStore emptyKeyStore() throws IOException, CertificateException {
+        KeyStore store;
+        try {
+            store = KeyStore.getInstance("PKCS12");
+            store.load(null, null);
+            return store;
+        } catch (KeyStoreException | NoSuchAlgorithmException e) {
+            throw new RuntimeException("Could not create KeyStore for certificate", e);
+        }
+    }
+
+    private static SSLSocketFactory hashBasedSocketFactory(String hashDigits) {
+        TrustManager trustManager = new HashBasedTrustManager(hashDigits);
+        try {
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(null, new TrustManager[]{ trustManager}, null);
+            return context.getSocketFactory();
+        } catch (NoSuchAlgorithmException | KeyManagementException e) {
+            throw new RuntimeException("Could not create SSLContext", e);
+        }
+
+    }
+
+    private static class HashBasedTrustManager implements X509TrustManager {
+        private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();
+        private final String hashDigits;
+
+        public HashBasedTrustManager(String hashDigits) {
+            this.hashDigits = hashDigits;
+        }
+
+
+        @Override
+        public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+            throw new RuntimeException("this TrustManager is only suitable for client side connections");
+        }
+
+        @Override
+        public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+            X509Certificate cert = x509Certificates[0];
+            byte[] certBytes = cert.getEncoded();
+
+            // for now it's always SHA256.
+            byte[] hashBytes;
+            try {
+                MessageDigest hasher = MessageDigest.getInstance("SHA-256");
+                hasher.update(certBytes);
+                hashBytes = hasher.digest();
+            } catch (NoSuchAlgorithmException e) {
+                throw new RuntimeException("failed to instantiate hash digest");
+            }
+
+            // convert to hex digits
+            StringBuilder buffer = new StringBuilder(2 * hashBytes.length);
+            for (byte b: hashBytes) {
+                int hi = (b & 0xF0) >> 4;
+                int lo = b & 0x0F;
+                buffer.append(HEXDIGITS[hi]);
+                buffer.append(HEXDIGITS[lo]);
+            }
+            String certDigits = buffer.toString();
+
+            if (!certDigits.startsWith(hashDigits)) {
+                throw new CertificateException("Certificate hash does not start with '" + hashDigits + "': " + certDigits);
+            }
+
+
+        }
+
+        @Override
+        public X509Certificate[] getAcceptedIssuers() {
+            return new X509Certificate[0];
+        }
+    }
 }

From ae794e346d3c77283e48d32c129b19e3abe0910f Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Mon, 11 Dec 2023 13:45:12 +0100
Subject: [PATCH 15/41] Add tests using monetdb-tlstester

---
 tests/TLSTester.java | 304 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 304 insertions(+)
 create mode 100644 tests/TLSTester.java

diff --git a/tests/TLSTester.java b/tests/TLSTester.java
new file mode 100644
index 0000000..dffaadc
--- /dev/null
+++ b/tests/TLSTester.java
@@ -0,0 +1,304 @@
+import org.monetdb.mcl.net.Parameter;
+
+import java.io.*;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Properties;
+
+public class TLSTester {
+    int verbose = 0;
+    String serverHost = null;
+    String altHost = null;
+    int serverPort = -1;
+    boolean enableTrusted = false;
+    File tempDir = null;
+    final HashMap fileCache = new HashMap<>();
+
+    public TLSTester(String[] args) {
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            if (arg.equals("-v")) {
+                verbose = 1;
+            } else if (arg.equals("-a")) {
+                altHost = args[++i];
+            } else if (arg.equals("-t")) {
+                enableTrusted = true;
+            } else if (!arg.startsWith("-") && serverHost == null) {
+                int idx = arg.indexOf(':');
+                if (idx > 0) {
+                    serverHost = arg.substring(0, idx);
+                    try {
+                        serverPort = Integer.parseInt(arg.substring(idx + 1));
+                        if (serverPort > 0 && serverPort < 65536)
+                            continue;
+                    } catch (NumberFormatException ignored) {
+                    }
+                }
+                // if we get here it wasn't very valid
+                throw new IllegalArgumentException("Invalid argument: " + arg);
+            } else {
+                throw new IllegalArgumentException("Unexpected argument: " + arg);
+            }
+        }
+    }
+
+    public static void main(String[] args) throws IOException, SQLException, ClassNotFoundException {
+        Class.forName("org.monetdb.jdbc.MonetDriver");
+        TLSTester main = new TLSTester(args);
+        main.run();
+    }
+
+    private HashMap loadPortMap(String testName) throws IOException {
+        HashMap portMap = new HashMap<>();
+        InputStream in = fetchData("/?test=" + testName);
+        BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
+        for (String line = br.readLine(); line != null; line = br.readLine()) {
+            int idx = line.indexOf(':');
+            String service = line.substring(0, idx);
+            int port;
+            try {
+                port = Integer.parseInt(line.substring(idx + 1));
+            } catch (NumberFormatException e) {
+                throw new RuntimeException("Invalid port map line: " + line);
+            }
+            portMap.put(service, port);
+        }
+        return portMap;
+    }
+
+    private File resource(String resource) throws IOException {
+        if (!fileCache.containsKey(resource))
+            fetchResource(resource);
+        return fileCache.get(resource);
+    }
+
+    private void fetchResource(String resource) throws IOException {
+        if (!resource.startsWith("/")) {
+            throw new IllegalArgumentException("Resource must start with slash: " + resource);
+        }
+        if (tempDir == null) {
+            tempDir = Files.createTempDirectory("tlstest").toFile();
+            tempDir.deleteOnExit();
+        }
+        File outPath = new File(tempDir, resource.substring(1));
+        try (InputStream in = fetchData(resource); FileOutputStream out = new FileOutputStream(outPath)) {
+            byte[] buffer = new byte[12];
+            while (true) {
+                int n = in.read(buffer);
+                if (n <= 0)
+                    break;
+                out.write(buffer, 0, n);
+            }
+        }
+        fileCache.put(resource, outPath);
+    }
+
+    private byte[] fetchBytes(String resource) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        try (InputStream in = fetchData(resource)) {
+            byte[] buffer = new byte[22];
+            while (true) {
+                int nread = in.read(buffer);
+                if (nread <= 0)
+                    break;
+                out.write(buffer, 0, nread);
+            }
+            return out.toByteArray();
+        }
+    }
+
+    private InputStream fetchData(String resource) throws IOException {
+        URL url = new URL("http://" + serverHost + ":" + serverPort + resource);
+        URLConnection conn = url.openConnection();
+        conn.connect();
+        return conn.getInputStream();
+    }
+
+    private void run() throws IOException, SQLException {
+        test_connect_plain();
+        test_connect_tls();
+        test_refuse_no_cert();
+        test_refuse_wrong_cert();
+        test_refuse_wrong_host();
+        test_refuse_tlsv12();
+        test_refuse_expired();
+//        test_connect_client_auth1();
+//        test_connect_client_auth2();
+        test_fail_tls_to_plain();
+//        test_fail_plain_to_tls();
+//        test_connect_server_name();
+//        test_connect_alpn_mapi9();
+        test_connect_trusted();
+        test_refuse_trusted_wrong_host();
+    }
+
+    private void test_connect_plain() throws IOException, SQLException {
+        attempt("connect_plain", "plain").with(Parameter.TLS, false).expectSuccess();
+    }
+
+    private void test_connect_tls() throws IOException, SQLException {
+        Attempt attempt = attempt("connect_tls", "server1");
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
+    }
+
+    private void test_refuse_no_cert() throws IOException, SQLException {
+        attempt("refuse_no_cert", "server1").expectFailure("PKIX path building failed");
+    }
+
+    private void test_refuse_wrong_cert() throws IOException, SQLException {
+        Attempt attempt = attempt("refuse_wrong_cert", "server1");
+        attempt.withFile(Parameter.CERT, "/ca2.crt").expectFailure("PKIX path building failed");
+    }
+
+    private void test_refuse_wrong_host() throws IOException, SQLException {
+        Attempt attempt = attempt("refuse_wrong_host", "server1").with(Parameter.HOST, altHost);
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("No subject alternative DNS name");
+    }
+
+    private void test_refuse_tlsv12() throws IOException, SQLException {
+        Attempt attempt = attempt("refuse_tlsv12", "tls12");
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("protocol_version");
+    }
+
+    private void test_refuse_expired() throws IOException, SQLException {
+        Attempt attempt = attempt("refuse_expired", "expiredcert");
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("PKIX path validation failed");
+    }
+
+    private void test_connect_client_auth1() throws IOException, SQLException {
+        attempt("connect_client_auth1", "clientauth")
+                .withFile(Parameter.CERT, "/ca1.crt")
+                .withFile(Parameter.CLIENTKEY, "/client2.keycrt")
+                .expectSuccess();
+    }
+
+    private void test_connect_client_auth2() throws IOException, SQLException {
+        attempt("connect_client_auth2", "clientauth")
+                .withFile(Parameter.CERT, "/ca1.crt")
+                .withFile(Parameter.CLIENTKEY, "/client2.key")
+                .withFile(Parameter.CLIENTCERT, "/client2.crt")
+                .expectSuccess();
+    }
+
+    private void test_fail_tls_to_plain() throws IOException, SQLException {
+        Attempt attempt = attempt("fail_tls_to_plain", "plain");
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("");
+
+    }
+
+    private void test_fail_plain_to_tls() throws IOException, SQLException {
+        attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("asdf");
+    }
+
+    private void test_connect_server_name() throws IOException, SQLException {
+        Attempt attempt = attempt("connect_server_name", "sni");
+        attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
+    }
+
+    private void test_connect_alpn_mapi9() throws IOException, SQLException {
+        attempt("connect_alpn_mapi9", "");
+    }
+
+    private void test_connect_trusted() throws IOException, SQLException {
+        attempt("connect_trusted", "alpn_mapi9")
+                .with(Parameter.HOST, "monetdb.ergates.nl")
+                .with(Parameter.PORT, 50000)
+                .expectSuccess();
+    }
+
+    private void test_refuse_trusted_wrong_host() throws IOException, SQLException {
+        attempt("connect_trusted", null)
+                .with(Parameter.HOST, "monetdbxyz.ergates.nl")
+                .with(Parameter.PORT, 50000)
+                .expectFailure("No subject alternative DNS name");
+    }
+
+    private Attempt attempt(String testName, String portName) throws IOException {
+        return new Attempt(testName, portName);
+    }
+
+    private class Attempt {
+        private final String testName;
+        private final Properties props = new Properties();
+        boolean disabled = false;
+
+        public Attempt(String testName, String portName) throws IOException {
+            HashMap portMap = loadPortMap(testName);
+
+            this.testName = testName;
+            with(Parameter.TLS, true);
+            with(Parameter.HOST, serverHost);
+            with(Parameter.SO_TIMEOUT, 3000);
+            if (portName != null) {
+                Integer port = portMap.get(portName);
+                if (port != null) {
+                    with(Parameter.PORT, port);
+                } else {
+                    throw new RuntimeException("Unknown port name: " + portName);
+                }
+            }
+        }
+
+        private Attempt with(Parameter parm, String value) {
+            props.setProperty(parm.name, value);
+            return this;
+        }
+
+        private Attempt with(Parameter parm, int value) {
+            props.setProperty(parm.name, Integer.toString(value));
+            return this;
+        }
+
+        private Attempt with(Parameter parm, boolean value) {
+            props.setProperty(parm.name, value ? "true" : "false");
+            return this;
+        }
+
+        private Attempt withFile(Parameter parm, String certResource) throws IOException {
+            File certFile = resource(certResource);
+            String path = certFile.getPath();
+            with(parm, path);
+            return this;
+        }
+
+        public void expectSuccess() throws SQLException {
+            if (disabled)
+                return;
+            try {
+                Connection conn = DriverManager.getConnection("jdbc:monetdb:", props);
+                conn.close();
+            } catch (SQLException e) {
+                if (e.getMessage().startsWith("Sorry, this is not a real MonetDB instance")) {
+                    // it looks like a failure but this is actually our success scenario
+                    // because this is what the TLS Tester does when the connection succeeds.
+                    return;
+                }
+                // other exceptions ARE errors and should be reported.
+                throw e;
+            }
+        }
+
+        public void expectFailure(String... expectedMessages) throws SQLException {
+            if (disabled)
+                return;
+            try {
+                expectSuccess();
+                throw new RuntimeException("Expected test " + testName + " to throw an exception but it didn't");
+            } catch (SQLException e) {
+                for (String expected : expectedMessages)
+                    if (e.getMessage().contains(expected))
+                        return;
+                String message = "Test " + testName + " threw the wrong exception: " + e.getMessage() + '\n' + "Expected:\n        <" + String.join(">\n        <", expectedMessages) + ">";
+                throw new RuntimeException(message);
+
+            }
+        }
+
+    }
+}

From 397ac7f824414e3867b71e8b4e39cd5ab46e3dcd Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Mon, 11 Dec 2023 14:47:41 +0100
Subject: [PATCH 16/41] Send NUL bytes on non-TLS connect

This avoids a hang if we accidentally make a non-TLS connection to a
TLS server.

The hang occurs because in that situation, the MAPI client ends up
waiting for the server to send a MAPI challenge, while the TLS server
ends up waiting for the client to send a TLS Client Hello message.

The NUL bytes are illegal as a Client Hello and a no-op as a MAPI
message.
---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 15 +++++++++++++--
 tests/TLSTester.java                              |  4 ++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 60e090a..4495f63 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -86,6 +86,7 @@
  * @see org.monetdb.mcl.io.BufferedMCLWriter
  */
 public final class MapiSocket {
+	public static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0, };
 	private static final String[][] KNOWN_ALGORITHMS = new String[][] {
 			{"SHA512", "SHA-512"},
 			{"SHA384", "SHA-384"},
@@ -348,8 +349,18 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx
 	private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException {
         if (validated.getTls())
             return SecureSocket.wrap(validated, sock);
-        else
-            return sock;
+        else {
+			// Send an even number of NUL bytes.
+			// We expect the server to speak MAPI and in that case, it's a NOP.
+			// If we're accidentally connecting to a TLS server, the bytes are
+			// invalid as a Client Hello message and most TLS implementations
+			// drop the connection.
+			// This is nice because otherwise we would hang, as the TLS server
+			// is waiting for us to send a TLS CLient Hello, and we are waiting
+			// for a MAPI server to send a server challenge.
+			sock.getOutputStream().write(NUL_BYTES);
+		}
+        return sock;
     }
 
 	private boolean handshake(Target.Validated validated, OptionsCallback callback, ArrayList warnings) throws IOException, MCLException {
diff --git a/tests/TLSTester.java b/tests/TLSTester.java
index dffaadc..390b722 100644
--- a/tests/TLSTester.java
+++ b/tests/TLSTester.java
@@ -131,7 +131,7 @@ private void run() throws IOException, SQLException {
 //        test_connect_client_auth1();
 //        test_connect_client_auth2();
         test_fail_tls_to_plain();
-//        test_fail_plain_to_tls();
+        test_fail_plain_to_tls();
 //        test_connect_server_name();
 //        test_connect_alpn_mapi9();
         test_connect_trusted();
@@ -193,7 +193,7 @@ private void test_fail_tls_to_plain() throws IOException, SQLException {
     }
 
     private void test_fail_plain_to_tls() throws IOException, SQLException {
-        attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("asdf");
+        attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect");
     }
 
     private void test_connect_server_name() throws IOException, SQLException {

From 0f1f585d0c91ca9f132f352d202209b64bda5f16 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Mon, 11 Dec 2023 15:04:59 +0100
Subject: [PATCH 17/41] Send SNI (Server Name Indication)

---
 .../org/monetdb/mcl/net/SecureSocket.java     | 25 ++++++++++---------
 tests/TLSTester.java                          |  2 +-
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index d3a2093..fbb279d 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -8,6 +8,8 @@
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
+import java.util.Collections;
+import java.util.List;
 
 public class SecureSocket {
     private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
@@ -16,32 +18,29 @@ public class SecureSocket {
     public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
         Target.Verify verify = validated.connectVerify();
         SSLSocketFactory socketFactory;
+        boolean checkName = true;
         try {
             switch (verify) {
                 case System:
                     socketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
-                    return wrapSocket(inner, validated, socketFactory, true);
+                    break;
                 case Cert:
-                        KeyStore keyStore = keyStoreForCert(validated.getCert());
-                        socketFactory = certBasedSocketFactory(keyStore);
-                    return wrapSocket(inner, validated, socketFactory, true);
+                    KeyStore keyStore = keyStoreForCert(validated.getCert());
+                    socketFactory = certBasedSocketFactory(keyStore);
+                    break;
                 case Hash:
-                    return wrapHash(validated, inner);
+                    socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits());
+                    checkName = false;
+                    break;
                 default:
                     throw new RuntimeException("unreachable: unexpected verification strategy " + verify.name());
             }
+            return wrapSocket(inner, validated, socketFactory, checkName);
         } catch (CertificateException e) {
             throw new SSLException(e.getMessage(), e);
         }
     }
 
-    private static Socket wrapHash(Target.Validated validated, Socket inner) throws IOException, CertificateException {
-        SSLSocketFactory socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits());
-        SSLSocket sock = wrapSocket(inner, validated, socketFactory, false);
-
-        return sock;
-    }
-
     private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException {
         SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true);
 
@@ -50,6 +49,8 @@ private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SS
 
         if (checkName) {
             SSLParameters parameters = sock.getSSLParameters();
+            SNIServerName serverName = new SNIHostName(validated.connectTcp());
+            parameters.setServerNames(Collections.singletonList(serverName));
             parameters.setEndpointIdentificationAlgorithm("HTTPS");
             sock.setSSLParameters(parameters);
         }
diff --git a/tests/TLSTester.java b/tests/TLSTester.java
index 390b722..0ff19d6 100644
--- a/tests/TLSTester.java
+++ b/tests/TLSTester.java
@@ -132,7 +132,7 @@ private void run() throws IOException, SQLException {
 //        test_connect_client_auth2();
         test_fail_tls_to_plain();
         test_fail_plain_to_tls();
-//        test_connect_server_name();
+        test_connect_server_name();
 //        test_connect_alpn_mapi9();
         test_connect_trusted();
         test_refuse_trusted_wrong_host();

From 8f1b69900255d95de638eca1c17a0809ca192932 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Mon, 11 Dec 2023 15:47:19 +0100
Subject: [PATCH 18/41] Set ALPN protocol if the runtime supports it

(Use introspection because Java 8 can't do it)
---
 .../org/monetdb/mcl/net/SecureSocket.java     | 26 +++++++++++++------
 tests/TLSTester.java                          | 21 ++++++++++++---
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index fbb279d..de54d40 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -3,17 +3,18 @@
 import javax.net.ssl.*;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.net.Socket;
 import java.security.*;
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import java.util.Collections;
-import java.util.List;
 
 public class SecureSocket {
     private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
-    final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
+    private static final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
 
     public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
         Target.Verify verify = validated.connectVerify();
@@ -43,17 +44,26 @@ public static Socket wrap(Target.Validated validated, Socket inner) throws IOExc
 
     private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException {
         SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true);
-
         sock.setUseClientMode(true);
-        sock.setEnabledProtocols(ENABLED_PROTOCOLS);
+        SSLParameters parameters = sock.getSSLParameters();
+
+        parameters.setProtocols(ENABLED_PROTOCOLS);
+
+        parameters.setServerNames(Collections.singletonList(new SNIHostName(validated.connectTcp())));
 
         if (checkName) {
-            SSLParameters parameters = sock.getSSLParameters();
-            SNIServerName serverName = new SNIHostName(validated.connectTcp());
-            parameters.setServerNames(Collections.singletonList(serverName));
             parameters.setEndpointIdentificationAlgorithm("HTTPS");
-            sock.setSSLParameters(parameters);
         }
+
+        // Unfortunately, SSLParameters.setApplicationProtocols is only available
+        // since language level 9 and currently we're on 8.
+        // Still call it if it happens to be available.
+        try {
+            Method setApplicationProtocols = SSLParameters.class.getMethod("setApplicationProtocols", String[].class);
+            setApplicationProtocols.invoke(parameters, (Object) APPLICATION_PROTOCOLS);
+        } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) {}
+
+        sock.setSSLParameters(parameters);
         sock.startHandshake();
         return sock;
     }
diff --git a/tests/TLSTester.java b/tests/TLSTester.java
index 0ff19d6..f6298ef 100644
--- a/tests/TLSTester.java
+++ b/tests/TLSTester.java
@@ -9,7 +9,9 @@
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Properties;
+import java.util.stream.Collectors;
 
 public class TLSTester {
     int verbose = 0;
@@ -19,6 +21,7 @@ public class TLSTester {
     boolean enableTrusted = false;
     File tempDir = null;
     final HashMap fileCache = new HashMap<>();
+    private HashSet preparedButNotRun = new HashSet<>();
 
     public TLSTester(String[] args) {
         for (int i = 0; i < args.length; i++) {
@@ -133,9 +136,15 @@ private void run() throws IOException, SQLException {
         test_fail_tls_to_plain();
         test_fail_plain_to_tls();
         test_connect_server_name();
-//        test_connect_alpn_mapi9();
+        test_connect_alpn_mapi9();
         test_connect_trusted();
         test_refuse_trusted_wrong_host();
+
+        // did we forget to call expectSucceed and expectFailure somewhere?
+        if (!preparedButNotRun.isEmpty()) {
+            String names = String.join(", ", preparedButNotRun);
+            throw new RuntimeException("Not all tests called expectSuccess/expectFailure: " + names);
+        }
     }
 
     private void test_connect_plain() throws IOException, SQLException {
@@ -202,24 +211,27 @@ private void test_connect_server_name() throws IOException, SQLException {
     }
 
     private void test_connect_alpn_mapi9() throws IOException, SQLException {
-        attempt("connect_alpn_mapi9", "");
+        attempt("connect_alpn_mapi9", "alpn_mapi9")
+                .withFile(Parameter.CERT, "/ca1.crt")
+                .expectSuccess();
     }
 
     private void test_connect_trusted() throws IOException, SQLException {
-        attempt("connect_trusted", "alpn_mapi9")
+        attempt("connect_trusted", null)
                 .with(Parameter.HOST, "monetdb.ergates.nl")
                 .with(Parameter.PORT, 50000)
                 .expectSuccess();
     }
 
     private void test_refuse_trusted_wrong_host() throws IOException, SQLException {
-        attempt("connect_trusted", null)
+        attempt("test_refuse_trusted_wrong_host", null)
                 .with(Parameter.HOST, "monetdbxyz.ergates.nl")
                 .with(Parameter.PORT, 50000)
                 .expectFailure("No subject alternative DNS name");
     }
 
     private Attempt attempt(String testName, String portName) throws IOException {
+        preparedButNotRun.add(testName);
         return new Attempt(testName, portName);
     }
 
@@ -268,6 +280,7 @@ private Attempt withFile(Parameter parm, String certResource) throws IOException
         }
 
         public void expectSuccess() throws SQLException {
+            preparedButNotRun.remove(testName);
             if (disabled)
                 return;
             try {

From 6ee421265d43bf853ec3945f92b4f3d3a70b5971 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 11:56:58 +0100
Subject: [PATCH 19/41] Change default reply size to 250

(I accidentally changed it to 200)
---
 src/main/java/org/monetdb/mcl/net/Parameter.java | 2 +-
 src/main/java/org/monetdb/mcl/net/Target.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java
index fb236a4..f199423 100644
--- a/src/main/java/org/monetdb/mcl/net/Parameter.java
+++ b/src/main/java/org/monetdb/mcl/net/Parameter.java
@@ -23,7 +23,7 @@ public enum Parameter {
     SCHEMA("schema", ParameterType.Str, "", "initial schema", false),
     TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false),
     BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false),
-    REPLYSIZE("replysize", ParameterType.Int, 200, "rows beyond this limit are retrieved on demand, <1 means unlimited", false),
+    REPLYSIZE("replysize", ParameterType.Int, 250, "rows beyond this limit are retrieved on demand, <1 means unlimited", false),
     FETCHSIZE("fetchsize", ParameterType.Int, null, "alias for replysize, specific to jdbc", false),
     HASH("hash", ParameterType.Str, "", "specific to jdbc", false),
     DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false),
diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index 8b20b2a..4acec2b 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -22,7 +22,7 @@ public class Target {
     private String schema = "";
     private int timezone;
     private String binary = "on";
-    private int replySize = 200;
+    private int replySize = 250;
     private String hash = "";
     private boolean debug = false;
     private String logfile = "";

From 13e5cb6d5c2adbbb21cc1099dfbb1721e8f34828 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 11:58:04 +0100
Subject: [PATCH 20/41] Move code

---
 src/main/java/org/monetdb/mcl/net/Target.java | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index 4acec2b..bad8c86 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -60,6 +60,17 @@ public static String packHost(String host) {
         }
     }
 
+    public static String unpackHost(String host) {
+        switch (host) {
+            case "localhost.":
+                return "localhost";
+            case "localhost":
+                return "";
+            default:
+                return host;
+        }
+    }
+
     public void setString(String key, String value) throws ValidationError {
         Parameter parm = Parameter.forName(key);
         if (parm != null)
@@ -153,17 +164,6 @@ public Object getObject(Parameter parm) {
         }
     }
 
-    public static String unpackHost(String host) {
-        switch (host) {
-            case "localhost.":
-                return "localhost";
-            case "localhost":
-                return "";
-            default:
-                return host;
-        }
-    }
-
     public boolean isTls() {
         return tls;
     }

From e64879e331c563d227d77aaaab1d1a62ff82f92d Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 11:59:03 +0100
Subject: [PATCH 21/41] Fully implement Target.buildURL

---
 .../org/monetdb/mcl/net/MonetUrlParser.java   | 11 ++++++-
 src/main/java/org/monetdb/mcl/net/Target.java | 30 +++++++++++++++----
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
index 036fae1..6a42cfe 100644
--- a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
+++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
@@ -4,6 +4,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLDecoder;
+import java.net.URLEncoder;
 
 public class MonetUrlParser {
     private final Target target;
@@ -49,7 +50,7 @@ public static void parse(Target target, String url) throws URISyntaxException, V
         target.barrier();
     }
 
-    private static String percentDecode(String context, String text) throws URISyntaxException {
+    public static String percentDecode(String context, String text) throws URISyntaxException {
         try {
             return URLDecoder.decode(text, "UTF-8");
         } catch (UnsupportedEncodingException e) {
@@ -59,6 +60,14 @@ private static String percentDecode(String context, String text) throws URISynta
         }
     }
 
+    public static String percentEncode(String text) {
+        try {
+            return URLEncoder.encode(text, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private void parseModern() throws URISyntaxException, ValidationError {
         clearBasic();
 
diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index bad8c86..1bfacea 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -409,11 +409,31 @@ public Validated validate() throws ValidationError {
 
     public String buildUrl() {
         final StringBuilder sb = new StringBuilder(128);
-        sb.append("jdbc:monetdb://").append(host)
-                .append(':').append(port)
-                .append('/').append(database);
-        if (!language.equals("sql"))
-            sb.append("?language=").append(language);
+        sb.append("jdbc:");
+        sb.append(tls ? "monetdbs": "monetdb");
+        sb.append("://");
+        sb.append(packHost(host));
+        if (!Parameter.PORT.getDefault().equals(port)) {
+            sb.append(':');
+            sb.append(port);
+        }
+        sb.append('/').append(database);
+        String sep = "?";
+        for (Parameter parm: Parameter.values()) {
+            if (parm.isCore || parm == Parameter.USER || parm == Parameter.PASSWORD)
+                continue;
+            Object defaultValue = parm.getDefault();
+            if (defaultValue == null)
+                continue;
+            Object value = getObject(parm);
+            if (value.equals(defaultValue))
+                continue;
+            sb.append(sep).append(parm.name).append('=');
+            String raw = getString(parm);
+            String encoded = MonetUrlParser.percentEncode(raw);
+            sb.append(encoded);
+            sep = "&";
+        }
         return sb.toString();
     }
 

From be9a3046f173f554bcbd9c367800a27ca4c232a8 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 12:03:24 +0100
Subject: [PATCH 22/41] Move more URL and properties parsing responsibilities
 to Target

---
 .../java/org/monetdb/jdbc/MonetDriver.java    | 23 ++------------
 src/main/java/org/monetdb/mcl/net/Target.java | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/monetdb/jdbc/MonetDriver.java b/src/main/java/org/monetdb/jdbc/MonetDriver.java
index 0ad1551..019b6ac 100644
--- a/src/main/java/org/monetdb/jdbc/MonetDriver.java
+++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java
@@ -106,31 +106,12 @@ public Connection connect(final String url, Properties info)
 		if (!acceptsURL(url))
 			return null;
 
-		Target target = new Target();
-
 		try {
-			// If properties are given, add those first
-			if (info != null) {
-				for (String key : info.stringPropertyNames()) {
-					String value = info.getProperty(key);
-					if (key.equals(Parameter.HOST.name))
-						value = Target.unpackHost(value);
-					target.setString(key, value);
-				}
-			}
-
-			// If url is exactly "jdbc:monetdb:", use just the properties.
-			// This is different from, say, jdbc:monetdb://, because the
-			// latter will clear preexisting host, port, TLS and database settings.
-			// Useful in combination with Target.toProperties().
-			if (!url.equals("jdbc:monetdb:"))
-				MonetUrlParser.parse(target, url.substring(5));
+			Target target = new Target(url, info);
+			return new MonetConnection(target);
 		} catch (ValidationError | URISyntaxException e) {
 			throw new SQLException(e.getMessage());
 		}
-
-        // finally return the Connection object as requested
-		return new MonetConnection(target);
 	}
 
 	/**
diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index 1bfacea..66732f8 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -1,5 +1,7 @@
 package org.monetdb.mcl.net;
 
+import java.net.URISyntaxException;
+import java.util.Properties;
 import java.util.regex.Pattern;
 
 public class Target {
@@ -42,6 +44,12 @@ public Target() {
         this.timezone = (int)Parameter.TIMEZONE.getDefault();
     }
 
+    public Target(String url, Properties props) throws URISyntaxException, ValidationError {
+        this();
+        setProperties(props);
+        parseUrl(url);
+    }
+
     public void barrier() {
         if (userWasSet && !passwordWasSet)
             password = "";
@@ -89,6 +97,28 @@ public void clear(Parameter parm) {
         assign(parm, parm.getDefault());
     }
 
+    public void setProperties(Properties props) throws ValidationError {
+        if (props != null) {
+            for (String key : props.stringPropertyNames()) {
+                String value = props.getProperty(key);
+                if (key.equals(Parameter.HOST.name))
+                    value = Target.unpackHost(value);
+                setString(key, value);
+            }
+        }
+    }
+
+    public void parseUrl(String url) throws URISyntaxException, ValidationError {
+        if (url == null)
+            return;
+        if (url.startsWith("jdbc:"))
+            url = url.substring(5);
+        if (url.equals("monetdb:")) {
+            return;
+        }
+        MonetUrlParser.parse(this, url);
+    }
+
     private void assign(Parameter parm, Object value) {
         switch (parm) {
             case TLS: setTls((boolean)value); break;

From 8878caf29022d4abf4628fd9d1cc7db5a6409bae Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 12:04:15 +0100
Subject: [PATCH 23/41] Add interface to MonetConnection for retrieving
 connection info as Properties

---
 .../java/org/monetdb/jdbc/MonetConnection.java    |  4 ++++
 src/main/java/org/monetdb/mcl/net/Target.java     | 15 +++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java
index 52fd1cf..d101898 100644
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -1248,6 +1248,10 @@ public boolean isValid(final int timeout) throws SQLException {
 		return isValid;
 	}
 
+	public Properties getConnectionProperties() {
+		return target.getProperties();
+	}
+
 	/**
 	 * Returns the value of the client info property specified by name.
 	 * This method may return null if the specified client info property
diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index 66732f8..29cded1 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -467,6 +467,21 @@ public String buildUrl() {
         return sb.toString();
     }
 
+    public Properties getProperties() {
+        Properties props = new Properties();
+        for (Parameter parm: Parameter.values()) {
+            Object defaultValue = parm.getDefault();
+            if (defaultValue == null || defaultValue.equals(getObject(parm)))
+                continue;
+            String value = getString(parm);
+            if (parm == Parameter.HOST)
+                value = packHost(host);
+            props.setProperty(parm.name, value);
+        }
+
+        return props;
+    }
+
     public class Validated {
 
         private final int nbinary;

From 2e29ec3c09a2b943d08541ebee04ab5afb7d02c0 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 12:06:39 +0100
Subject: [PATCH 24/41] Add MapiSocket.connect(url, properties) method

---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 4495f63..6e9c647 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -21,12 +21,7 @@
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import org.monetdb.mcl.MCLException;
@@ -263,6 +258,10 @@ public List connect(final String host, final int port, final String user
 		return connect(target, null);
 	}
 
+	public List connect(String url, Properties props) throws URISyntaxException, ValidationError, MCLException, MCLParseException, IOException {
+		return connect(new Target(url, props), null);
+	}
+
 	public List connect(Target target, OptionsCallback callback) throws MCLException, MCLParseException, IOException {
 		// get rid of any earlier connection state, including the existing target
 		close();
@@ -695,7 +694,6 @@ public boolean isDebug() {
 		return target.isDebug();
 	}
 
-
 	/**
 	 * Inner class that is used to write data on a normal stream as a
 	 * blocked stream.  A call to the flush() method will write a

From 069fc5e528a0d1d02deea91bc1eacf3e57785959 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 12:10:07 +0100
Subject: [PATCH 25/41] Remove naive url check because the world has become
 more complicated

---
 tests/JDBC_API_Tester.java | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/JDBC_API_Tester.java b/tests/JDBC_API_Tester.java
index 75041dd..989b32b 100644
--- a/tests/JDBC_API_Tester.java
+++ b/tests/JDBC_API_Tester.java
@@ -2159,12 +2159,10 @@ private void Test_PSlargeresponse(String conURL) {
 			sb.append("1. DatabaseMetadata environment retrieval... ");
 
 			// retrieve this to simulate a bug report
-			DatabaseMetaData dbmd = con.getMetaData();
-			if (conURL.startsWith(dbmd.getURL()))
-				sb.append("oke");
-			else
-				sb.append("not oke ").append(dbmd.getURL());
-			sb.append("\n");
+			con.getMetaData().getURL();
+			// There used to be a test "if (conURL.startsWith(dbmdURL))" here
+			// but with the new URLs that is too simplistic.
+			sb.append("oke").append("\n");
 
 			pstmt = con.prepareStatement("select * from columns");
 			sb.append("2. empty call...");

From fad2bcfb09edcf60952065dfb36e49ea9d536b84 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Tue, 12 Dec 2023 12:11:11 +0100
Subject: [PATCH 26/41] Use new MapiSocket.connect() method In CopyIntoSTDIN
 test

---
 tests/JDBC_API_Tester.java | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/tests/JDBC_API_Tester.java b/tests/JDBC_API_Tester.java
index 989b32b..1517199 100644
--- a/tests/JDBC_API_Tester.java
+++ b/tests/JDBC_API_Tester.java
@@ -12,13 +12,11 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.sql.Date;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Iterator;
-import java.util.List;
-import java.util.TimeZone;
+import java.util.*;
 
+import org.monetdb.jdbc.MonetConnection;
 import org.monetdb.jdbc.types.INET;
 import org.monetdb.jdbc.types.URL;
 
@@ -6728,20 +6726,11 @@ private void fillTableUsingCopyIntoSTDIN(final String conn_URL, final String tab
 
 		org.monetdb.mcl.net.MapiSocket server = new org.monetdb.mcl.net.MapiSocket();
 		try {
-			server.setLanguage("sql");
-
-			// extract from conn_URL the used connection properties
-			String host = extractFromJDBCURL(conn_URL, "host");
-			int port = Integer.parseInt(extractFromJDBCURL(conn_URL, "port"));
-			String login = extractFromJDBCURL(conn_URL, "user");
-			String passw = extractFromJDBCURL(conn_URL, "password");
-			String database = extractFromJDBCURL(conn_URL, "database");
-			// sb.append("conn_URL: " + conn_URL + "\n");
-			// sb.append("host: " + host + " port: " + port + " dbname: " + database + " login: " + login + " passwd: " + passw + "\n");
+			MonetConnection mcon = (MonetConnection) con;
+			Properties props = mcon.getConnectionProperties();
 
 			sb.append("Before connecting to MonetDB server via MapiSocket\n");
-			server.setDatabase(database);
-			List warning = server.connect(host, port, login, passw);
+			List warning = server.connect("jdbc:monetdb:", props);
 			if (warning != null) {
 				for (Iterator it = warning.iterator(); it.hasNext(); ) {
 					sb.append("Warning: ").append(it.next().toString()).append("\n");

From 41ed7f0dae1b7e8500354361d923a540ee457e4c Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Wed, 13 Dec 2023 15:39:47 +0100
Subject: [PATCH 27/41] Comments and formatting

---
 .../java/org/monetdb/client/JdbcClient.java   |    3 -
 .../org/monetdb/jdbc/MonetConnection.java     |   45 +-
 .../java/org/monetdb/jdbc/MonetDriver.java    |   16 +-
 .../java/org/monetdb/mcl/net/MapiSocket.java  |   57 +-
 .../org/monetdb/mcl/net/MonetUrlParser.java   |  497 +++---
 .../java/org/monetdb/mcl/net/Parameter.java   |  219 ++-
 .../org/monetdb/mcl/net/ParameterType.java    |  155 +-
 .../org/monetdb/mcl/net/SecureSocket.java     |  335 ++--
 src/main/java/org/monetdb/mcl/net/Target.java | 1543 +++++++++--------
 .../org/monetdb/mcl/net/ValidationError.java  |   12 +-
 tests/TLSTester.java                          |  595 ++++---
 tests/UrlTester.java                          |  812 ++++-----
 12 files changed, 2252 insertions(+), 2037 deletions(-)

diff --git a/src/main/java/org/monetdb/client/JdbcClient.java b/src/main/java/org/monetdb/client/JdbcClient.java
index 8722577..eeaa521 100644
--- a/src/main/java/org/monetdb/client/JdbcClient.java
+++ b/src/main/java/org/monetdb/client/JdbcClient.java
@@ -192,9 +192,6 @@ public final static void main(String[] args) throws Exception {
 				"statements read.  Batching can greatly speedup the " +
 				"process of restoring a database dump.");
 
-//	This  file can contain defaults for the flags user, password, language,
-//	database, save_history, format, host, port, and width.  For example, an
-
 		copts.addIgnored("save_history");
 		copts.addIgnored("format");
 		copts.addIgnored("width");
diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java
index d101898..e6627c7 100644
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -75,7 +75,7 @@ public class MonetConnection
 	extends MonetWrapper
 	implements Connection, AutoCloseable
 {
-	/** All connection parameters */
+	/* All connection parameters */
 	Target target;
 	/** A connection to mserver5 using a TCP socket */
 	private final MapiSocket server;
@@ -139,13 +139,11 @@ public class MonetConnection
 	private DatabaseMetaData dbmd;
 
 	/**
-	 * Constructor of a Connection for MonetDB. At this moment the
-	 * current implementation limits itself to storing the given host,
-	 * database, username and password for later use by the
-	 * createStatement() call.  This constructor is only accessible to
+	 * Constructor of a Connection for MonetDB.
+	 * This constructor is only accessible to
 	 * classes from the jdbc package.
 	 *
-	 * @param target a Target object holding the connection parameters
+	 * @param target a {@link Target} object containing all connection parameters
 	 * @throws SQLException if a database error occurs
 	 * @throws IllegalArgumentException is one of the arguments is null or empty
 	 */
@@ -184,20 +182,20 @@ public class MonetConnection
 		switch (target.getLanguage()) {
 			case "sql":
 				lang = LANG_SQL;
-				queryTempl[0] = "s";		// pre
-				queryTempl[1] = "\n;";		// post
-				queryTempl[2] = "\n;\n";	// separator
-				commandTempl[0] = "X";		// pre
-				commandTempl[1] = "";		// post
+				queryTempl[0] = "s";        // pre
+				queryTempl[1] = "\n;";      // post
+				queryTempl[2] = "\n;\n";    // separator
+				commandTempl[0] = "X";      // pre
+				commandTempl[1] = "";       // post
 				callback = new SqlOptionsCallback();
 				break;
 			case "mal":
 				lang = LANG_MAL;
-				queryTempl[0] = "";		// pre
-				queryTempl[1] = ";\n";		// post
-				queryTempl[2] = ";\n";		// separator
-				commandTempl[0] = "";		// pre
-				commandTempl[1] = "";		// post
+				queryTempl[0] = "";         // pre
+				queryTempl[1] = ";\n";      // post
+				queryTempl[2] = ";\n";      // separator
+				commandTempl[0] = "";       // pre
+				commandTempl[1] = "";       // post
 				break;
 			default:
 				lang = LANG_UNKNOWN;
@@ -1248,6 +1246,15 @@ public boolean isValid(final int timeout) throws SQLException {
 		return isValid;
 	}
 
+	/**
+	 * Construct a Properties object holding all connection parameters such
+	 * as host, port, TLS configuration, autocommit, etc.
+	 * Passing this to {@link DriverManager.getConnection()} together
+	 * with the URL "jdbc:monetdb:" will create a new connection identical to
+	 * the current one.
+	 * @return
+	 */
+
 	public Properties getConnectionProperties() {
 		return target.getProperties();
 	}
@@ -3747,7 +3754,8 @@ public void close() throws IOException {
 		}
 	}
 
-	public static enum SqlOption {
+	/* encode knowledge of currently available handshake options as an enum. */
+	enum SqlOption {
 		Autocommit(1, "auto_commit"),
 		ReplySize(2, "reply_size"),
 		SizeHeader(3, "size_header"),
@@ -3766,6 +3774,7 @@ public static enum SqlOption {
 
 	private class SqlOptionsCallback extends MapiSocket.OptionsCallback {
 		private int level;
+
 		@Override
 		public void addOptions(String lang, int level) {
 			if (!lang.equals("sql"))
@@ -3784,7 +3793,7 @@ public void addOptions(String lang, int level) {
 		}
 
 		private boolean contribute(SqlOption opt, int value) {
-			if (this.level <= opt.level)
+			if (opt.level >= this.level)
 				return false;
 			contribute(opt.field, value);
 			return true;
diff --git a/src/main/java/org/monetdb/jdbc/MonetDriver.java b/src/main/java/org/monetdb/jdbc/MonetDriver.java
index 019b6ac..20e53b7 100644
--- a/src/main/java/org/monetdb/jdbc/MonetDriver.java
+++ b/src/main/java/org/monetdb/jdbc/MonetDriver.java
@@ -8,19 +8,11 @@
 
 package org.monetdb.jdbc;
 
-import org.monetdb.mcl.net.MonetUrlParser;
-import org.monetdb.mcl.net.Parameter;
 import org.monetdb.mcl.net.Target;
 import org.monetdb.mcl.net.ValidationError;
 
 import java.net.URISyntaxException;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.DriverManager;
-import java.sql.DriverPropertyInfo;
-import java.sql.SQLException;
-import java.sql.SQLFeatureNotSupportedException;
-import java.sql.Types;
+import java.sql.*;
 import java.util.Map.Entry;
 import java.util.Properties;
 
@@ -70,11 +62,9 @@ public final class MonetDriver implements Driver {
 	 */
 	@Override
 	public boolean acceptsURL(final String url) {
-        if (url == null)
+		if (url == null)
 			return false;
-        if (url.startsWith("jdbc:monetdb:") || url.startsWith("jdbc:monetdbs:"))
-			return true;
-        return false;
+		return url.startsWith("jdbc:monetdb:") || url.startsWith("jdbc:monetdbs:");
 	}
 
 	/**
diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 6e9c647..120e25e 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -22,7 +22,6 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
-import java.util.stream.Collectors;
 
 import org.monetdb.mcl.MCLException;
 import org.monetdb.mcl.io.BufferedMCLReader;
@@ -81,7 +80,12 @@
  * @see org.monetdb.mcl.io.BufferedMCLWriter
  */
 public final class MapiSocket {
-	public static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0, };
+	/* an even number of NUL bytes used during the handshake */
+	private static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0, };
+
+	/* A mapping between hash algorithm names as used in the MAPI
+	 * protocol, and the names by which the Java runtime knows them.
+	 */
 	private static final String[][] KNOWN_ALGORITHMS = new String[][] {
 			{"SHA512", "SHA-512"},
 			{"SHA384", "SHA-384"},
@@ -262,6 +266,26 @@ public List connect(String url, Properties props) throws URISyntaxExcept
 		return connect(new Target(url, props), null);
 	}
 
+		/**
+	 * Connect according to the settings in the 'target' parameter.
+	 * If followRedirect is false, a RedirectionException is
+	 * thrown when a redirect is encountered.
+	 * 
+	 * Some settings, such as the initial reply size, can already be configured
+	 * during the handshake, saving a command round-trip later on.
+	 * To do so, create and pass a subclass of {@link MapiSocket.OptionsCallback}.
+	 * 
+	 * @param target the connection settings
+	 * @param callback will be called if the server allows options to be set during the
+	 * initial handshake
+	 * @return A List with informational (warning) messages. If this
+	 *		list is empty; then there are no warnings.
+	 * @throws IOException if an I/O error occurs when creating the socket
+	 * @throws SocketException - if there is an error in the underlying protocol, such as a TCP error.
+	 * @throws UnknownHostException if the IP address of the host could not be determined
+	 * @throws MCLParseException if bogus data is received
+	 * @throws MCLException if an MCL related error occurs
+	 */
 	public List connect(Target target, OptionsCallback callback) throws MCLException, MCLParseException, IOException {
 		// get rid of any earlier connection state, including the existing target
 		close();
@@ -521,7 +545,7 @@ private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendP
             }
             return digest;
         }
-		String algoNames = algos.stream().collect(Collectors.joining());
+		String algoNames = String.join(",", algos);
 		throw new MCLException("No supported hash algorithm: " + algoNames);
 	}
 
@@ -1448,9 +1472,35 @@ public int read(final byte[] dest, int off, int len) throws IOException {
 		}
 	}
 
+	/**
+	 * Callback used during the initial MAPI handshake.
+	 * 
+	 * Newer MonetDB versions allow setting some options during the handshake.
+	 * The options are language-specific and each has a 'level'. The server
+	 * advertises up to which level options are supported for a given language
+	 * and for each language/option combination, {@link addOptions} will be invoked.
+	 * It should call {@link contribute} for each option it wants to set.
+	 * 
+	 * At the time of writing, only the 'sql' language supports options,
+	 * they are listed in enum mapi_handshake_options_levels in mapi.h.
+	 */
 	public static abstract class OptionsCallback {
 		private StringBuilder buffer;
 
+		/**
+		 * Callback called for each language/level combination supported by the
+		 * server. May call {@link contribute} for options with a level STRICTLY
+		 * LOWER than the level passed as a parameter.
+		 * @param lang language advertised by the server
+		 * @param level one higher than the maximum supported option
+		 */
+		public abstract void addOptions(String lang, int level);
+
+		/**
+		 * Pass option=value during the handshake
+		 * @param field
+		 * @param value
+		 */
 		protected void contribute(String field, int value) {
 			if (buffer.length() > 0)
 				buffer.append(',');
@@ -1459,7 +1509,6 @@ protected void contribute(String field, int value) {
 			buffer.append(value);
 		}
 
-		public abstract void addOptions(String lang, int level);
 
 		void setBuffer(StringBuilder buf) {
 			buffer = buf;
diff --git a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
index 6a42cfe..5b4534e 100644
--- a/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
+++ b/src/main/java/org/monetdb/mcl/net/MonetUrlParser.java
@@ -6,270 +6,275 @@
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 
+/**
+ * Helper class to keep the URL parsing code separate from the rest of
+ * the {@link Target} class.
+ */
 public class MonetUrlParser {
-    private final Target target;
-    private final String urlText;
-    private final URI url;
+	private final Target target;
+	private final String urlText;
+	private final URI url;
 
-    public MonetUrlParser(Target target, String url) throws URISyntaxException {
-        this.target = target;
-        this.urlText = url;
-        // we want to accept monetdb:// but the Java URI parser rejects that.
-        switch (url) {
-            case "monetdb:-":
-            case "monetdbs:-":
-                throw new URISyntaxException(url, "invalid MonetDB URL");
-            case "monetdb://":
-            case "monetdbs://":
-                url += "-";
-                break;
-        }
-        this.url = new URI(url);
-    }
+	private MonetUrlParser(Target target, String url) throws URISyntaxException {
+		this.target = target;
+		this.urlText = url;
+		// we want to accept monetdb:// but the Java URI parser rejects that.
+		switch (url) {
+			case "monetdb:-":
+			case "monetdbs:-":
+				throw new URISyntaxException(url, "invalid MonetDB URL");
+			case "monetdb://":
+			case "monetdbs://":
+				url += "-";
+				break;
+		}
+		this.url = new URI(url);
+	}
 
-    public static void parse(Target target, String url) throws URISyntaxException, ValidationError {
-        if (url.equals("monetdb://")) {
-            // deal with peculiarity of Java's URI parser
-            url = "monetdb:///";
-        }
+	public static void parse(Target target, String url) throws URISyntaxException, ValidationError {
+		if (url.equals("monetdb://")) {
+			// deal with peculiarity of Java's URI parser
+			url = "monetdb:///";
+		}
 
-        target.barrier();
-        if (url.startsWith("mapi:")) {
-            try {
-                MonetUrlParser parser = new MonetUrlParser(target, url.substring(5));
-                parser.parseClassic();
-            } catch (URISyntaxException e) {
-                URISyntaxException exc = new URISyntaxException(e.getInput(), e.getReason(), -1);
-                exc.setStackTrace(e.getStackTrace());
-                throw exc;
-            }
-        } else {
-            MonetUrlParser parser = new MonetUrlParser(target, url);
-            parser.parseModern();
-        }
-        target.barrier();
-    }
+		target.barrier();
+		if (url.startsWith("mapi:")) {
+			try {
+				MonetUrlParser parser = new MonetUrlParser(target, url.substring(5));
+				parser.parseClassic();
+			} catch (URISyntaxException e) {
+				URISyntaxException exc = new URISyntaxException(e.getInput(), e.getReason(), -1);
+				exc.setStackTrace(e.getStackTrace());
+				throw exc;
+			}
+		} else {
+			MonetUrlParser parser = new MonetUrlParser(target, url);
+			parser.parseModern();
+		}
+		target.barrier();
+	}
 
-    public static String percentDecode(String context, String text) throws URISyntaxException {
-        try {
-            return URLDecoder.decode(text, "UTF-8");
-        } catch (UnsupportedEncodingException e) {
-            throw new IllegalStateException("should be unreachable: UTF-8 unknown??", e);
-        } catch (IllegalArgumentException e) {
-            throw new URISyntaxException(text, context + ": invalid percent escape");
-        }
-    }
+	public static String percentDecode(String context, String text) throws URISyntaxException {
+		try {
+			return URLDecoder.decode(text, "UTF-8");
+		} catch (UnsupportedEncodingException e) {
+			throw new IllegalStateException("should be unreachable: UTF-8 unknown??", e);
+		} catch (IllegalArgumentException e) {
+			throw new URISyntaxException(text, context + ": invalid percent escape");
+		}
+	}
 
-    public static String percentEncode(String text) {
-        try {
-            return URLEncoder.encode(text, "UTF-8");
-        } catch (UnsupportedEncodingException e) {
-            throw new RuntimeException(e);
-        }
-    }
+	public static String percentEncode(String text) {
+		try {
+			return URLEncoder.encode(text, "UTF-8");
+		} catch (UnsupportedEncodingException e) {
+			throw new RuntimeException(e);
+		}
+	}
 
-    private void parseModern() throws URISyntaxException, ValidationError {
-        clearBasic();
+	private void parseModern() throws URISyntaxException, ValidationError {
+		clearBasic();
 
-        String scheme = url.getScheme();
-        if (scheme == null) throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://");
-        switch (scheme) {
-            case "monetdb":
-                target.setTls(false);
-                break;
-            case "monetdbs":
-                target.setTls(true);
-                break;
-            default:
-                throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://");
-        }
+		String scheme = url.getScheme();
+		if (scheme == null)
+			throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://");
+		switch (scheme) {
+			case "monetdb":
+				target.setTls(false);
+				break;
+			case "monetdbs":
+				target.setTls(true);
+				break;
+			default:
+				throw new URISyntaxException(urlText, "URL scheme must be monetdb:// or monetdbs://");
+		}
 
-        // The built-in getHost and getPort methods do strange things
-        // in edge cases such as percent-encoded host names and
-        // invalid port numbers
-        String authority = url.getAuthority();
-        String host;
-        String remainder;
-        int pos;
-        if (authority == null) {
-            if (!url.getRawSchemeSpecificPart().startsWith("//")) {
-                throw new URISyntaxException(urlText, "expected //");
-            }
-            host = "";
-            remainder = "";
-        } else if (authority.equals("-")) {
-            host = "";
-            remainder = "";
-        } else {
-            if (authority.startsWith("[")) {
-                // IPv6
-                pos = authority.indexOf(']');
-                if (pos < 0)
-                    throw new URISyntaxException(urlText, "unmatched '['");
-                host = authority.substring(1, pos);
-                remainder = authority.substring(pos + 1);
-            } else if ((pos = authority.indexOf(':')) >= 0) {
-                host = authority.substring(0, pos);
-                remainder = authority.substring(pos);
-            } else {
-                host = authority;
-                remainder = "";
-            }
-        }
-        host = Target.unpackHost(host);
-        target.setHost(host);
+		// The built-in getHost and getPort methods do strange things
+		// in edge cases such as percent-encoded host names and
+		// invalid port numbers
+		String authority = url.getAuthority();
+		String host;
+		String remainder;
+		int pos;
+		if (authority == null) {
+			if (!url.getRawSchemeSpecificPart().startsWith("//")) {
+				throw new URISyntaxException(urlText, "expected //");
+			}
+			host = "";
+			remainder = "";
+		} else if (authority.equals("-")) {
+			host = "";
+			remainder = "";
+		} else {
+			if (authority.startsWith("[")) {
+				// IPv6
+				pos = authority.indexOf(']');
+				if (pos < 0)
+					throw new URISyntaxException(urlText, "unmatched '['");
+				host = authority.substring(1, pos);
+				remainder = authority.substring(pos + 1);
+			} else if ((pos = authority.indexOf(':')) >= 0) {
+				host = authority.substring(0, pos);
+				remainder = authority.substring(pos);
+			} else {
+				host = authority;
+				remainder = "";
+			}
+		}
+		host = Target.unpackHost(host);
+		target.setHost(host);
 
-        if (remainder.isEmpty()) {
-            // do nothing
-        } else if (remainder.startsWith(":")) {
-            String portStr = remainder.substring(1);
-            try {
-                int port = Integer.parseInt(portStr);
-                if (port <= 0 || port > 65535)
-                    portStr = null;
-            } catch (NumberFormatException e) {
-                portStr = null;
-            }
-            if (portStr == null)
-                throw new ValidationError(urlText, "invalid port number");
-            target.setString(Parameter.PORT, portStr);
-        }
+		if (remainder.isEmpty()) {
+			// do nothing
+		} else if (remainder.startsWith(":")) {
+			String portStr = remainder.substring(1);
+			try {
+				int port = Integer.parseInt(portStr);
+				if (port <= 0 || port > 65535)
+					portStr = null;
+			} catch (NumberFormatException e) {
+				portStr = null;
+			}
+			if (portStr == null)
+				throw new ValidationError(urlText, "invalid port number");
+			target.setString(Parameter.PORT, portStr);
+		}
 
-        String path = url.getRawPath();
-        String[] parts = path.split("/", 4);
-        // <0: empty before leading slash> / <1: database> / <2: tableschema> / <3: table> / <4: should not exist>
-        switch (parts.length) {
-            case 4:
-                target.setString(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3]));
-                // fallthrough
-            case 3:
-                target.setString(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2]));
-                // fallthrough
-            case 2:
-                target.setString(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1]));
-            case 1:
-            case 0:
-                // fallthrough
-                break;
-        }
+		String path = url.getRawPath();
+		String[] parts = path.split("/", 4);
+		// <0: empty before leading slash> / <1: database> / <2: tableschema> / <3: table> / <4: should not exist>
+		switch (parts.length) {
+			case 4:
+				target.setString(Parameter.TABLE, percentDecode(Parameter.TABLE.name, parts[3]));
+				// fallthrough
+			case 3:
+				target.setString(Parameter.TABLESCHEMA, percentDecode(Parameter.TABLESCHEMA.name, parts[2]));
+				// fallthrough
+			case 2:
+				target.setString(Parameter.DATABASE, percentDecode(Parameter.DATABASE.name, parts[1]));
+			case 1:
+			case 0:
+				// fallthrough
+				break;
+		}
 
-        final String query = url.getRawQuery();
-        if (query != null) {
-            final String args[] = query.split("&");
-            for (int i = 0; i < args.length; i++) {
-                pos = args[i].indexOf('=');
-                if (pos <= 0) {
-                    throw new URISyntaxException(args[i], "invalid key=value pair");
-                }
-                String key = args[i].substring(0, pos);
-                key = percentDecode(key, key);
-                Parameter parm = Parameter.forName(key);
-                if (parm != null && parm.isCore)
-                    throw new URISyntaxException(key, key + "= is not allowed as a query parameter");
+		final String query = url.getRawQuery();
+		if (query != null) {
+			final String[] args = query.split("&");
+			for (int i = 0; i < args.length; i++) {
+				pos = args[i].indexOf('=');
+				if (pos <= 0) {
+					throw new URISyntaxException(args[i], "invalid key=value pair");
+				}
+				String key = args[i].substring(0, pos);
+				key = percentDecode(key, key);
+				Parameter parm = Parameter.forName(key);
+				if (parm != null && parm.isCore)
+					throw new URISyntaxException(key, key + "= is not allowed as a query parameter");
 
-                String value = args[i].substring(pos + 1);
-                target.setString(key, percentDecode(key, value));
-            }
-        }
-    }
+				String value = args[i].substring(pos + 1);
+				target.setString(key, percentDecode(key, value));
+			}
+		}
+	}
 
-    private void parseClassic() throws URISyntaxException, ValidationError {
-        if (!url.getRawSchemeSpecificPart().startsWith("//")) {
-            throw new URISyntaxException(urlText, "expected //");
-        }
+	private void parseClassic() throws URISyntaxException, ValidationError {
+		if (!url.getRawSchemeSpecificPart().startsWith("//")) {
+			throw new URISyntaxException(urlText, "expected //");
+		}
 
-        String scheme = url.getScheme();
-        if (scheme == null)
-            scheme = "";
-        switch (scheme) {
-            case "monetdb":
-                parseClassicAuthorityAndPath();
-                break;
-            case "merovingian":
-                String authority = url.getRawAuthority();
-                // authority must be "proxy" ignore authority and path
-                boolean valid = urlText.startsWith("merovingian://proxy?") || urlText.equals("merovingian://proxy");
-                if (!valid)
-                    throw new URISyntaxException(urlText, "with mapi:merovingian:, only //proxy is supported");
-                break;
-            default:
-                throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://");
-        }
+		String scheme = url.getScheme();
+		if (scheme == null)
+			scheme = "";
+		switch (scheme) {
+			case "monetdb":
+				parseClassicAuthorityAndPath();
+				break;
+			case "merovingian":
+				String authority = url.getRawAuthority();
+				// authority must be "proxy" ignore authority and path
+				boolean valid = urlText.startsWith("merovingian://proxy?") || urlText.equals("merovingian://proxy");
+				if (!valid)
+					throw new URISyntaxException(urlText, "with mapi:merovingian:, only //proxy is supported");
+				break;
+			default:
+				throw new URISyntaxException(urlText, "URL scheme must be mapi:monetdb:// or mapi:merovingian://");
+		}
 
-        final String query = url.getRawQuery();
-        if (query != null) {
-            final String args[] = query.split("&");
-            for (int i = 0; i < args.length; i++) {
-                String arg = args[i];
-                if (arg.startsWith("language=")) {
-                    String language = arg.substring(9);
-                    target.setString(Parameter.LANGUAGE, language);
-                } else if (arg.startsWith("database=")) {
-                    String database = arg.substring(9);
-                    target.setString(Parameter.DATABASE, database);
-                } else {
-                    // ignore
-                }
-            }
-        }
-    }
+		final String query = url.getRawQuery();
+		if (query != null) {
+			final String[] args = query.split("&");
+			for (int i = 0; i < args.length; i++) {
+				String arg = args[i];
+				if (arg.startsWith("language=")) {
+					String language = arg.substring(9);
+					target.setString(Parameter.LANGUAGE, language);
+				} else if (arg.startsWith("database=")) {
+					String database = arg.substring(9);
+					target.setString(Parameter.DATABASE, database);
+				} else {
+					// ignore
+				}
+			}
+		}
+	}
 
-    private void parseClassicAuthorityAndPath() throws URISyntaxException, ValidationError {
-        clearBasic();
-        String authority = url.getRawAuthority();
-        String host;
-        String portStr;
-        int pos;
-        if (authority == null) {
-            host = "";
-            portStr = "";
-        } else if (authority.indexOf('@') >= 0) {
-            throw new URISyntaxException(urlText, "user@host syntax is not allowed");
-        } else if ((pos = authority.indexOf(':')) >= 0) {
-            host = authority.substring(0, pos);
-            portStr = authority.substring(pos + 1);
-        } else {
-            host = authority;
-            portStr = "";
-        }
+	private void parseClassicAuthorityAndPath() throws URISyntaxException, ValidationError {
+		clearBasic();
+		String authority = url.getRawAuthority();
+		String host;
+		String portStr;
+		int pos;
+		if (authority == null) {
+			host = "";
+			portStr = "";
+		} else if (authority.indexOf('@') >= 0) {
+			throw new URISyntaxException(urlText, "user@host syntax is not allowed");
+		} else if ((pos = authority.indexOf(':')) >= 0) {
+			host = authority.substring(0, pos);
+			portStr = authority.substring(pos + 1);
+		} else {
+			host = authority;
+			portStr = "";
+		}
 
-        if (!portStr.isEmpty()) {
-            int port;
-            try {
-                port = Integer.parseInt(portStr);
-            } catch (NumberFormatException e) {
-                port = -1;
-            }
-            if (port <= 0) {
-                throw new ValidationError(urlText, "invalid port number");
-            }
-            target.setString(Parameter.PORT, portStr);
-        }
+		if (!portStr.isEmpty()) {
+			int port;
+			try {
+				port = Integer.parseInt(portStr);
+			} catch (NumberFormatException e) {
+				port = -1;
+			}
+			if (port <= 0) {
+				throw new ValidationError(urlText, "invalid port number");
+			}
+			target.setString(Parameter.PORT, portStr);
+		}
 
-        String path = url.getRawPath();
-        if (host.isEmpty() && portStr.isEmpty()) {
-            // socket
-            target.clear(Parameter.HOST);
-            target.setString(Parameter.SOCK, path != null ? path : "");
-        } else {
-            // tcp
-            target.clear(Parameter.SOCK);
-            target.setString(Parameter.HOST, host);
-            if (path == null || path.isEmpty()) {
-                // do nothing
-            } else if (!path.startsWith("/")) {
-                throw new URISyntaxException(urlText, "expect path to start with /");
-            } else {
-                String database = path.substring(1);
-                target.setString(Parameter.DATABASE, database);
-            }
-        }
-    }
+		String path = url.getRawPath();
+		if (host.isEmpty() && portStr.isEmpty()) {
+			// socket
+			target.clear(Parameter.HOST);
+			target.setString(Parameter.SOCK, path != null ? path : "");
+		} else {
+			// tcp
+			target.clear(Parameter.SOCK);
+			target.setString(Parameter.HOST, host);
+			if (path == null || path.isEmpty()) {
+				// do nothing
+			} else if (!path.startsWith("/")) {
+				throw new URISyntaxException(urlText, "expect path to start with /");
+			} else {
+				String database = path.substring(1);
+				target.setString(Parameter.DATABASE, database);
+			}
+		}
+	}
 
-    private void clearBasic() {
-        target.clear(Parameter.TLS);
-        target.clear(Parameter.HOST);
-        target.clear(Parameter.PORT);
-        target.clear(Parameter.DATABASE);
-    }
+	private void clearBasic() {
+		target.clear(Parameter.TLS);
+		target.clear(Parameter.HOST);
+		target.clear(Parameter.PORT);
+		target.clear(Parameter.DATABASE);
+	}
 }
diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java
index f199423..e1bbf68 100644
--- a/src/main/java/org/monetdb/mcl/net/Parameter.java
+++ b/src/main/java/org/monetdb/mcl/net/Parameter.java
@@ -3,99 +3,142 @@
 
 import java.util.Calendar;
 
+/**
+ * Enumerates things that can be configured on a connection to MonetDB.
+ */
 public enum Parameter {
-    TLS("tls", ParameterType.Bool, false, "secure the connection using TLS", true),
-    HOST("host", ParameterType.Str, "", "IP number, domain name or one of the special values `localhost` and `localhost.`", true),
-    PORT("port", ParameterType.Int, -1, "Port to connect to, 1..65535 or -1 for 'not set'", true),
-    DATABASE("database", ParameterType.Str, "", "name of database to connect to", true),
-    TABLESCHEMA("tableschema", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true),
-    TABLE("table", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true),
-    SOCK("sock", ParameterType.Path, "", "path to Unix domain socket to connect to", false),
-    SOCKDIR("sockdir", ParameterType.Path, "/tmp", "Directory for implicit Unix domain sockets (.s.monetdb.PORT)", false),
-    CERT("cert", ParameterType.Path, "", "path to TLS certificate to authenticate server with", false),
-    CERTHASH("certhash", ParameterType.Str, "", "hash of server TLS certificate must start with these hex digits; overrides cert", false),
-    CLIENTKEY("clientkey", ParameterType.Path, "", "path to TLS key (+certs) to authenticate with as client", false),
-    CLIENTCERT("clientcert", ParameterType.Path, "", "path to TLS certs for 'clientkey', if not included there", false),
-    USER("user", ParameterType.Str, "", "user name to authenticate as", false),
-    PASSWORD("password", ParameterType.Str, "", "password to authenticate with", false),
-    LANGUAGE("language", ParameterType.Str, "sql", "for example, \"sql\", \"mal\", \"msql\", \"profiler\"", false),
-    AUTOCOMMIT("autocommit", ParameterType.Bool, true, "initial value of autocommit", false),
-    SCHEMA("schema", ParameterType.Str, "", "initial schema", false),
-    TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false),
-    BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false),
-    REPLYSIZE("replysize", ParameterType.Int, 250, "rows beyond this limit are retrieved on demand, <1 means unlimited", false),
-    FETCHSIZE("fetchsize", ParameterType.Int, null, "alias for replysize, specific to jdbc", false),
-    HASH("hash", ParameterType.Str, "", "specific to jdbc", false),
-    DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false),
-    LOGFILE("logfile", ParameterType.Str, "", "specific to jdbc", false),
+	TLS("tls", ParameterType.Bool, false, "secure the connection using TLS", true),
+	HOST("host", ParameterType.Str, "", "IP number, domain name or one of the special values `localhost` and `localhost.`", true),
+	PORT("port", ParameterType.Int, -1, "Port to connect to, 1..65535 or -1 for 'not set'", true),
+	DATABASE("database", ParameterType.Str, "", "name of database to connect to", true),
+	TABLESCHEMA("tableschema", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true),
+	TABLE("table", ParameterType.Str, "", "only used for REMOTE TABLE, otherwise unused", true),
+	SOCK("sock", ParameterType.Path, "", "path to Unix domain socket to connect to", false),
+	SOCKDIR("sockdir", ParameterType.Path, "/tmp", "Directory for implicit Unix domain sockets (.s.monetdb.PORT)", false),
+	CERT("cert", ParameterType.Path, "", "path to TLS certificate to authenticate server with", false),
+	CERTHASH("certhash", ParameterType.Str, "", "hash of server TLS certificate must start with these hex digits; overrides cert", false),
+	CLIENTKEY("clientkey", ParameterType.Path, "", "path to TLS key (+certs) to authenticate with as client", false),
+	CLIENTCERT("clientcert", ParameterType.Path, "", "path to TLS certs for 'clientkey', if not included there", false),
+	USER("user", ParameterType.Str, "", "user name to authenticate as", false),
+	PASSWORD("password", ParameterType.Str, "", "password to authenticate with", false),
+	LANGUAGE("language", ParameterType.Str, "sql", "for example, \"sql\", \"mal\", \"msql\", \"profiler\"", false),
+	AUTOCOMMIT("autocommit", ParameterType.Bool, true, "initial value of autocommit", false),
+	SCHEMA("schema", ParameterType.Str, "", "initial schema", false),
+	TIMEZONE("timezone", ParameterType.Int, null, "client time zone as minutes east of UTC", false),
+	BINARY("binary", ParameterType.Str, "on", "whether to use binary result set format (number or bool)", false),
+	REPLYSIZE("replysize", ParameterType.Int, 250, "rows beyond this limit are retrieved on demand, <1 means unlimited", false),
+	FETCHSIZE("fetchsize", ParameterType.Int, null, "alias for replysize, specific to jdbc", false),
+	HASH("hash", ParameterType.Str, "", "specific to jdbc", false),
+	DEBUG("debug", ParameterType.Bool, false, "specific to jdbc", false),
+	LOGFILE("logfile", ParameterType.Str, "", "specific to jdbc", false),
 
-    SO_TIMEOUT("so_timeout", ParameterType.Int, 0, "abort if network I/O does not complete in this many milliseconds", false),
-    CLOB_AS_VARCHAR("treat_clob_as_varchar", ParameterType.Bool, true, "return CLOB/TEXT data as type VARCHAR instead of type CLOB", false),
-    BLOB_AS_BINARY("treat_blob_as_binary", ParameterType.Bool, true, "return BLOB data as type BINARY instead of type BLOB", false),
-    ;
+	SO_TIMEOUT("so_timeout", ParameterType.Int, 0, "abort if network I/O does not complete in this many milliseconds", false), CLOB_AS_VARCHAR("treat_clob_as_varchar", ParameterType.Bool, true, "return CLOB/TEXT data as type VARCHAR instead of type CLOB", false), BLOB_AS_BINARY("treat_blob_as_binary", ParameterType.Bool, true, "return BLOB data as type BINARY instead of type BLOB", false),
+	;
 
-    public final String name;
-    public final ParameterType type;
-    private final Object defaultValue;
-    public final String description;
-    public final boolean isCore;
+	public final String name;
+	public final ParameterType type;
+	public final String description;
+	public final boolean isCore;
+	private final Object defaultValue;
 
-    Parameter(String name, ParameterType type, Object defaultValue, String description, boolean isCore) {
-        this.name = name;
-        this.type = type;
-        this.isCore = isCore;
-        this.defaultValue = defaultValue;
-        this.description = description;
-    }
+	Parameter(String name, ParameterType type, Object defaultValue, String description, boolean isCore) {
+		this.name = name;
+		this.type = type;
+		this.isCore = isCore;
+		this.defaultValue = defaultValue;
+		this.description = description;
+	}
 
-    public static Parameter forName(String name) {
-        switch (name) {
-            case "tls": return TLS;
-            case "host": return HOST;
-            case "port": return PORT;
-            case "database": return DATABASE;
-            case "tableschema": return TABLESCHEMA;
-            case "table": return TABLE;
-            case "sock": return SOCK;
-            case "sockdir": return SOCKDIR;
-            case "cert": return CERT;
-            case "certhash": return CERTHASH;
-            case "clientkey": return CLIENTKEY;
-            case "clientcert": return CLIENTCERT;
-            case "user": return USER;
-            case "password": return PASSWORD;
-            case "language": return LANGUAGE;
-            case "autocommit": return AUTOCOMMIT;
-            case "schema": return SCHEMA;
-            case "timezone": return TIMEZONE;
-            case "binary": return BINARY;
-            case "replysize": return REPLYSIZE;
-            case "fetchsize": return FETCHSIZE;
-            case "hash": return HASH;
-            case "debug": return DEBUG;
-            case "logfile": return LOGFILE;
-            case "so_timeout": return SO_TIMEOUT;
-            case "treat_clob_as_varchar": return CLOB_AS_VARCHAR;
-            case "treat_blob_as_binary": return BLOB_AS_BINARY;
-            default: return null;
-        }
-    }
+	public static Parameter forName(String name) {
+		switch (name) {
+			case "tls":
+				return TLS;
+			case "host":
+				return HOST;
+			case "port":
+				return PORT;
+			case "database":
+				return DATABASE;
+			case "tableschema":
+				return TABLESCHEMA;
+			case "table":
+				return TABLE;
+			case "sock":
+				return SOCK;
+			case "sockdir":
+				return SOCKDIR;
+			case "cert":
+				return CERT;
+			case "certhash":
+				return CERTHASH;
+			case "clientkey":
+				return CLIENTKEY;
+			case "clientcert":
+				return CLIENTCERT;
+			case "user":
+				return USER;
+			case "password":
+				return PASSWORD;
+			case "language":
+				return LANGUAGE;
+			case "autocommit":
+				return AUTOCOMMIT;
+			case "schema":
+				return SCHEMA;
+			case "timezone":
+				return TIMEZONE;
+			case "binary":
+				return BINARY;
+			case "replysize":
+				return REPLYSIZE;
+			case "fetchsize":
+				return FETCHSIZE;
+			case "hash":
+				return HASH;
+			case "debug":
+				return DEBUG;
+			case "logfile":
+				return LOGFILE;
+			case "so_timeout":
+				return SO_TIMEOUT;
+			case "treat_clob_as_varchar":
+				return CLOB_AS_VARCHAR;
+			case "treat_blob_as_binary":
+				return BLOB_AS_BINARY;
+			default:
+				return null;
+		}
+	}
 
-    public static boolean isIgnored(String name) {
-        if (Parameter.forName(name) != null)
-            return false;
-        return name.contains("_");
-    }
+	/**
+	 * Determine if a given setting can safely be ignored.
+	 * The ground rule is that if we encounter an unknown setting
+	 * without an underscore in the name, it is an error. If it has
+	 * an underscore in its name, it can be ignored.
+	 * @param name the name of the setting to check
+	 * @return true if it can safely be ignored
+	 */
+	public static boolean isIgnored(String name) {
+		if (Parameter.forName(name) != null)
+			return false;
+		return name.contains("_");
+	}
 
-    public Object getDefault() {
-        switch (this) {
-            case TIMEZONE:
-                Calendar cal = Calendar.getInstance();
-                int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET);
-                int offsetSeconds = offsetMillis / 1000;
-                return (Integer)offsetSeconds;
-            default:
-                return defaultValue;
-        }
-    }
+	/**
+	 * Return a default value for the given setting, as an Object of the appropriate type.
+	 * Note that the value returned for TIMEZONE may change if the system time zone
+	 * is changed or if Daylight Saving Time starts or ends.
+	 * @return
+	 */
+	public Object getDefault() {
+		switch (this) {
+			case TIMEZONE:
+				Calendar cal = Calendar.getInstance();
+				int offsetMillis = cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET);
+				int offsetSeconds = offsetMillis / 1000;
+				return offsetSeconds;
+			default:
+				return defaultValue;
+		}
+	}
 }
diff --git a/src/main/java/org/monetdb/mcl/net/ParameterType.java b/src/main/java/org/monetdb/mcl/net/ParameterType.java
index 15bad0c..e0c5191 100644
--- a/src/main/java/org/monetdb/mcl/net/ParameterType.java
+++ b/src/main/java/org/monetdb/mcl/net/ParameterType.java
@@ -1,68 +1,103 @@
 package org.monetdb.mcl.net;
 
+/**
+ * Enumeration of the types a {@link Parameter} may have.
+ */
 public enum ParameterType {
-    Str,
-    Int,
-    Bool,
-    Path;
+	/**
+	 * The Parameter is an arbitrary string
+	 */
+	Str,
+	/** The Parameter can be interpreted as an {@link Integer} */
+	Int,
+	/** The Parameter is a {@link Boolean} and can be
+	 * written "true", "false", "on", "off", "yes" or "no".
+	 * Uppercase letters are also accepted
+	 */
+	Bool,
+	/**
+	 * Functionally the same as {@link ParameterType.Str } but
+	 * indicates the value is to be interpreted as a path on the
+	 * client's file system.
+	 */
+	Path;
 
-    public Object parse(String name, String value) throws ValidationError {
-        if (value == null)
-            return null;
+	/**
+	 * Convert a string to a boolean, accepting true/false/yes/no/on/off.
+	 * 
+	 * Uppercase is also accepted.
+	 * @param value text to be parsed
+	 * @return boolean interpretation of the text
+	 */
+	public static boolean parseBool(String value) {
+		boolean lowered = false;
+		String original = value;
+		while (true) {
+			switch (value) {
+				case "true":
+				case "yes":
+				case "on":
+					return true;
+				case "false":
+				case "no":
+				case "off":
+					return false;
+				default:
+					if (!lowered) {
+						value = value.toLowerCase();
+						lowered = true;
+						continue;
+					}
+					throw new IllegalArgumentException("invalid boolean value: " + original);
+			}
+		}
+	}
 
-        try {
-            switch (this) {
-                case Bool:
-                    return parseBool(value);
-                case Int:
-                    return Integer.parseInt(value);
-                case Str:
-                case Path:
-                    return value;
-                default:
-                    throw new IllegalStateException("unreachable");
-            }
-        } catch (IllegalArgumentException e) {
-            String message = e.toString();
-            throw new ValidationError(name, message);
-        }
-    }
+	/**
+	 * Convert text into an Object of the appropriate type
+	 * @param name name of the setting for use in error messages
+	 * @param value text to be converted
+	 * @return Object representation of the text
+	 * @throws ValidationError if the text cannot be converted
+	 */
+	public Object parse(String name, String value) throws ValidationError {
+		if (value == null)
+			return null;
 
-    public String format(Object value) {
-        switch (this) {
-                case Bool:
-                    return (Boolean)value ? "true": "false";
-                case Int:
-                    return Integer.toString((Integer)value);
-                case Str:
-                case Path:
-                    return (String) value;
-                default:
-                    throw new IllegalStateException("unreachable");
-            }
-    }
+		try {
+			switch (this) {
+				case Bool:
+					return parseBool(value);
+				case Int:
+					return Integer.parseInt(value);
+				case Str:
+				case Path:
+					return value;
+				default:
+					throw new IllegalStateException("unreachable");
+			}
+		} catch (IllegalArgumentException e) {
+			String message = e.toString();
+			throw new ValidationError(name, message);
+		}
+	}
 
-    public static boolean parseBool(String value) {
-        boolean lowered = false;
-        String original = value;
-        while (true) {
-            switch (value) {
-                case "true":
-                case "yes":
-                case "on":
-                    return true;
-                case "false":
-                case "no":
-                case "off":
-                    return false;
-                default:
-                    if (!lowered) {
-                        value = value.toLowerCase();
-                        lowered = true;
-                        continue;
-                    }
-                    throw new IllegalArgumentException("invalid boolean value: " + original);
-            }
-        }
-    }
+	/**
+	 * Represent the object as a string.
+	 * @param value, must be of the appropriate type
+	 * @return textual representation
+	 */
+	public String format(Object value) {
+		switch (this) {
+			case Bool:
+				return (Boolean) value ? "true" : "false";
+			case Int:
+				return Integer.toString((Integer) value);
+			case Str:
+			case Path:
+				return (String) value;
+			default:
+				throw new IllegalStateException("unreachable");
+		}
+	}
 }
diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index de54d40..9fcb7b7 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -13,171 +13,172 @@
 import java.util.Collections;
 
 public class SecureSocket {
-    private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
-    private static final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
-
-    public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
-        Target.Verify verify = validated.connectVerify();
-        SSLSocketFactory socketFactory;
-        boolean checkName = true;
-        try {
-            switch (verify) {
-                case System:
-                    socketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
-                    break;
-                case Cert:
-                    KeyStore keyStore = keyStoreForCert(validated.getCert());
-                    socketFactory = certBasedSocketFactory(keyStore);
-                    break;
-                case Hash:
-                    socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits());
-                    checkName = false;
-                    break;
-                default:
-                    throw new RuntimeException("unreachable: unexpected verification strategy " + verify.name());
-            }
-            return wrapSocket(inner, validated, socketFactory, checkName);
-        } catch (CertificateException e) {
-            throw new SSLException(e.getMessage(), e);
-        }
-    }
-
-    private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException {
-        SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true);
-        sock.setUseClientMode(true);
-        SSLParameters parameters = sock.getSSLParameters();
-
-        parameters.setProtocols(ENABLED_PROTOCOLS);
-
-        parameters.setServerNames(Collections.singletonList(new SNIHostName(validated.connectTcp())));
-
-        if (checkName) {
-            parameters.setEndpointIdentificationAlgorithm("HTTPS");
-        }
-
-        // Unfortunately, SSLParameters.setApplicationProtocols is only available
-        // since language level 9 and currently we're on 8.
-        // Still call it if it happens to be available.
-        try {
-            Method setApplicationProtocols = SSLParameters.class.getMethod("setApplicationProtocols", String[].class);
-            setApplicationProtocols.invoke(parameters, (Object) APPLICATION_PROTOCOLS);
-        } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) {}
-
-        sock.setSSLParameters(parameters);
-        sock.startHandshake();
-        return sock;
-    }
-
-    private static X509Certificate loadCertificate(String path) throws CertificateException, IOException {
-        CertificateFactory factory = CertificateFactory.getInstance("X509");
-        try (FileInputStream s = new FileInputStream(path)) {
-            return (X509Certificate) factory.generateCertificate(s);
-        }
-    }
-
-    private static SSLSocketFactory certBasedSocketFactory(KeyStore store) throws IOException, CertificateException {
-        TrustManagerFactory trustManagerFactory;
-        try {
-            trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
-            trustManagerFactory.init(store);
-        } catch (NoSuchAlgorithmException | KeyStoreException e) {
-            throw new RuntimeException("Could not create TrustManagerFactory", e);
-        }
-
-        SSLContext context;
-        try {
-            context = SSLContext.getInstance("TLS");
-            context.init(null, trustManagerFactory.getTrustManagers(), null);
-        } catch (NoSuchAlgorithmException | KeyManagementException e) {
-            throw new RuntimeException("Could not create SSLContext", e);
-        }
-
-        return context.getSocketFactory();
-    }
-
-    private static KeyStore keyStoreForCert(String path) throws IOException, CertificateException {
-        try {
-            X509Certificate cert = loadCertificate(path);
-            KeyStore store = emptyKeyStore();
-            store.setCertificateEntry("root", cert);
-            return store;
-        } catch (KeyStoreException e) {
-            throw new RuntimeException("Could not create KeyStore for certificate", e);
-        }
-    }
-
-    private static KeyStore emptyKeyStore() throws IOException, CertificateException {
-        KeyStore store;
-        try {
-            store = KeyStore.getInstance("PKCS12");
-            store.load(null, null);
-            return store;
-        } catch (KeyStoreException | NoSuchAlgorithmException e) {
-            throw new RuntimeException("Could not create KeyStore for certificate", e);
-        }
-    }
-
-    private static SSLSocketFactory hashBasedSocketFactory(String hashDigits) {
-        TrustManager trustManager = new HashBasedTrustManager(hashDigits);
-        try {
-            SSLContext context = SSLContext.getInstance("TLS");
-            context.init(null, new TrustManager[]{ trustManager}, null);
-            return context.getSocketFactory();
-        } catch (NoSuchAlgorithmException | KeyManagementException e) {
-            throw new RuntimeException("Could not create SSLContext", e);
-        }
-
-    }
-
-    private static class HashBasedTrustManager implements X509TrustManager {
-        private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();
-        private final String hashDigits;
-
-        public HashBasedTrustManager(String hashDigits) {
-            this.hashDigits = hashDigits;
-        }
-
-
-        @Override
-        public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
-            throw new RuntimeException("this TrustManager is only suitable for client side connections");
-        }
-
-        @Override
-        public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
-            X509Certificate cert = x509Certificates[0];
-            byte[] certBytes = cert.getEncoded();
-
-            // for now it's always SHA256.
-            byte[] hashBytes;
-            try {
-                MessageDigest hasher = MessageDigest.getInstance("SHA-256");
-                hasher.update(certBytes);
-                hashBytes = hasher.digest();
-            } catch (NoSuchAlgorithmException e) {
-                throw new RuntimeException("failed to instantiate hash digest");
-            }
-
-            // convert to hex digits
-            StringBuilder buffer = new StringBuilder(2 * hashBytes.length);
-            for (byte b: hashBytes) {
-                int hi = (b & 0xF0) >> 4;
-                int lo = b & 0x0F;
-                buffer.append(HEXDIGITS[hi]);
-                buffer.append(HEXDIGITS[lo]);
-            }
-            String certDigits = buffer.toString();
-
-            if (!certDigits.startsWith(hashDigits)) {
-                throw new CertificateException("Certificate hash does not start with '" + hashDigits + "': " + certDigits);
-            }
-
-
-        }
-
-        @Override
-        public X509Certificate[] getAcceptedIssuers() {
-            return new X509Certificate[0];
-        }
-    }
+	private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
+	private static final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
+
+	public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
+		Target.Verify verify = validated.connectVerify();
+		SSLSocketFactory socketFactory;
+		boolean checkName = true;
+		try {
+			switch (verify) {
+				case System:
+					socketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
+					break;
+				case Cert:
+					KeyStore keyStore = keyStoreForCert(validated.getCert());
+					socketFactory = certBasedSocketFactory(keyStore);
+					break;
+				case Hash:
+					socketFactory = hashBasedSocketFactory(validated.connectCertHashDigits());
+					checkName = false;
+					break;
+				default:
+					throw new RuntimeException("unreachable: unexpected verification strategy " + verify.name());
+			}
+			return wrapSocket(inner, validated, socketFactory, checkName);
+		} catch (CertificateException e) {
+			throw new SSLException(e.getMessage(), e);
+		}
+	}
+
+	private static SSLSocket wrapSocket(Socket inner, Target.Validated validated, SSLSocketFactory socketFactory, boolean checkName) throws IOException {
+		SSLSocket sock = (SSLSocket) socketFactory.createSocket(inner, validated.connectTcp(), validated.connectPort(), true);
+		sock.setUseClientMode(true);
+		SSLParameters parameters = sock.getSSLParameters();
+
+		parameters.setProtocols(ENABLED_PROTOCOLS);
+
+		parameters.setServerNames(Collections.singletonList(new SNIHostName(validated.connectTcp())));
+
+		if (checkName) {
+			parameters.setEndpointIdentificationAlgorithm("HTTPS");
+		}
+
+		// Unfortunately, SSLParameters.setApplicationProtocols is only available
+		// since language level 9 and currently we're on 8.
+		// Still call it if it happens to be available.
+		try {
+			Method setApplicationProtocols = SSLParameters.class.getMethod("setApplicationProtocols", String[].class);
+			setApplicationProtocols.invoke(parameters, (Object) APPLICATION_PROTOCOLS);
+		} catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException ignored) {
+		}
+
+		sock.setSSLParameters(parameters);
+		sock.startHandshake();
+		return sock;
+	}
+
+	private static X509Certificate loadCertificate(String path) throws CertificateException, IOException {
+		CertificateFactory factory = CertificateFactory.getInstance("X509");
+		try (FileInputStream s = new FileInputStream(path)) {
+			return (X509Certificate) factory.generateCertificate(s);
+		}
+	}
+
+	private static SSLSocketFactory certBasedSocketFactory(KeyStore store) throws IOException, CertificateException {
+		TrustManagerFactory trustManagerFactory;
+		try {
+			trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+			trustManagerFactory.init(store);
+		} catch (NoSuchAlgorithmException | KeyStoreException e) {
+			throw new RuntimeException("Could not create TrustManagerFactory", e);
+		}
+
+		SSLContext context;
+		try {
+			context = SSLContext.getInstance("TLS");
+			context.init(null, trustManagerFactory.getTrustManagers(), null);
+		} catch (NoSuchAlgorithmException | KeyManagementException e) {
+			throw new RuntimeException("Could not create SSLContext", e);
+		}
+
+		return context.getSocketFactory();
+	}
+
+	private static KeyStore keyStoreForCert(String path) throws IOException, CertificateException {
+		try {
+			X509Certificate cert = loadCertificate(path);
+			KeyStore store = emptyKeyStore();
+			store.setCertificateEntry("root", cert);
+			return store;
+		} catch (KeyStoreException e) {
+			throw new RuntimeException("Could not create KeyStore for certificate", e);
+		}
+	}
+
+	private static KeyStore emptyKeyStore() throws IOException, CertificateException {
+		KeyStore store;
+		try {
+			store = KeyStore.getInstance("PKCS12");
+			store.load(null, null);
+			return store;
+		} catch (KeyStoreException | NoSuchAlgorithmException e) {
+			throw new RuntimeException("Could not create KeyStore for certificate", e);
+		}
+	}
+
+	private static SSLSocketFactory hashBasedSocketFactory(String hashDigits) {
+		TrustManager trustManager = new HashBasedTrustManager(hashDigits);
+		try {
+			SSLContext context = SSLContext.getInstance("TLS");
+			context.init(null, new TrustManager[]{trustManager}, null);
+			return context.getSocketFactory();
+		} catch (NoSuchAlgorithmException | KeyManagementException e) {
+			throw new RuntimeException("Could not create SSLContext", e);
+		}
+
+	}
+
+	private static class HashBasedTrustManager implements X509TrustManager {
+		private static final char[] HEXDIGITS = "0123456789abcdef".toCharArray();
+		private final String hashDigits;
+
+		public HashBasedTrustManager(String hashDigits) {
+			this.hashDigits = hashDigits;
+		}
+
+
+		@Override
+		public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+			throw new RuntimeException("this TrustManager is only suitable for client side connections");
+		}
+
+		@Override
+		public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+			X509Certificate cert = x509Certificates[0];
+			byte[] certBytes = cert.getEncoded();
+
+			// for now it's always SHA256.
+			byte[] hashBytes;
+			try {
+				MessageDigest hasher = MessageDigest.getInstance("SHA-256");
+				hasher.update(certBytes);
+				hashBytes = hasher.digest();
+			} catch (NoSuchAlgorithmException e) {
+				throw new RuntimeException("failed to instantiate hash digest");
+			}
+
+			// convert to hex digits
+			StringBuilder buffer = new StringBuilder(2 * hashBytes.length);
+			for (byte b : hashBytes) {
+				int hi = (b & 0xF0) >> 4;
+				int lo = b & 0x0F;
+				buffer.append(HEXDIGITS[hi]);
+				buffer.append(HEXDIGITS[lo]);
+			}
+			String certDigits = buffer.toString();
+
+			if (!certDigits.startsWith(hashDigits)) {
+				throw new CertificateException("Certificate hash does not start with '" + hashDigits + "': " + certDigits);
+			}
+
+
+		}
+
+		@Override
+		public X509Certificate[] getAcceptedIssuers() {
+			return new X509Certificate[0];
+		}
+	}
 }
diff --git a/src/main/java/org/monetdb/mcl/net/Target.java b/src/main/java/org/monetdb/mcl/net/Target.java
index 29cded1..4aa7402 100644
--- a/src/main/java/org/monetdb/mcl/net/Target.java
+++ b/src/main/java/org/monetdb/mcl/net/Target.java
@@ -5,731 +5,820 @@
 import java.util.regex.Pattern;
 
 public class Target {
-    private boolean tls = false;
-    private String host = "";
-    private int port = -1;
-    private String database = "";
-    private String tableschema = "";
-    private String table = "";
-    private String sock = "";
-    private String sockdir = "/tmp";
-    private String cert = "";
-    private String certhash = "";
-    private String clientkey = "";
-    private String clientcert = "";
-    private String user = "";
-    private String password = "";
-    private String language = "sql";
-    private boolean autocommit = true;
-    private String schema = "";
-    private int timezone;
-    private String binary = "on";
-    private int replySize = 250;
-    private String hash = "";
-    private boolean debug = false;
-    private String logfile = "";
-    private int soTimeout = 0;
-    private boolean treatClobAsVarchar = true;
-    private boolean treatBlobAsBinary = true;
-
-    private boolean userWasSet = false;
-    private boolean passwordWasSet = false;
-    protected static final Target defaults = new Target();
-    private Validated validated = null;
-
-    private static Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$");
-    private static Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$");
-
-    public Target() {
-        this.timezone = (int)Parameter.TIMEZONE.getDefault();
-    }
-
-    public Target(String url, Properties props) throws URISyntaxException, ValidationError {
-        this();
-        setProperties(props);
-        parseUrl(url);
-    }
-
-    public void barrier() {
-        if (userWasSet && !passwordWasSet)
-            password = "";
-        userWasSet = false;
-        passwordWasSet = false;
-    }
-
-    public static String packHost(String host) {
-        switch (host) {
-            case "localhost":
-                return "localhost.";
-            case "":
-                return "localhost";
-            default:
-                return host;
-        }
-    }
-
-    public static String unpackHost(String host) {
-        switch (host) {
-            case "localhost.":
-                return "localhost";
-            case "localhost":
-                return "";
-            default:
-                return host;
-        }
-    }
-
-    public void setString(String key, String value) throws ValidationError {
-        Parameter parm = Parameter.forName(key);
-        if (parm != null)
-            setString(parm, value);
-        else if (!Parameter.isIgnored(key))
-            throw new ValidationError(key, "unknown parameter");
-    }
-
-    public void setString(Parameter parm, String value) throws ValidationError {
-        if (value == null)
-            throw new NullPointerException("'value' must not be null");
-        assign(parm, parm.type.parse(parm.name, value));
-    }
-
-    public void clear(Parameter parm) {
-        assign(parm, parm.getDefault());
-    }
-
-    public void setProperties(Properties props) throws ValidationError {
-        if (props != null) {
-            for (String key : props.stringPropertyNames()) {
-                String value = props.getProperty(key);
-                if (key.equals(Parameter.HOST.name))
-                    value = Target.unpackHost(value);
-                setString(key, value);
-            }
-        }
-    }
-
-    public void parseUrl(String url) throws URISyntaxException, ValidationError {
-        if (url == null)
-            return;
-        if (url.startsWith("jdbc:"))
-            url = url.substring(5);
-        if (url.equals("monetdb:")) {
-            return;
-        }
-        MonetUrlParser.parse(this, url);
-    }
-
-    private void assign(Parameter parm, Object value) {
-        switch (parm) {
-            case TLS: setTls((boolean)value); break;
-            case HOST: setHost((String)value); break;
-            case PORT: setPort((int)value); break;
-            case DATABASE: setDatabase((String)value); break;
-            case TABLESCHEMA: setTableschema((String)value); break;
-            case TABLE: setTable((String)value); break;
-            case SOCK: setSock((String)value); break;
-            case SOCKDIR: setSockdir((String)value); break;
-            case CERT: setCert((String)value); break;
-            case CERTHASH: setCerthash((String)value); break;
-            case CLIENTKEY: setClientkey((String)value); break;
-            case CLIENTCERT: setClientcert((String)value); break;
-            case USER: setUser((String)value); break;
-            case PASSWORD: setPassword((String)value); break;
-            case LANGUAGE: setLanguage((String)value); break;
-            case AUTOCOMMIT: setAutocommit((boolean)value); break;
-            case SCHEMA: setSchema((String)value); break;
-            case TIMEZONE: setTimezone((int)value); break;
-            case BINARY: setBinary((String)value); break;
-            case REPLYSIZE: setReplySize((int)value); break;
-            case FETCHSIZE: setReplySize((int)value); break;
-            case HASH: setHash((String)value); break;
-            case DEBUG: setDebug((boolean)value); break;
-            case LOGFILE: setLogfile((String)value); break;
-
-            case SO_TIMEOUT: setSoTimeout((int)value); break;
-            case CLOB_AS_VARCHAR: setTreatClobAsVarchar((boolean)value); break;
-            case BLOB_AS_BINARY: setTreatBlobAsBinary((boolean)value); break;
-
-            default:
-                throw new IllegalStateException("unreachable -- missing case: " + parm.name);
-        }
-    }
-
-    public String getString(Parameter parm) {
-        Object value = getObject(parm);
-        return parm.type.format(value);
-    }
-
-    public Object getObject(Parameter parm) {
-        switch (parm) {
-            case TLS: return tls;
-            case HOST: return host;
-            case PORT: return port;
-            case DATABASE: return database;
-            case TABLESCHEMA: return tableschema;
-            case TABLE: return table;
-            case SOCK: return sock;
-            case SOCKDIR: return sockdir;
-            case CERT: return cert;
-            case CERTHASH: return certhash;
-            case CLIENTKEY: return clientkey;
-            case CLIENTCERT: return clientcert;
-            case USER: return user;
-            case PASSWORD: return password;
-            case LANGUAGE: return language;
-            case AUTOCOMMIT: return autocommit;
-            case SCHEMA: return schema;
-            case TIMEZONE: return timezone;
-            case BINARY: return binary;
-            case REPLYSIZE: return replySize;
-            case FETCHSIZE: return replySize;
-            case HASH: return hash;
-            case DEBUG: return debug;
-            case LOGFILE: return logfile;
-            case SO_TIMEOUT: return soTimeout;
-            case CLOB_AS_VARCHAR: return treatClobAsVarchar;
-            case BLOB_AS_BINARY: return treatBlobAsBinary;
-            default:
-                throw new IllegalStateException("unreachable -- missing case");
-        }
-    }
-
-    public boolean isTls() {
-        return tls;
-    }
-
-    public void setTls(boolean tls) {
-        this.tls = tls;
-        validated = null;
-    }
-
-    public String getHost() {
-        return host;
-    }
-
-    public void setHost(String host) {
-        this.host = host;
-        validated = null;
-    }
-
-    public int getPort() {
-        return port;
-    }
-
-    public void setPort(int port) {
-        this.port = port;
-        validated = null;
-    }
-
-    public String getDatabase() {
-        return database;
-    }
-
-    public void setDatabase(String database) {
-        this.database = database;
-        validated = null;
-    }
-
-    public String getTableschema() {
-        return tableschema;
-    }
-
-    public void setTableschema(String tableschema) {
-        this.tableschema = tableschema;
-        validated = null;
-    }
-
-    public String getTable() {
-        return table;
-    }
-
-    public void setTable(String table) {
-        this.table = table;
-        validated = null;
-    }
-
-    public String getSock() {
-        return sock;
-    }
-
-    public void setSock(String sock) {
-        this.sock = sock;
-        validated = null;
-    }
-
-    public String getSockdir() {
-        return sockdir;
-    }
-
-    public void setSockdir(String sockdir) {
-        this.sockdir = sockdir;
-        validated = null;
-    }
-
-    public String getCert() {
-        return cert;
-    }
-
-    public void setCert(String cert) {
-        this.cert = cert;
-        validated = null;
-    }
-
-    public String getCerthash() {
-        return certhash;
-    }
-
-    public void setCerthash(String certhash) {
-        this.certhash = certhash;
-        validated = null;
-    }
-
-    public String getClientkey() {
-        return clientkey;
-    }
-
-    public void setClientkey(String clientkey) {
-        this.clientkey = clientkey;
-        validated = null;
-    }
-
-    public String getClientcert() {
-        return clientcert;
-    }
-
-    public void setClientcert(String clientcert) {
-        this.clientcert = clientcert;
-        validated = null;
-    }
-
-    public String getUser() {
-        return user;
-    }
-
-    public void setUser(String user) {
-        this.user = user;
-        this.userWasSet = true;
-        validated = null;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    public void setPassword(String password) {
-        this.password = password;
-        this.passwordWasSet = true;
-        validated = null;
-    }
-
-    public String getLanguage() {
-        return language;
-    }
-
-    public void setLanguage(String language) {
-        this.language = language;
-        validated = null;
-    }
-
-    public boolean isAutocommit() {
-        return autocommit;
-    }
-
-    public void setAutocommit(boolean autocommit) {
-        this.autocommit = autocommit;
-        validated = null;
-    }
-
-    public String getSchema() {
-        return schema;
-    }
-
-    public void setSchema(String schema) {
-        this.schema = schema;
-        validated = null;
-    }
-
-    public int getTimezone() {
-        return timezone;
-    }
-
-    public void setTimezone(int timezone) {
-        this.timezone = timezone;
-        validated = null;
-    }
-
-    public String getBinary() {
-        return binary;
-    }
-
-    public void setBinary(String binary) {
-        this.binary = binary;
-        validated = null;
-    }
-
-    public int getReplySize() {
-        return replySize;
-    }
-
-    public void setReplySize(int replySize) {
-        this.replySize = replySize;
-        validated = null;
-    }
-
-    public String getHash() {
-        return hash;
-    }
-
-    public void setHash(String hash) {
-        this.hash = hash;
-        validated = null;
-    }
-
-    public boolean isDebug() {
-        return debug;
-    }
-
-    public void setDebug(boolean debug) {
-        this.debug = debug;
-        validated = null;
-    }
-
-    public String getLogfile() {
-        return logfile;
-    }
-
-    public void setLogfile(String logfile) {
-        this.logfile = logfile;
-        validated = null;
-    }
-
-    public int getSoTimeout() {
-        return soTimeout;
-    }
-
-
-    public void setSoTimeout(int soTimeout) {
-        this.soTimeout = soTimeout;
-        validated = null;
-    }
-
-    public void setTreatClobAsVarchar(boolean treatClobAsVarchar) {
-        this.treatClobAsVarchar = treatClobAsVarchar;
-        validated = null;
-    }
-
-    public boolean isTreatClobAsVarchar() {
-        return treatClobAsVarchar;
-    }
-
-    public boolean isTreatBlobAsBinary() {
-        return treatBlobAsBinary;
-    }
-
-    public void setTreatBlobAsBinary(boolean treatBlobAsBinary) {
-        this.treatBlobAsBinary = treatBlobAsBinary;
-        validated = null;
-    }
-
-    public Validated validate() throws ValidationError {
-        if (validated == null)
-            validated = new Validated();
-        return validated;
-    }
-
-    public String buildUrl() {
-        final StringBuilder sb = new StringBuilder(128);
-        sb.append("jdbc:");
-        sb.append(tls ? "monetdbs": "monetdb");
-        sb.append("://");
-        sb.append(packHost(host));
-        if (!Parameter.PORT.getDefault().equals(port)) {
-            sb.append(':');
-            sb.append(port);
-        }
-        sb.append('/').append(database);
-        String sep = "?";
-        for (Parameter parm: Parameter.values()) {
-            if (parm.isCore || parm == Parameter.USER || parm == Parameter.PASSWORD)
-                continue;
-            Object defaultValue = parm.getDefault();
-            if (defaultValue == null)
-                continue;
-            Object value = getObject(parm);
-            if (value.equals(defaultValue))
-                continue;
-            sb.append(sep).append(parm.name).append('=');
-            String raw = getString(parm);
-            String encoded = MonetUrlParser.percentEncode(raw);
-            sb.append(encoded);
-            sep = "&";
-        }
-        return sb.toString();
-    }
-
-    public Properties getProperties() {
-        Properties props = new Properties();
-        for (Parameter parm: Parameter.values()) {
-            Object defaultValue = parm.getDefault();
-            if (defaultValue == null || defaultValue.equals(getObject(parm)))
-                continue;
-            String value = getString(parm);
-            if (parm == Parameter.HOST)
-                value = packHost(host);
-            props.setProperty(parm.name, value);
-        }
-
-        return props;
-    }
-
-    public class Validated {
-
-        private final int nbinary;
-
-        Validated() throws ValidationError {
-
-            // 1. The parameters have the types listed in the table in [Section
-            //    Parameters](#parameters).
-
-            String binaryString = binary;
-            int binaryInt;
-            try {
-                binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString);
-            } catch (ValidationError e) {
-                try {
-                    boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString);
-                    binaryInt = b ? 65535 : 0;
-                } catch (ValidationError ee) {
-                    throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off");
-                }
-            }
-            if (binaryInt < 0)
-                throw new ValidationError("binary= cannot be negative");
-            nbinary = binaryInt;
-
-
-            // 2. At least one of **sock** and **host** must be empty.
-            if (!sock.isEmpty() && !host.isEmpty())
-                throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host);
-
-            // 3. The string parameter **binary** must either parse as a boolean or as a
-            //    non-negative integer.
-            //
-            // (checked above)
-
-            // 4. If **sock** is not empty, **tls** must be 'off'.
-            if (!sock.isEmpty() && tls) throw new ValidationError("monetdbs:// cannot be combined with sock=");
-
-            // 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits`
-            //    where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons.
-            // TODO
-            if (!certhash.isEmpty()) {
-                if (!certhash.toLowerCase().startsWith("sha256:"))
-                    throw new ValidationError("certificate hash must start with 'sha256:'");
-                if (!hashPattern.matcher(certhash).matches())
-                    throw new ValidationError("invalid certificate hash");
-            }
-
-            // 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well.
-            if (!tls) {
-                if (!cert.isEmpty() || !certhash.isEmpty())
-                    throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://");
-            }
-
-            // 7. Parameters **database**, **tableschema** and **table** must consist only of
-            //    upper- and lowercase letters, digits, periods, dashes and underscores. They must not
-            //    start with a dash.
-            //    If **table** is not empty, **tableschema** must also not be empty.
-            //    If **tableschema** is not empty, **database** must also not be empty.
-            if (database.isEmpty() && !tableschema.isEmpty())
-                throw new ValidationError("table schema cannot be set without database");
-            if (tableschema.isEmpty() && !table.isEmpty())
-                throw new ValidationError("table cannot be set without schema");
-            if (!database.isEmpty() && !namePattern.matcher(database).matches())
-                throw new ValidationError("invalid database name");
-            if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches())
-                throw new ValidationError("invalid table schema name");
-            if (!table.isEmpty() && !namePattern.matcher(table).matches())
-                throw new ValidationError("invalid table name");
-
-
-            // 8. Parameter **port** must be -1 or in the range 1-65535.
-            if (port < -1 || port == 0 || port > 65535) throw new ValidationError("invalid port number " + port);
-
-            // 9. If **clientcert** is set, **clientkey** must also be set.
-            if (!clientcert.isEmpty() && clientkey.isEmpty())
-                throw new ValidationError("clientcert= is only valid in combination with clientkey=");
-
-            // JDBC specific
-            if (soTimeout < 0)
-                throw new ValidationError("so_timeout= must not be negative");
-        }
-
-        public boolean getTls() {
-            return tls;
-        }
-
-        // Getter is private because you probably want connectTcp() instead
-        private String getHost() {
-            return host;
-        }
-
-        // Getter is private because you probably want connectPort() instead
-        private int getPort() {
-            return port;
-        }
-
-        public String getDatabase() {
-            return database;
-        }
-
-        public String getTableschema() {
-            return tableschema;
-        }
-
-        public String getTable() {
-            return table;
-        }
-
-        // Getter is private because you probably want connectUnix() instead
-        private String getSock() {
-            return sock;
-        }
-
-        public String getSockdir() {
-            return sockdir;
-        }
-
-        public String getCert() {
-            return cert;
-        }
-
-        public String getCerthash() {
-            return certhash;
-        }
-
-        public String getClientkey() {
-            return clientkey;
-        }
-
-        public String getClientcert() {
-            return clientcert;
-        }
-
-        public String getUser() {
-            return user;
-        }
-
-        public String getPassword() {
-            return password;
-        }
-
-        public String getLanguage() {
-            return language;
-        }
-
-        public boolean getAutocommit() {
-            return autocommit;
-        }
-
-        public String getSchema() {
-            return schema;
-        }
-
-        public int getTimezone() {
-            return timezone;
-        }
-
-        // Getter is private because you probably want connectBinary() instead
-        public int getBinary() {
-            return nbinary;
-        }
-
-        public int getReplySize() {
-            return replySize;
-        }
-
-        public String getHash() {
-            return hash;
-        }
-
-        public boolean getDebug() {
-            return debug;
-        }
-
-        public String getLogfile() {
-            return logfile;
-        }
-
-        public int getSoTimeout() {
-            return soTimeout;
-        }
-
-        public boolean isTreatClobAsVarchar() {
-            return treatClobAsVarchar;
-        }
-
-        public boolean isTreatBlobAsBinary() {
-            return treatBlobAsBinary;
-        }
-
-        public boolean connectScan() {
-            if (database.isEmpty()) return false;
-            if (!sock.isEmpty() || !host.isEmpty() || port != -1) return false;
-            return !tls;
-        }
-
-        public int connectPort() {
-            return port == -1 ? 50000 : port;
-        }
-
-        public String connectUnix() {
-            if (!sock.isEmpty()) return sock;
-            if (tls) return "";
-            if (host.isEmpty()) return sockdir + "/.s.monetdb." + connectPort();
-            return "";
-        }
-
-        public String connectTcp() {
-            if (!sock.isEmpty()) return "";
-            if (host.isEmpty()) return "localhost";
-            return host;
-        }
-
-        public Verify connectVerify() {
-            if (!tls) return Verify.None;
-            if (!certhash.isEmpty()) return Verify.Hash;
-            if (!cert.isEmpty()) return Verify.Cert;
-            return Verify.System;
-        }
-
-        public String connectCertHashDigits() {
-            if (!tls) return null;
-            StringBuilder builder = new StringBuilder(certhash.length());
-            for (int i = "sha256:".length(); i < certhash.length(); i++) {
-                char c = certhash.charAt(i);
-                if (Character.digit(c, 16) >= 0) builder.append(Character.toLowerCase(c));
-            }
-            return builder.toString();
-        }
-
-        public int connectBinary() {
-            return nbinary;
-        }
-
-        public String connectClientKey() {
-            return clientkey;
-        }
-
-        public String connectClientCert() {
-            return clientcert.isEmpty() ? clientkey : clientcert;
-        }
-    }
-
-    public enum Verify {
-        None,
-        Cert,
-        Hash,
-        System;
-    }
+	protected static final Target defaults = new Target();
+	private static final Pattern namePattern = Pattern.compile("^[a-zzA-Z_][-a-zA-Z0-9_.]*$");
+	private static final Pattern hashPattern = Pattern.compile("^sha256:[0-9a-fA-F:]*$");
+	private boolean tls = false;
+	private String host = "";
+	private int port = -1;
+	private String database = "";
+	private String tableschema = "";
+	private String table = "";
+	private String sock = "";
+	private String sockdir = "/tmp";
+	private String cert = "";
+	private String certhash = "";
+	private String clientkey = "";
+	private String clientcert = "";
+	private String user = "";
+	private String password = "";
+	private String language = "sql";
+	private boolean autocommit = true;
+	private String schema = "";
+	private int timezone;
+	private String binary = "on";
+	private int replySize = 250;
+	private String hash = "";
+	private boolean debug = false;
+	private String logfile = "";
+	private int soTimeout = 0;
+	private boolean treatClobAsVarchar = true;
+	private boolean treatBlobAsBinary = true;
+	private boolean userWasSet = false;
+	private boolean passwordWasSet = false;
+	private Validated validated = null;
+
+	public Target() {
+		this.timezone = (int) Parameter.TIMEZONE.getDefault();
+	}
+
+	public Target(String url, Properties props) throws URISyntaxException, ValidationError {
+		this();
+		setProperties(props);
+		parseUrl(url);
+	}
+
+	public static String packHost(String host) {
+		switch (host) {
+			case "localhost":
+				return "localhost.";
+			case "":
+				return "localhost";
+			default:
+				return host;
+		}
+	}
+
+	public static String unpackHost(String host) {
+		switch (host) {
+			case "localhost.":
+				return "localhost";
+			case "localhost":
+				return "";
+			default:
+				return host;
+		}
+	}
+
+	public void barrier() {
+		if (userWasSet && !passwordWasSet)
+			password = "";
+		userWasSet = false;
+		passwordWasSet = false;
+	}
+
+	public void setString(String key, String value) throws ValidationError {
+		Parameter parm = Parameter.forName(key);
+		if (parm != null)
+			setString(parm, value);
+		else if (!Parameter.isIgnored(key))
+			throw new ValidationError(key, "unknown parameter");
+	}
+
+	public void setString(Parameter parm, String value) throws ValidationError {
+		if (value == null)
+			throw new NullPointerException("'value' must not be null");
+		assign(parm, parm.type.parse(parm.name, value));
+	}
+
+	public void clear(Parameter parm) {
+		assign(parm, parm.getDefault());
+	}
+
+	public void parseUrl(String url) throws URISyntaxException, ValidationError {
+		if (url == null)
+			return;
+		if (url.startsWith("jdbc:"))
+			url = url.substring(5);
+		if (url.equals("monetdb:")) {
+			return;
+		}
+		MonetUrlParser.parse(this, url);
+	}
+
+	private void assign(Parameter parm, Object value) {
+		switch (parm) {
+			case TLS:
+				setTls((boolean) value);
+				break;
+			case HOST:
+				setHost((String) value);
+				break;
+			case PORT:
+				setPort((int) value);
+				break;
+			case DATABASE:
+				setDatabase((String) value);
+				break;
+			case TABLESCHEMA:
+				setTableschema((String) value);
+				break;
+			case TABLE:
+				setTable((String) value);
+				break;
+			case SOCK:
+				setSock((String) value);
+				break;
+			case SOCKDIR:
+				setSockdir((String) value);
+				break;
+			case CERT:
+				setCert((String) value);
+				break;
+			case CERTHASH:
+				setCerthash((String) value);
+				break;
+			case CLIENTKEY:
+				setClientkey((String) value);
+				break;
+			case CLIENTCERT:
+				setClientcert((String) value);
+				break;
+			case USER:
+				setUser((String) value);
+				break;
+			case PASSWORD:
+				setPassword((String) value);
+				break;
+			case LANGUAGE:
+				setLanguage((String) value);
+				break;
+			case AUTOCOMMIT:
+				setAutocommit((boolean) value);
+				break;
+			case SCHEMA:
+				setSchema((String) value);
+				break;
+			case TIMEZONE:
+				setTimezone((int) value);
+				break;
+			case BINARY:
+				setBinary((String) value);
+				break;
+			case REPLYSIZE:
+				setReplySize((int) value);
+				break;
+			case FETCHSIZE:
+				setReplySize((int) value);
+				break;
+			case HASH:
+				setHash((String) value);
+				break;
+			case DEBUG:
+				setDebug((boolean) value);
+				break;
+			case LOGFILE:
+				setLogfile((String) value);
+				break;
+
+			case SO_TIMEOUT:
+				setSoTimeout((int) value);
+				break;
+			case CLOB_AS_VARCHAR:
+				setTreatClobAsVarchar((boolean) value);
+				break;
+			case BLOB_AS_BINARY:
+				setTreatBlobAsBinary((boolean) value);
+				break;
+
+			default:
+				throw new IllegalStateException("unreachable -- missing case: " + parm.name);
+		}
+	}
+
+	public String getString(Parameter parm) {
+		Object value = getObject(parm);
+		return parm.type.format(value);
+	}
+
+	public Object getObject(Parameter parm) {
+		switch (parm) {
+			case TLS:
+				return tls;
+			case HOST:
+				return host;
+			case PORT:
+				return port;
+			case DATABASE:
+				return database;
+			case TABLESCHEMA:
+				return tableschema;
+			case TABLE:
+				return table;
+			case SOCK:
+				return sock;
+			case SOCKDIR:
+				return sockdir;
+			case CERT:
+				return cert;
+			case CERTHASH:
+				return certhash;
+			case CLIENTKEY:
+				return clientkey;
+			case CLIENTCERT:
+				return clientcert;
+			case USER:
+				return user;
+			case PASSWORD:
+				return password;
+			case LANGUAGE:
+				return language;
+			case AUTOCOMMIT:
+				return autocommit;
+			case SCHEMA:
+				return schema;
+			case TIMEZONE:
+				return timezone;
+			case BINARY:
+				return binary;
+			case REPLYSIZE:
+				return replySize;
+			case FETCHSIZE:
+				return replySize;
+			case HASH:
+				return hash;
+			case DEBUG:
+				return debug;
+			case LOGFILE:
+				return logfile;
+			case SO_TIMEOUT:
+				return soTimeout;
+			case CLOB_AS_VARCHAR:
+				return treatClobAsVarchar;
+			case BLOB_AS_BINARY:
+				return treatBlobAsBinary;
+			default:
+				throw new IllegalStateException("unreachable -- missing case");
+		}
+	}
+
+	public boolean isTls() {
+		return tls;
+	}
+
+	public void setTls(boolean tls) {
+		this.tls = tls;
+		validated = null;
+	}
+
+	public String getHost() {
+		return host;
+	}
+
+	public void setHost(String host) {
+		this.host = host;
+		validated = null;
+	}
+
+	public int getPort() {
+		return port;
+	}
+
+	public void setPort(int port) {
+		this.port = port;
+		validated = null;
+	}
+
+	public String getDatabase() {
+		return database;
+	}
+
+	public void setDatabase(String database) {
+		this.database = database;
+		validated = null;
+	}
+
+	public String getTableschema() {
+		return tableschema;
+	}
+
+	public void setTableschema(String tableschema) {
+		this.tableschema = tableschema;
+		validated = null;
+	}
+
+	public String getTable() {
+		return table;
+	}
+
+	public void setTable(String table) {
+		this.table = table;
+		validated = null;
+	}
+
+	public String getSock() {
+		return sock;
+	}
+
+	public void setSock(String sock) {
+		this.sock = sock;
+		validated = null;
+	}
+
+	public String getSockdir() {
+		return sockdir;
+	}
+
+	public void setSockdir(String sockdir) {
+		this.sockdir = sockdir;
+		validated = null;
+	}
+
+	public String getCert() {
+		return cert;
+	}
+
+	public void setCert(String cert) {
+		this.cert = cert;
+		validated = null;
+	}
+
+	public String getCerthash() {
+		return certhash;
+	}
+
+	public void setCerthash(String certhash) {
+		this.certhash = certhash;
+		validated = null;
+	}
+
+	public String getClientkey() {
+		return clientkey;
+	}
+
+	public void setClientkey(String clientkey) {
+		this.clientkey = clientkey;
+		validated = null;
+	}
+
+	public String getClientcert() {
+		return clientcert;
+	}
+
+	public void setClientcert(String clientcert) {
+		this.clientcert = clientcert;
+		validated = null;
+	}
+
+	public String getUser() {
+		return user;
+	}
+
+	public void setUser(String user) {
+		this.user = user;
+		this.userWasSet = true;
+		validated = null;
+	}
+
+	public String getPassword() {
+		return password;
+	}
+
+	public void setPassword(String password) {
+		this.password = password;
+		this.passwordWasSet = true;
+		validated = null;
+	}
+
+	public String getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(String language) {
+		this.language = language;
+		validated = null;
+	}
+
+	public boolean isAutocommit() {
+		return autocommit;
+	}
+
+	public void setAutocommit(boolean autocommit) {
+		this.autocommit = autocommit;
+		validated = null;
+	}
+
+	public String getSchema() {
+		return schema;
+	}
+
+	public void setSchema(String schema) {
+		this.schema = schema;
+		validated = null;
+	}
+
+	public int getTimezone() {
+		return timezone;
+	}
+
+	public void setTimezone(int timezone) {
+		this.timezone = timezone;
+		validated = null;
+	}
+
+	public String getBinary() {
+		return binary;
+	}
+
+	public void setBinary(String binary) {
+		this.binary = binary;
+		validated = null;
+	}
+
+	public int getReplySize() {
+		return replySize;
+	}
+
+	public void setReplySize(int replySize) {
+		this.replySize = replySize;
+		validated = null;
+	}
+
+	public String getHash() {
+		return hash;
+	}
+
+	public void setHash(String hash) {
+		this.hash = hash;
+		validated = null;
+	}
+
+	public boolean isDebug() {
+		return debug;
+	}
+
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+		validated = null;
+	}
+
+	public String getLogfile() {
+		return logfile;
+	}
+
+	public void setLogfile(String logfile) {
+		this.logfile = logfile;
+		validated = null;
+	}
+
+	public int getSoTimeout() {
+		return soTimeout;
+	}
+
+	public void setSoTimeout(int soTimeout) {
+		this.soTimeout = soTimeout;
+		validated = null;
+	}
+
+	public boolean isTreatClobAsVarchar() {
+		return treatClobAsVarchar;
+	}
+
+	public void setTreatClobAsVarchar(boolean treatClobAsVarchar) {
+		this.treatClobAsVarchar = treatClobAsVarchar;
+		validated = null;
+	}
+
+	public boolean isTreatBlobAsBinary() {
+		return treatBlobAsBinary;
+	}
+
+	public void setTreatBlobAsBinary(boolean treatBlobAsBinary) {
+		this.treatBlobAsBinary = treatBlobAsBinary;
+		validated = null;
+	}
+
+	public Validated validate() throws ValidationError {
+		if (validated == null)
+			validated = new Validated();
+		return validated;
+	}
+
+	public String buildUrl() {
+		final StringBuilder sb = new StringBuilder(128);
+		sb.append("jdbc:");
+		sb.append(tls ? "monetdbs" : "monetdb");
+		sb.append("://");
+		sb.append(packHost(host));
+		if (!Parameter.PORT.getDefault().equals(port)) {
+			sb.append(':');
+			sb.append(port);
+		}
+		sb.append('/').append(database);
+		String sep = "?";
+		for (Parameter parm : Parameter.values()) {
+			if (parm.isCore || parm == Parameter.USER || parm == Parameter.PASSWORD)
+				continue;
+			Object defaultValue = parm.getDefault();
+			if (defaultValue == null)
+				continue;
+			Object value = getObject(parm);
+			if (value.equals(defaultValue))
+				continue;
+			sb.append(sep).append(parm.name).append('=');
+			String raw = getString(parm);
+			String encoded = MonetUrlParser.percentEncode(raw);
+			sb.append(encoded);
+			sep = "&";
+		}
+		return sb.toString();
+	}
+
+	public Properties getProperties() {
+		Properties props = new Properties();
+		for (Parameter parm : Parameter.values()) {
+			Object defaultValue = parm.getDefault();
+			if (defaultValue == null || defaultValue.equals(getObject(parm)))
+				continue;
+			String value = getString(parm);
+			if (parm == Parameter.HOST)
+				value = packHost(host);
+			props.setProperty(parm.name, value);
+		}
+
+		return props;
+	}
+
+	public void setProperties(Properties props) throws ValidationError {
+		if (props != null) {
+			for (String key : props.stringPropertyNames()) {
+				String value = props.getProperty(key);
+				if (key.equals(Parameter.HOST.name))
+					value = Target.unpackHost(value);
+				setString(key, value);
+			}
+		}
+	}
+
+	public enum Verify {
+		None, Cert, Hash, System
+	}
+
+	public class Validated {
+
+		private final int nbinary;
+
+		Validated() throws ValidationError {
+
+			// 1. The parameters have the types listed in the table in [Section
+			//    Parameters](#parameters).
+
+			String binaryString = binary;
+			int binaryInt;
+			try {
+				binaryInt = (int) ParameterType.Int.parse(Parameter.BINARY.name, binaryString);
+			} catch (ValidationError e) {
+				try {
+					boolean b = (boolean) ParameterType.Bool.parse(Parameter.BINARY.name, binaryString);
+					binaryInt = b ? 65535 : 0;
+				} catch (ValidationError ee) {
+					throw new ValidationError("binary= must be either a number or true/yes/on/false/no/off");
+				}
+			}
+			if (binaryInt < 0)
+				throw new ValidationError("binary= cannot be negative");
+			nbinary = binaryInt;
+
+
+			// 2. At least one of **sock** and **host** must be empty.
+			if (!sock.isEmpty() && !host.isEmpty())
+				throw new ValidationError("sock=" + sock + " cannot be combined with host=" + host);
+
+			// 3. The string parameter **binary** must either parse as a boolean or as a
+			//    non-negative integer.
+			//
+			// (checked above)
+
+			// 4. If **sock** is not empty, **tls** must be 'off'.
+			if (!sock.isEmpty() && tls)
+				throw new ValidationError("monetdbs:// cannot be combined with sock=");
+
+			// 5. If **certhash** is not empty, it must be of the form `{sha256}hexdigits`
+			//    where hexdigits is a non-empty sequence of 0-9, a-f, A-F and colons.
+			// TODO
+			if (!certhash.isEmpty()) {
+				if (!certhash.toLowerCase().startsWith("sha256:"))
+					throw new ValidationError("certificate hash must start with 'sha256:'");
+				if (!hashPattern.matcher(certhash).matches())
+					throw new ValidationError("invalid certificate hash");
+			}
+
+			// 6. If **tls** is 'off', **cert** and **certhash** must be 'off' as well.
+			if (!tls) {
+				if (!cert.isEmpty() || !certhash.isEmpty())
+					throw new ValidationError("cert= and certhash= are only allowed in combination with monetdbs://");
+			}
+
+			// 7. Parameters **database**, **tableschema** and **table** must consist only of
+			//    upper- and lowercase letters, digits, periods, dashes and underscores. They must not
+			//    start with a dash.
+			//    If **table** is not empty, **tableschema** must also not be empty.
+			//    If **tableschema** is not empty, **database** must also not be empty.
+			if (database.isEmpty() && !tableschema.isEmpty())
+				throw new ValidationError("table schema cannot be set without database");
+			if (tableschema.isEmpty() && !table.isEmpty())
+				throw new ValidationError("table cannot be set without schema");
+			if (!database.isEmpty() && !namePattern.matcher(database).matches())
+				throw new ValidationError("invalid database name");
+			if (!tableschema.isEmpty() && !namePattern.matcher(tableschema).matches())
+				throw new ValidationError("invalid table schema name");
+			if (!table.isEmpty() && !namePattern.matcher(table).matches())
+				throw new ValidationError("invalid table name");
+
+
+			// 8. Parameter **port** must be -1 or in the range 1-65535.
+			if (port < -1 || port == 0 || port > 65535)
+				throw new ValidationError("invalid port number " + port);
+
+			// 9. If **clientcert** is set, **clientkey** must also be set.
+			if (!clientcert.isEmpty() && clientkey.isEmpty())
+				throw new ValidationError("clientcert= is only valid in combination with clientkey=");
+
+			// JDBC specific
+			if (soTimeout < 0)
+				throw new ValidationError("so_timeout= must not be negative");
+		}
+
+		public boolean getTls() {
+			return tls;
+		}
+
+		// Getter is private because you probably want connectTcp() instead
+		private String getHost() {
+			return host;
+		}
+
+		// Getter is private because you probably want connectPort() instead
+		private int getPort() {
+			return port;
+		}
+
+		public String getDatabase() {
+			return database;
+		}
+
+		public String getTableschema() {
+			return tableschema;
+		}
+
+		public String getTable() {
+			return table;
+		}
+
+		// Getter is private because you probably want connectUnix() instead
+		private String getSock() {
+			return sock;
+		}
+
+		public String getSockdir() {
+			return sockdir;
+		}
+
+		public String getCert() {
+			return cert;
+		}
+
+		public String getCerthash() {
+			return certhash;
+		}
+
+		public String getClientkey() {
+			return clientkey;
+		}
+
+		public String getClientcert() {
+			return clientcert;
+		}
+
+		public String getUser() {
+			return user;
+		}
+
+		public String getPassword() {
+			return password;
+		}
+
+		public String getLanguage() {
+			return language;
+		}
+
+		public boolean getAutocommit() {
+			return autocommit;
+		}
+
+		public String getSchema() {
+			return schema;
+		}
+
+		public int getTimezone() {
+			return timezone;
+		}
+
+		// Getter is private because you probably want connectBinary() instead
+		public int getBinary() {
+			return nbinary;
+		}
+
+		public int getReplySize() {
+			return replySize;
+		}
+
+		public String getHash() {
+			return hash;
+		}
+
+		public boolean getDebug() {
+			return debug;
+		}
+
+		public String getLogfile() {
+			return logfile;
+		}
+
+		public int getSoTimeout() {
+			return soTimeout;
+		}
+
+		public boolean isTreatClobAsVarchar() {
+			return treatClobAsVarchar;
+		}
+
+		public boolean isTreatBlobAsBinary() {
+			return treatBlobAsBinary;
+		}
+
+		public boolean connectScan() {
+			if (database.isEmpty())
+				return false;
+			if (!sock.isEmpty() || !host.isEmpty() || port != -1)
+				return false;
+			return !tls;
+		}
+
+		public int connectPort() {
+			return port == -1 ? 50000 : port;
+		}
+
+		public String connectUnix() {
+			if (!sock.isEmpty())
+				return sock;
+			if (tls)
+				return "";
+			if (host.isEmpty())
+				return sockdir + "/.s.monetdb." + connectPort();
+			return "";
+		}
+
+		public String connectTcp() {
+			if (!sock.isEmpty())
+				return "";
+			if (host.isEmpty())
+				return "localhost";
+			return host;
+		}
+
+		public Verify connectVerify() {
+			if (!tls)
+				return Verify.None;
+			if (!certhash.isEmpty())
+				return Verify.Hash;
+			if (!cert.isEmpty())
+				return Verify.Cert;
+			return Verify.System;
+		}
+
+		public String connectCertHashDigits() {
+			if (!tls)
+				return null;
+			StringBuilder builder = new StringBuilder(certhash.length());
+			for (int i = "sha256:".length(); i < certhash.length(); i++) {
+				char c = certhash.charAt(i);
+				if (Character.digit(c, 16) >= 0)
+					builder.append(Character.toLowerCase(c));
+			}
+			return builder.toString();
+		}
+
+		public int connectBinary() {
+			return nbinary;
+		}
+
+		public String connectClientKey() {
+			return clientkey;
+		}
+
+		public String connectClientCert() {
+			return clientcert.isEmpty() ? clientkey : clientcert;
+		}
+	}
 }
diff --git a/src/main/java/org/monetdb/mcl/net/ValidationError.java b/src/main/java/org/monetdb/mcl/net/ValidationError.java
index bfc60f4..a9741f2 100644
--- a/src/main/java/org/monetdb/mcl/net/ValidationError.java
+++ b/src/main/java/org/monetdb/mcl/net/ValidationError.java
@@ -1,11 +1,11 @@
 package org.monetdb.mcl.net;
 
 public class ValidationError extends Exception {
-    public ValidationError(String parameter, String message) {
-        super(parameter + ": " + message);
-    }
+	public ValidationError(String parameter, String message) {
+		super(parameter + ": " + message);
+	}
 
-    public ValidationError(String message) {
-        super(message);
-    }
+	public ValidationError(String message) {
+		super(message);
+	}
 }
diff --git a/tests/TLSTester.java b/tests/TLSTester.java
index f6298ef..5410042 100644
--- a/tests/TLSTester.java
+++ b/tests/TLSTester.java
@@ -11,307 +11,304 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Properties;
-import java.util.stream.Collectors;
 
 public class TLSTester {
-    int verbose = 0;
-    String serverHost = null;
-    String altHost = null;
-    int serverPort = -1;
-    boolean enableTrusted = false;
-    File tempDir = null;
-    final HashMap fileCache = new HashMap<>();
-    private HashSet preparedButNotRun = new HashSet<>();
-
-    public TLSTester(String[] args) {
-        for (int i = 0; i < args.length; i++) {
-            String arg = args[i];
-            if (arg.equals("-v")) {
-                verbose = 1;
-            } else if (arg.equals("-a")) {
-                altHost = args[++i];
-            } else if (arg.equals("-t")) {
-                enableTrusted = true;
-            } else if (!arg.startsWith("-") && serverHost == null) {
-                int idx = arg.indexOf(':');
-                if (idx > 0) {
-                    serverHost = arg.substring(0, idx);
-                    try {
-                        serverPort = Integer.parseInt(arg.substring(idx + 1));
-                        if (serverPort > 0 && serverPort < 65536)
-                            continue;
-                    } catch (NumberFormatException ignored) {
-                    }
-                }
-                // if we get here it wasn't very valid
-                throw new IllegalArgumentException("Invalid argument: " + arg);
-            } else {
-                throw new IllegalArgumentException("Unexpected argument: " + arg);
-            }
-        }
-    }
-
-    public static void main(String[] args) throws IOException, SQLException, ClassNotFoundException {
-        Class.forName("org.monetdb.jdbc.MonetDriver");
-        TLSTester main = new TLSTester(args);
-        main.run();
-    }
-
-    private HashMap loadPortMap(String testName) throws IOException {
-        HashMap portMap = new HashMap<>();
-        InputStream in = fetchData("/?test=" + testName);
-        BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
-        for (String line = br.readLine(); line != null; line = br.readLine()) {
-            int idx = line.indexOf(':');
-            String service = line.substring(0, idx);
-            int port;
-            try {
-                port = Integer.parseInt(line.substring(idx + 1));
-            } catch (NumberFormatException e) {
-                throw new RuntimeException("Invalid port map line: " + line);
-            }
-            portMap.put(service, port);
-        }
-        return portMap;
-    }
-
-    private File resource(String resource) throws IOException {
-        if (!fileCache.containsKey(resource))
-            fetchResource(resource);
-        return fileCache.get(resource);
-    }
-
-    private void fetchResource(String resource) throws IOException {
-        if (!resource.startsWith("/")) {
-            throw new IllegalArgumentException("Resource must start with slash: " + resource);
-        }
-        if (tempDir == null) {
-            tempDir = Files.createTempDirectory("tlstest").toFile();
-            tempDir.deleteOnExit();
-        }
-        File outPath = new File(tempDir, resource.substring(1));
-        try (InputStream in = fetchData(resource); FileOutputStream out = new FileOutputStream(outPath)) {
-            byte[] buffer = new byte[12];
-            while (true) {
-                int n = in.read(buffer);
-                if (n <= 0)
-                    break;
-                out.write(buffer, 0, n);
-            }
-        }
-        fileCache.put(resource, outPath);
-    }
-
-    private byte[] fetchBytes(String resource) throws IOException {
-        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        try (InputStream in = fetchData(resource)) {
-            byte[] buffer = new byte[22];
-            while (true) {
-                int nread = in.read(buffer);
-                if (nread <= 0)
-                    break;
-                out.write(buffer, 0, nread);
-            }
-            return out.toByteArray();
-        }
-    }
-
-    private InputStream fetchData(String resource) throws IOException {
-        URL url = new URL("http://" + serverHost + ":" + serverPort + resource);
-        URLConnection conn = url.openConnection();
-        conn.connect();
-        return conn.getInputStream();
-    }
-
-    private void run() throws IOException, SQLException {
-        test_connect_plain();
-        test_connect_tls();
-        test_refuse_no_cert();
-        test_refuse_wrong_cert();
-        test_refuse_wrong_host();
-        test_refuse_tlsv12();
-        test_refuse_expired();
+	final HashMap fileCache = new HashMap<>();
+	int verbose = 0;
+	String serverHost = null;
+	String altHost = null;
+	int serverPort = -1;
+	boolean enableTrusted = false;
+	File tempDir = null;
+	private final HashSet preparedButNotRun = new HashSet<>();
+
+	public TLSTester(String[] args) {
+		for (int i = 0; i < args.length; i++) {
+			String arg = args[i];
+			if (arg.equals("-v")) {
+				verbose = 1;
+			} else if (arg.equals("-a")) {
+				altHost = args[++i];
+			} else if (arg.equals("-t")) {
+				enableTrusted = true;
+			} else if (!arg.startsWith("-") && serverHost == null) {
+				int idx = arg.indexOf(':');
+				if (idx > 0) {
+					serverHost = arg.substring(0, idx);
+					try {
+						serverPort = Integer.parseInt(arg.substring(idx + 1));
+						if (serverPort > 0 && serverPort < 65536)
+							continue;
+					} catch (NumberFormatException ignored) {
+					}
+				}
+				// if we get here it wasn't very valid
+				throw new IllegalArgumentException("Invalid argument: " + arg);
+			} else {
+				throw new IllegalArgumentException("Unexpected argument: " + arg);
+			}
+		}
+	}
+
+	public static void main(String[] args) throws IOException, SQLException, ClassNotFoundException {
+		Class.forName("org.monetdb.jdbc.MonetDriver");
+		TLSTester main = new TLSTester(args);
+		main.run();
+	}
+
+	private HashMap loadPortMap(String testName) throws IOException {
+		HashMap portMap = new HashMap<>();
+		InputStream in = fetchData("/?test=" + testName);
+		BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
+		for (String line = br.readLine(); line != null; line = br.readLine()) {
+			int idx = line.indexOf(':');
+			String service = line.substring(0, idx);
+			int port;
+			try {
+				port = Integer.parseInt(line.substring(idx + 1));
+			} catch (NumberFormatException e) {
+				throw new RuntimeException("Invalid port map line: " + line);
+			}
+			portMap.put(service, port);
+		}
+		return portMap;
+	}
+
+	private File resource(String resource) throws IOException {
+		if (!fileCache.containsKey(resource))
+			fetchResource(resource);
+		return fileCache.get(resource);
+	}
+
+	private void fetchResource(String resource) throws IOException {
+		if (!resource.startsWith("/")) {
+			throw new IllegalArgumentException("Resource must start with slash: " + resource);
+		}
+		if (tempDir == null) {
+			tempDir = Files.createTempDirectory("tlstest").toFile();
+			tempDir.deleteOnExit();
+		}
+		File outPath = new File(tempDir, resource.substring(1));
+		try (InputStream in = fetchData(resource); FileOutputStream out = new FileOutputStream(outPath)) {
+			byte[] buffer = new byte[12];
+			while (true) {
+				int n = in.read(buffer);
+				if (n <= 0)
+					break;
+				out.write(buffer, 0, n);
+			}
+		}
+		fileCache.put(resource, outPath);
+	}
+
+	private byte[] fetchBytes(String resource) throws IOException {
+		ByteArrayOutputStream out = new ByteArrayOutputStream();
+		try (InputStream in = fetchData(resource)) {
+			byte[] buffer = new byte[22];
+			while (true) {
+				int nread = in.read(buffer);
+				if (nread <= 0)
+					break;
+				out.write(buffer, 0, nread);
+			}
+			return out.toByteArray();
+		}
+	}
+
+	private InputStream fetchData(String resource) throws IOException {
+		URL url = new URL("http://" + serverHost + ":" + serverPort + resource);
+		URLConnection conn = url.openConnection();
+		conn.connect();
+		return conn.getInputStream();
+	}
+
+	private void run() throws IOException, SQLException {
+		test_connect_plain();
+		test_connect_tls();
+		test_refuse_no_cert();
+		test_refuse_wrong_cert();
+		test_refuse_wrong_host();
+		test_refuse_tlsv12();
+		test_refuse_expired();
 //        test_connect_client_auth1();
 //        test_connect_client_auth2();
-        test_fail_tls_to_plain();
-        test_fail_plain_to_tls();
-        test_connect_server_name();
-        test_connect_alpn_mapi9();
-        test_connect_trusted();
-        test_refuse_trusted_wrong_host();
-
-        // did we forget to call expectSucceed and expectFailure somewhere?
-        if (!preparedButNotRun.isEmpty()) {
-            String names = String.join(", ", preparedButNotRun);
-            throw new RuntimeException("Not all tests called expectSuccess/expectFailure: " + names);
-        }
-    }
-
-    private void test_connect_plain() throws IOException, SQLException {
-        attempt("connect_plain", "plain").with(Parameter.TLS, false).expectSuccess();
-    }
-
-    private void test_connect_tls() throws IOException, SQLException {
-        Attempt attempt = attempt("connect_tls", "server1");
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
-    }
-
-    private void test_refuse_no_cert() throws IOException, SQLException {
-        attempt("refuse_no_cert", "server1").expectFailure("PKIX path building failed");
-    }
-
-    private void test_refuse_wrong_cert() throws IOException, SQLException {
-        Attempt attempt = attempt("refuse_wrong_cert", "server1");
-        attempt.withFile(Parameter.CERT, "/ca2.crt").expectFailure("PKIX path building failed");
-    }
-
-    private void test_refuse_wrong_host() throws IOException, SQLException {
-        Attempt attempt = attempt("refuse_wrong_host", "server1").with(Parameter.HOST, altHost);
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("No subject alternative DNS name");
-    }
-
-    private void test_refuse_tlsv12() throws IOException, SQLException {
-        Attempt attempt = attempt("refuse_tlsv12", "tls12");
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("protocol_version");
-    }
-
-    private void test_refuse_expired() throws IOException, SQLException {
-        Attempt attempt = attempt("refuse_expired", "expiredcert");
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("PKIX path validation failed");
-    }
-
-    private void test_connect_client_auth1() throws IOException, SQLException {
-        attempt("connect_client_auth1", "clientauth")
-                .withFile(Parameter.CERT, "/ca1.crt")
-                .withFile(Parameter.CLIENTKEY, "/client2.keycrt")
-                .expectSuccess();
-    }
-
-    private void test_connect_client_auth2() throws IOException, SQLException {
-        attempt("connect_client_auth2", "clientauth")
-                .withFile(Parameter.CERT, "/ca1.crt")
-                .withFile(Parameter.CLIENTKEY, "/client2.key")
-                .withFile(Parameter.CLIENTCERT, "/client2.crt")
-                .expectSuccess();
-    }
-
-    private void test_fail_tls_to_plain() throws IOException, SQLException {
-        Attempt attempt = attempt("fail_tls_to_plain", "plain");
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("");
-
-    }
-
-    private void test_fail_plain_to_tls() throws IOException, SQLException {
-        attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect");
-    }
-
-    private void test_connect_server_name() throws IOException, SQLException {
-        Attempt attempt = attempt("connect_server_name", "sni");
-        attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
-    }
-
-    private void test_connect_alpn_mapi9() throws IOException, SQLException {
-        attempt("connect_alpn_mapi9", "alpn_mapi9")
-                .withFile(Parameter.CERT, "/ca1.crt")
-                .expectSuccess();
-    }
-
-    private void test_connect_trusted() throws IOException, SQLException {
-        attempt("connect_trusted", null)
-                .with(Parameter.HOST, "monetdb.ergates.nl")
-                .with(Parameter.PORT, 50000)
-                .expectSuccess();
-    }
-
-    private void test_refuse_trusted_wrong_host() throws IOException, SQLException {
-        attempt("test_refuse_trusted_wrong_host", null)
-                .with(Parameter.HOST, "monetdbxyz.ergates.nl")
-                .with(Parameter.PORT, 50000)
-                .expectFailure("No subject alternative DNS name");
-    }
-
-    private Attempt attempt(String testName, String portName) throws IOException {
-        preparedButNotRun.add(testName);
-        return new Attempt(testName, portName);
-    }
-
-    private class Attempt {
-        private final String testName;
-        private final Properties props = new Properties();
-        boolean disabled = false;
-
-        public Attempt(String testName, String portName) throws IOException {
-            HashMap portMap = loadPortMap(testName);
-
-            this.testName = testName;
-            with(Parameter.TLS, true);
-            with(Parameter.HOST, serverHost);
-            with(Parameter.SO_TIMEOUT, 3000);
-            if (portName != null) {
-                Integer port = portMap.get(portName);
-                if (port != null) {
-                    with(Parameter.PORT, port);
-                } else {
-                    throw new RuntimeException("Unknown port name: " + portName);
-                }
-            }
-        }
-
-        private Attempt with(Parameter parm, String value) {
-            props.setProperty(parm.name, value);
-            return this;
-        }
-
-        private Attempt with(Parameter parm, int value) {
-            props.setProperty(parm.name, Integer.toString(value));
-            return this;
-        }
-
-        private Attempt with(Parameter parm, boolean value) {
-            props.setProperty(parm.name, value ? "true" : "false");
-            return this;
-        }
-
-        private Attempt withFile(Parameter parm, String certResource) throws IOException {
-            File certFile = resource(certResource);
-            String path = certFile.getPath();
-            with(parm, path);
-            return this;
-        }
-
-        public void expectSuccess() throws SQLException {
-            preparedButNotRun.remove(testName);
-            if (disabled)
-                return;
-            try {
-                Connection conn = DriverManager.getConnection("jdbc:monetdb:", props);
-                conn.close();
-            } catch (SQLException e) {
-                if (e.getMessage().startsWith("Sorry, this is not a real MonetDB instance")) {
-                    // it looks like a failure but this is actually our success scenario
-                    // because this is what the TLS Tester does when the connection succeeds.
-                    return;
-                }
-                // other exceptions ARE errors and should be reported.
-                throw e;
-            }
-        }
-
-        public void expectFailure(String... expectedMessages) throws SQLException {
-            if (disabled)
-                return;
-            try {
-                expectSuccess();
-                throw new RuntimeException("Expected test " + testName + " to throw an exception but it didn't");
-            } catch (SQLException e) {
-                for (String expected : expectedMessages)
-                    if (e.getMessage().contains(expected))
-                        return;
-                String message = "Test " + testName + " threw the wrong exception: " + e.getMessage() + '\n' + "Expected:\n        <" + String.join(">\n        <", expectedMessages) + ">";
-                throw new RuntimeException(message);
-
-            }
-        }
-
-    }
+		test_fail_tls_to_plain();
+		test_fail_plain_to_tls();
+		test_connect_server_name();
+		test_connect_alpn_mapi9();
+		test_connect_trusted();
+		test_refuse_trusted_wrong_host();
+
+		// did we forget to call expectSucceed and expectFailure somewhere?
+		if (!preparedButNotRun.isEmpty()) {
+			String names = String.join(", ", preparedButNotRun);
+			throw new RuntimeException("Not all tests called expectSuccess/expectFailure: " + names);
+		}
+	}
+
+	private void test_connect_plain() throws IOException, SQLException {
+		attempt("connect_plain", "plain").with(Parameter.TLS, false).expectSuccess();
+	}
+
+	private void test_connect_tls() throws IOException, SQLException {
+		Attempt attempt = attempt("connect_tls", "server1");
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
+	}
+
+	private void test_refuse_no_cert() throws IOException, SQLException {
+		attempt("refuse_no_cert", "server1").expectFailure("PKIX path building failed");
+	}
+
+	private void test_refuse_wrong_cert() throws IOException, SQLException {
+		Attempt attempt = attempt("refuse_wrong_cert", "server1");
+		attempt.withFile(Parameter.CERT, "/ca2.crt").expectFailure("PKIX path building failed");
+	}
+
+	private void test_refuse_wrong_host() throws IOException, SQLException {
+		Attempt attempt = attempt("refuse_wrong_host", "server1").with(Parameter.HOST, altHost);
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("No subject alternative DNS name");
+	}
+
+	private void test_refuse_tlsv12() throws IOException, SQLException {
+		Attempt attempt = attempt("refuse_tlsv12", "tls12");
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("protocol_version");
+	}
+
+	private void test_refuse_expired() throws IOException, SQLException {
+		Attempt attempt = attempt("refuse_expired", "expiredcert");
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("PKIX path validation failed");
+	}
+
+	private void test_connect_client_auth1() throws IOException, SQLException {
+		attempt("connect_client_auth1", "clientauth")
+				.withFile(Parameter.CERT, "/ca1.crt")
+				.withFile(Parameter.CLIENTKEY, "/client2.keycrt")
+				.expectSuccess();
+	}
+
+	private void test_connect_client_auth2() throws IOException, SQLException {
+		attempt("connect_client_auth2", "clientauth")
+				.withFile(Parameter.CERT, "/ca1.crt")
+				.withFile(Parameter.CLIENTKEY, "/client2.key")
+				.withFile(Parameter.CLIENTCERT, "/client2.crt")
+				.expectSuccess();
+	}
+
+	private void test_fail_tls_to_plain() throws IOException, SQLException {
+		Attempt attempt = attempt("fail_tls_to_plain", "plain");
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectFailure("");
+
+	}
+
+	private void test_fail_plain_to_tls() throws IOException, SQLException {
+		attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect");
+	}
+
+	private void test_connect_server_name() throws IOException, SQLException {
+		Attempt attempt = attempt("connect_server_name", "sni");
+		attempt.withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
+	}
+
+	private void test_connect_alpn_mapi9() throws IOException, SQLException {
+		attempt("connect_alpn_mapi9", "alpn_mapi9").withFile(Parameter.CERT, "/ca1.crt").expectSuccess();
+	}
+
+	private void test_connect_trusted() throws IOException, SQLException {
+		attempt("connect_trusted", null)
+				.with(Parameter.HOST, "monetdb.ergates.nl")
+				.with(Parameter.PORT, 50000)
+				.expectSuccess();
+	}
+
+	private void test_refuse_trusted_wrong_host() throws IOException, SQLException {
+		attempt("test_refuse_trusted_wrong_host", null)
+				.with(Parameter.HOST, "monetdbxyz.ergates.nl")
+				.with(Parameter.PORT, 50000)
+				.expectFailure("No subject alternative DNS name");
+	}
+
+	private Attempt attempt(String testName, String portName) throws IOException {
+		preparedButNotRun.add(testName);
+		return new Attempt(testName, portName);
+	}
+
+	private class Attempt {
+		private final String testName;
+		private final Properties props = new Properties();
+		boolean disabled = false;
+
+		public Attempt(String testName, String portName) throws IOException {
+			HashMap portMap = loadPortMap(testName);
+
+			this.testName = testName;
+			with(Parameter.TLS, true);
+			with(Parameter.HOST, serverHost);
+			with(Parameter.SO_TIMEOUT, 3000);
+			if (portName != null) {
+				Integer port = portMap.get(portName);
+				if (port != null) {
+					with(Parameter.PORT, port);
+				} else {
+					throw new RuntimeException("Unknown port name: " + portName);
+				}
+			}
+		}
+
+		private Attempt with(Parameter parm, String value) {
+			props.setProperty(parm.name, value);
+			return this;
+		}
+
+		private Attempt with(Parameter parm, int value) {
+			props.setProperty(parm.name, Integer.toString(value));
+			return this;
+		}
+
+		private Attempt with(Parameter parm, boolean value) {
+			props.setProperty(parm.name, value ? "true" : "false");
+			return this;
+		}
+
+		private Attempt withFile(Parameter parm, String certResource) throws IOException {
+			File certFile = resource(certResource);
+			String path = certFile.getPath();
+			with(parm, path);
+			return this;
+		}
+
+		public void expectSuccess() throws SQLException {
+			preparedButNotRun.remove(testName);
+			if (disabled)
+				return;
+			try {
+				Connection conn = DriverManager.getConnection("jdbc:monetdb:", props);
+				conn.close();
+			} catch (SQLException e) {
+				if (e.getMessage().startsWith("Sorry, this is not a real MonetDB instance")) {
+					// it looks like a failure but this is actually our success scenario
+					// because this is what the TLS Tester does when the connection succeeds.
+					return;
+				}
+				// other exceptions ARE errors and should be reported.
+				throw e;
+			}
+		}
+
+		public void expectFailure(String... expectedMessages) throws SQLException {
+			if (disabled)
+				return;
+			try {
+				expectSuccess();
+				throw new RuntimeException("Expected test " + testName + " to throw an exception but it didn't");
+			} catch (SQLException e) {
+				for (String expected : expectedMessages)
+					if (e.getMessage().contains(expected))
+						return;
+				String message = "Test " + testName + " threw the wrong exception: " + e.getMessage() + '\n' + "Expected:\n        <" + String.join(">\n        <", expectedMessages) + ">";
+				throw new RuntimeException(message);
+
+			}
+		}
+
+	}
 }
diff --git a/tests/UrlTester.java b/tests/UrlTester.java
index 3be4f4f..e68031e 100644
--- a/tests/UrlTester.java
+++ b/tests/UrlTester.java
@@ -5,410 +5,410 @@
 import java.util.ArrayList;
 
 public class UrlTester {
-    final String filename;
-    final int verbose;
-    final BufferedReader reader;
-    int lineno = 0;
-    int testCount = 0;
-    Target target = null;
-    Target.Validated validated = null;
-
-    public UrlTester(String filename, BufferedReader reader, int verbose) {
-        this.filename = filename;
-        this.verbose = verbose;
-        this.reader = reader;
-    }
-
-    public UrlTester(String filename, int verbose) throws IOException {
-        this.filename = filename;
-        this.verbose = verbose;
-        this.reader = new BufferedReader(new FileReader(filename));
-    }
-
-    public static void main(String[] args) throws IOException {
-        ArrayList filenames = new ArrayList<>();
-        int verbose = 0;
-        for (String arg : args) {
-            switch (arg) {
-                case "-vvv":
-                    verbose++;
-                case "-vv":
-                    verbose++;
-                case "-v":
-                    verbose++;
-                    break;
-                case "-h":
-                case "--help":
-                    exitUsage(null);
-                    break;
-                default:
-                    if (!arg.startsWith("-")) {
-                        filenames.add(arg);
-                    } else {
-                        exitUsage("Unexpected argument: " + arg);
-                    }
-                    break;
-            }
-        }
-
-        runUnitTests();
-
-        try {
-            if (filenames.isEmpty()) {
-                runAllTests();
-            } else {
-                for (String filename : filenames) {
-                    new UrlTester(filename, verbose).run();
-                }
-            }
-        } catch (Failure e) {
-            System.err.println("Test failed: " + e.getMessage());
-            System.exit(1);
-        }
-    }
-
-    private static void exitUsage(String message) {
-        if (message != null) {
-            System.err.println(message);
-        }
-        System.err.println("Usage: UrlTester OPTIONS [FILENAME..]");
-        System.err.println("Options:");
-        System.err.println("   -v        Be more verbose");
-        System.err.println("   -h --help Show this help");
-        int status = message == null ? 0 : 1;
-        System.exit(status);
-    }
-
-    public static UrlTester forResource(String resourceName, int verbose) throws FileNotFoundException {
-        InputStream stream = UrlTester.class.getResourceAsStream(resourceName);
-        if (stream == null) {
-            throw new FileNotFoundException("Resource " + resourceName);
-        }
-        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
-        return new UrlTester(resourceName, reader, verbose);
-    }
-
-    public static void runAllTests() throws IOException, Failure {
-        runUnitTests();
-        UrlTester.forResource("/tests.md", 0).run();
-        UrlTester.forResource("/javaspecific.md", 0).run();
-    }
-
-    public static void runUnitTests() {
-        testDefaults();
-        testParameters();
-    }
-
-    private static void testDefaults() {
-        Target target = new Target();
-
-        for (Parameter parm : Parameter.values()) {
-            Object expected = parm.getDefault();
-            if (expected == null)
-                continue;
-            Object actual = target.getObject(parm);
-            if (!expected.equals(actual)) {
-                throw new RuntimeException("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">");
-            }
-        }
-    }
-
-    private static void testParameters() {
-        for (Parameter parm : Parameter.values()) {
-            Parameter found = Parameter.forName(parm.name);
-            if (parm != found) {
-                String foundStr = found != null ? found.name : "null";
-                throw new RuntimeException("Looking up <" + parm.name + ">, found <" + foundStr);
-            }
-        }
-    }
-
-    public void run() throws Failure, IOException {
-        try {
-            processFile();
-        } catch (Failure e) {
-            if (e.getFilename() == null) {
-                e.setFilename(filename);
-                e.setLineno(lineno);
-                throw e;
-            }
-        }
-    }
-
-    private void processFile() throws IOException, Failure {
-        while (true) {
-            String line = reader.readLine();
-            if (line == null)
-                break;
-            lineno++;
-            processLine(line);
-        }
-        if (verbose >= 1) {
-            System.out.println();
-            System.out.println("Ran " + testCount + " tests in " + lineno + " lines");
-        }
-    }
-
-    private void processLine(String line) throws Failure {
-        line = line.replaceFirst("\\s+$", ""); // remove trailing
-        if (target == null && line.equals("```test")) {
-            if (verbose >= 2) {
-                if (testCount > 0) {
-                    System.out.println();
-                }
-                System.out.println("\u25B6 " + filename + ":" + lineno);
-            }
-            target = new Target();
-            testCount++;
-            return;
-        }
-        if (target != null) {
-            if (line.equals("```")) {
-                stopProcessing();
-                return;
-            }
-            handleCommand(line);
-        }
-    }
-
-    private void stopProcessing() {
-        target = null;
-        validated = null;
-    }
-
-    private void handleCommand(String line) throws Failure {
-        if (verbose >= 3) {
-            System.out.println(line);
-        }
-        if (line.isEmpty())
-            return;
-
-        String[] parts = line.split("\\s+", 2);
-        String command = parts[0];
-        switch (command.toUpperCase()) {
-            case "ONLY":
-                handleOnly(true, parts[1]);
-                return;
-            case "NOT":
-                handleOnly(false, parts[1]);
-                return;
-            case "PARSE":
-                handleParse(parts[1], null);
-                return;
-            case "ACCEPT":
-                handleParse(parts[1], true);
-                return;
-            case "REJECT":
-                handleParse(parts[1], false);
-                return;
-            case "SET":
-                handleSet(parts[1]);
-                return;
-            case "EXPECT":
-                handleExpect(parts[1]);
-                return;
-            default:
-                throw new Failure("Unexpected command: " + command);
-        }
-
-    }
-
-    private void handleOnly(boolean mustBePresent, String rest) throws Failure {
-        boolean found = false;
-        for (String part : rest.split("\\s+")) {
-            if (part.equals("jdbc")) {
-                found = true;
-                break;
-            }
-        }
-        if (found != mustBePresent) {
-            // do not further process this block
-            stopProcessing();
-        }
-    }
-
-    private int findEqualSign(String rest) throws Failure {
-        int index = rest.indexOf('=');
-        if (index < -1)
-            throw new Failure("Expected to find a '='");
-        return index;
-    }
-
-    private String splitKey(String rest) throws Failure {
-        int index = findEqualSign(rest);
-        return rest.substring(0, index);
-    }
-
-    private String splitValue(String rest) throws Failure {
-        int index = findEqualSign(rest);
-        return rest.substring(index + 1);
-    }
-
-    private void handleSet(String rest) throws Failure {
-        validated = null;
-        String key = splitKey(rest);
-        String value = splitValue(rest);
-
-        try {
-            target.setString(key, value);
-        } catch (ValidationError e) {
-            throw new Failure(e.getMessage());
-        }
-    }
-
-    private void handleParse(String rest, Boolean shouldSucceed) throws Failure {
-        URISyntaxException parseError = null;
-        ValidationError validationError = null;
-
-        validated = null;
-        try {
-            target.barrier();
-            MonetUrlParser.parse(target, rest);
-        } catch (URISyntaxException e) {
-            parseError = e;
-        } catch (ValidationError e) {
-            validationError = e;
-        }
-
-        if (parseError == null && validationError == null) {
-            try {
-                tryValidate();
-            } catch (ValidationError e) {
-                validationError = e;
-            }
-        }
-
-        if (shouldSucceed == Boolean.FALSE) {
-            if (parseError != null || validationError != null)
-                return; // happy
-            else
-                throw new Failure("URL unexpectedly parsed and validated");
-        }
-
-        if (parseError != null)
-            throw new Failure("Parse error: " + parseError);
-        if (validationError != null && shouldSucceed == Boolean.TRUE)
-            throw new Failure("Validation error: " + validationError);
-    }
-
-    private void handleExpect(String rest) throws Failure {
-        String key = splitKey(rest);
-        String expectedString = splitValue(rest);
-
-        Object actual = null;
-        try {
-            actual = extract(key);
-        } catch (ValidationError e) {
-            throw new Failure(e.getMessage());
-        }
-
-        Object expected;
-        try {
-            if (actual instanceof Boolean)
-                expected = ParameterType.Bool.parse(key, expectedString);
-            else if (actual instanceof Integer)
-                expected = ParameterType.Int.parse(key, expectedString);
-            else
-                expected = expectedString;
-        } catch (ValidationError e) {
-            String typ = actual.getClass().getName();
-            throw new Failure("Cannot convert expected value <" + expectedString + "> to " + typ + ": " + e.getMessage());
-        }
-
-        if (actual.equals(expected))
-            return;
-        throw new Failure("Expected " + key + "=<" + expectedString + ">, found <" + actual + ">");
-    }
-
-    private Target.Validated tryValidate() throws ValidationError {
-        if (validated == null)
-            validated = target.validate();
-        return validated;
-    }
-
-    private Object extract(String key) throws ValidationError, Failure {
-        switch (key) {
-            case "valid":
-                try {
-                    tryValidate();
-                } catch (ValidationError e) {
-                    return Boolean.FALSE;
-                }
-                return Boolean.TRUE;
-
-            case "connect_scan":
-                return tryValidate().connectScan();
-            case "connect_port":
-                return tryValidate().connectPort();
-            case "connect_unix":
-                return tryValidate().connectUnix();
-            case "connect_tcp":
-                return tryValidate().connectTcp();
-            case "connect_tls_verify":
-                switch (tryValidate().connectVerify()) {
-                    case None:
-                        return "";
-                    case Cert:
-                        return "cert";
-                    case Hash:
-                        return "hash";
-                    case System:
-                        return "system";
-                    default:
-                        throw new IllegalStateException("unreachable");
-                }
-            case "connect_certhash_digits":
-                return tryValidate().connectCertHashDigits();
-            case "connect_binary":
-                return tryValidate().connectBinary();
-            case "connect_clientkey":
-                return tryValidate().connectClientKey();
-            case "connect_clientcert":
-                return tryValidate().connectClientCert();
-
-            default:
-                Parameter parm = Parameter.forName(key);
-                if (parm != null)
-                    return target.getObject(parm);
-                else
-                    throw new Failure("Unknown attribute: " + key);
-        }
-    }
-
-    public static class Failure extends Exception {
-        private String filename = null;
-        private int lineno = -1;
-
-        public Failure(String message) {
-            super(message);
-        }
-
-        @Override
-        public String getMessage() {
-            StringBuilder buffer = new StringBuilder();
-            if (filename != null) {
-                buffer.append(filename).append(":");
-                if (lineno >= 0)
-                    buffer.append(lineno).append(":");
-            }
-            buffer.append(super.getMessage());
-            return buffer.toString();
-        }
-
-        public String getFilename() {
-            return filename;
-        }
-
-        public void setFilename(String filename) {
-            this.filename = filename;
-        }
-
-        public int getLineno() {
-            return lineno;
-        }
-
-        public void setLineno(int lineno) {
-            this.lineno = lineno;
-        }
-    }
+	final String filename;
+	final int verbose;
+	final BufferedReader reader;
+	int lineno = 0;
+	int testCount = 0;
+	Target target = null;
+	Target.Validated validated = null;
+
+	public UrlTester(String filename, BufferedReader reader, int verbose) {
+		this.filename = filename;
+		this.verbose = verbose;
+		this.reader = reader;
+	}
+
+	public UrlTester(String filename, int verbose) throws IOException {
+		this.filename = filename;
+		this.verbose = verbose;
+		this.reader = new BufferedReader(new FileReader(filename));
+	}
+
+	public static void main(String[] args) throws IOException {
+		ArrayList filenames = new ArrayList<>();
+		int verbose = 0;
+		for (String arg : args) {
+			switch (arg) {
+				case "-vvv":
+					verbose++;
+				case "-vv":
+					verbose++;
+				case "-v":
+					verbose++;
+					break;
+				case "-h":
+				case "--help":
+					exitUsage(null);
+					break;
+				default:
+					if (!arg.startsWith("-")) {
+						filenames.add(arg);
+					} else {
+						exitUsage("Unexpected argument: " + arg);
+					}
+					break;
+			}
+		}
+
+		runUnitTests();
+
+		try {
+			if (filenames.isEmpty()) {
+				runAllTests();
+			} else {
+				for (String filename : filenames) {
+					new UrlTester(filename, verbose).run();
+				}
+			}
+		} catch (Failure e) {
+			System.err.println("Test failed: " + e.getMessage());
+			System.exit(1);
+		}
+	}
+
+	private static void exitUsage(String message) {
+		if (message != null) {
+			System.err.println(message);
+		}
+		System.err.println("Usage: UrlTester OPTIONS [FILENAME..]");
+		System.err.println("Options:");
+		System.err.println("   -v        Be more verbose");
+		System.err.println("   -h --help Show this help");
+		int status = message == null ? 0 : 1;
+		System.exit(status);
+	}
+
+	public static UrlTester forResource(String resourceName, int verbose) throws FileNotFoundException {
+		InputStream stream = UrlTester.class.getResourceAsStream(resourceName);
+		if (stream == null) {
+			throw new FileNotFoundException("Resource " + resourceName);
+		}
+		BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+		return new UrlTester(resourceName, reader, verbose);
+	}
+
+	public static void runAllTests() throws IOException, Failure {
+		runUnitTests();
+		UrlTester.forResource("/tests.md", 0).run();
+		UrlTester.forResource("/javaspecific.md", 0).run();
+	}
+
+	public static void runUnitTests() {
+		testDefaults();
+		testParameters();
+	}
+
+	private static void testDefaults() {
+		Target target = new Target();
+
+		for (Parameter parm : Parameter.values()) {
+			Object expected = parm.getDefault();
+			if (expected == null)
+				continue;
+			Object actual = target.getObject(parm);
+			if (!expected.equals(actual)) {
+				throw new RuntimeException("Default for " + parm.name + " expected to be <" + expected + "> but is <" + actual + ">");
+			}
+		}
+	}
+
+	private static void testParameters() {
+		for (Parameter parm : Parameter.values()) {
+			Parameter found = Parameter.forName(parm.name);
+			if (parm != found) {
+				String foundStr = found != null ? found.name : "null";
+				throw new RuntimeException("Looking up <" + parm.name + ">, found <" + foundStr);
+			}
+		}
+	}
+
+	public void run() throws Failure, IOException {
+		try {
+			processFile();
+		} catch (Failure e) {
+			if (e.getFilename() == null) {
+				e.setFilename(filename);
+				e.setLineno(lineno);
+				throw e;
+			}
+		}
+	}
+
+	private void processFile() throws IOException, Failure {
+		while (true) {
+			String line = reader.readLine();
+			if (line == null)
+				break;
+			lineno++;
+			processLine(line);
+		}
+		if (verbose >= 1) {
+			System.out.println();
+			System.out.println("Ran " + testCount + " tests in " + lineno + " lines");
+		}
+	}
+
+	private void processLine(String line) throws Failure {
+		line = line.replaceFirst("\\s+$", ""); // remove trailing
+		if (target == null && line.equals("```test")) {
+			if (verbose >= 2) {
+				if (testCount > 0) {
+					System.out.println();
+				}
+				System.out.println("\u25B6 " + filename + ":" + lineno);
+			}
+			target = new Target();
+			testCount++;
+			return;
+		}
+		if (target != null) {
+			if (line.equals("```")) {
+				stopProcessing();
+				return;
+			}
+			handleCommand(line);
+		}
+	}
+
+	private void stopProcessing() {
+		target = null;
+		validated = null;
+	}
+
+	private void handleCommand(String line) throws Failure {
+		if (verbose >= 3) {
+			System.out.println(line);
+		}
+		if (line.isEmpty())
+			return;
+
+		String[] parts = line.split("\\s+", 2);
+		String command = parts[0];
+		switch (command.toUpperCase()) {
+			case "ONLY":
+				handleOnly(true, parts[1]);
+				return;
+			case "NOT":
+				handleOnly(false, parts[1]);
+				return;
+			case "PARSE":
+				handleParse(parts[1], null);
+				return;
+			case "ACCEPT":
+				handleParse(parts[1], true);
+				return;
+			case "REJECT":
+				handleParse(parts[1], false);
+				return;
+			case "SET":
+				handleSet(parts[1]);
+				return;
+			case "EXPECT":
+				handleExpect(parts[1]);
+				return;
+			default:
+				throw new Failure("Unexpected command: " + command);
+		}
+
+	}
+
+	private void handleOnly(boolean mustBePresent, String rest) throws Failure {
+		boolean found = false;
+		for (String part : rest.split("\\s+")) {
+			if (part.equals("jdbc")) {
+				found = true;
+				break;
+			}
+		}
+		if (found != mustBePresent) {
+			// do not further process this block
+			stopProcessing();
+		}
+	}
+
+	private int findEqualSign(String rest) throws Failure {
+		int index = rest.indexOf('=');
+		if (index < -1)
+			throw new Failure("Expected to find a '='");
+		return index;
+	}
+
+	private String splitKey(String rest) throws Failure {
+		int index = findEqualSign(rest);
+		return rest.substring(0, index);
+	}
+
+	private String splitValue(String rest) throws Failure {
+		int index = findEqualSign(rest);
+		return rest.substring(index + 1);
+	}
+
+	private void handleSet(String rest) throws Failure {
+		validated = null;
+		String key = splitKey(rest);
+		String value = splitValue(rest);
+
+		try {
+			target.setString(key, value);
+		} catch (ValidationError e) {
+			throw new Failure(e.getMessage());
+		}
+	}
+
+	private void handleParse(String rest, Boolean shouldSucceed) throws Failure {
+		URISyntaxException parseError = null;
+		ValidationError validationError = null;
+
+		validated = null;
+		try {
+			target.barrier();
+			MonetUrlParser.parse(target, rest);
+		} catch (URISyntaxException e) {
+			parseError = e;
+		} catch (ValidationError e) {
+			validationError = e;
+		}
+
+		if (parseError == null && validationError == null) {
+			try {
+				tryValidate();
+			} catch (ValidationError e) {
+				validationError = e;
+			}
+		}
+
+		if (shouldSucceed == Boolean.FALSE) {
+			if (parseError != null || validationError != null)
+				return; // happy
+			else
+				throw new Failure("URL unexpectedly parsed and validated");
+		}
+
+		if (parseError != null)
+			throw new Failure("Parse error: " + parseError);
+		if (validationError != null && shouldSucceed == Boolean.TRUE)
+			throw new Failure("Validation error: " + validationError);
+	}
+
+	private void handleExpect(String rest) throws Failure {
+		String key = splitKey(rest);
+		String expectedString = splitValue(rest);
+
+		Object actual = null;
+		try {
+			actual = extract(key);
+		} catch (ValidationError e) {
+			throw new Failure(e.getMessage());
+		}
+
+		Object expected;
+		try {
+			if (actual instanceof Boolean)
+				expected = ParameterType.Bool.parse(key, expectedString);
+			else if (actual instanceof Integer)
+				expected = ParameterType.Int.parse(key, expectedString);
+			else
+				expected = expectedString;
+		} catch (ValidationError e) {
+			String typ = actual.getClass().getName();
+			throw new Failure("Cannot convert expected value <" + expectedString + "> to " + typ + ": " + e.getMessage());
+		}
+
+		if (actual.equals(expected))
+			return;
+		throw new Failure("Expected " + key + "=<" + expectedString + ">, found <" + actual + ">");
+	}
+
+	private Target.Validated tryValidate() throws ValidationError {
+		if (validated == null)
+			validated = target.validate();
+		return validated;
+	}
+
+	private Object extract(String key) throws ValidationError, Failure {
+		switch (key) {
+			case "valid":
+				try {
+					tryValidate();
+				} catch (ValidationError e) {
+					return Boolean.FALSE;
+				}
+				return Boolean.TRUE;
+
+			case "connect_scan":
+				return tryValidate().connectScan();
+			case "connect_port":
+				return tryValidate().connectPort();
+			case "connect_unix":
+				return tryValidate().connectUnix();
+			case "connect_tcp":
+				return tryValidate().connectTcp();
+			case "connect_tls_verify":
+				switch (tryValidate().connectVerify()) {
+					case None:
+						return "";
+					case Cert:
+						return "cert";
+					case Hash:
+						return "hash";
+					case System:
+						return "system";
+					default:
+						throw new IllegalStateException("unreachable");
+				}
+			case "connect_certhash_digits":
+				return tryValidate().connectCertHashDigits();
+			case "connect_binary":
+				return tryValidate().connectBinary();
+			case "connect_clientkey":
+				return tryValidate().connectClientKey();
+			case "connect_clientcert":
+				return tryValidate().connectClientCert();
+
+			default:
+				Parameter parm = Parameter.forName(key);
+				if (parm != null)
+					return target.getObject(parm);
+				else
+					throw new Failure("Unknown attribute: " + key);
+		}
+	}
+
+	public static class Failure extends Exception {
+		private String filename = null;
+		private int lineno = -1;
+
+		public Failure(String message) {
+			super(message);
+		}
+
+		@Override
+		public String getMessage() {
+			StringBuilder buffer = new StringBuilder();
+			if (filename != null) {
+				buffer.append(filename).append(":");
+				if (lineno >= 0)
+					buffer.append(lineno).append(":");
+			}
+			buffer.append(super.getMessage());
+			return buffer.toString();
+		}
+
+		public String getFilename() {
+			return filename;
+		}
+
+		public void setFilename(String filename) {
+			this.filename = filename;
+		}
+
+		public int getLineno() {
+			return lineno;
+		}
+
+		public void setLineno(int lineno) {
+			this.lineno = lineno;
+		}
+	}
 }

From dc11d04cc4e4100fdffb4272ff78216644c29d70 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 10:51:19 +0100
Subject: [PATCH 28/41] Suppress warning about unchecked conversion

---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 120e25e..24e1eb2 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -506,9 +506,9 @@ private String hashPassword(String challenge, String password, String passwordAl
 		StringBuilder output = new StringBuilder(10 + maxHashLength / 4);
 		MessageDigest passwordDigest = pickBestAlgorithm(Collections.singleton(passwordAlgo), output);
 
-		Set algoSet  = new HashSet(Arrays.asList(serverSupportedAlgos.split(",")));
+		Set algoSet  = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(",")));
 		if (!configuredHashes.isEmpty()) {
-			Set keep = new HashSet(Arrays.asList(configuredHashes.toUpperCase().split("[, ]")));
+			Set keep = new HashSet<>(Arrays.asList(configuredHashes.toUpperCase().split("[, ]")));
 			algoSet.retainAll(keep);
 			if (algoSet.isEmpty()) {
 				throw new MCLException("None of the hash algorithms <" + configuredHashes + "> are supported, server only supports <" + serverSupportedAlgos + ">");

From 32c8324eb8c19eacc042d1012aa30c73a618b806 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 11:05:16 +0100
Subject: [PATCH 29/41] Heed followRedirect

---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 24e1eb2..616e104 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -112,7 +112,12 @@ public final class MapiSocket {
 	/** protocol version of the connection */
 	private int version;
 
-	/** Whether we should follow redirects */
+	/** Whether we should follow redirects.
+	 * Not sure why this needs to be separate
+	 * from 'ttl' but someone someday explicitly documented setTtl
+	 * with 'to disable completely, use followRedirects' so
+	 * apparently there is a use case.
+	 */
 	private boolean followRedirects = true;
 	/** How many redirections do we follow until we're fed up with it? */
 	private int ttl = 10;
@@ -316,7 +321,7 @@ public List connect(Target target, OptionsCallback callback) throws MCLE
 				if (!ok)
 					close();
 			}
-		} while (attempts++ < this.ttl);
+		} while (followRedirects && attempts++ < this.ttl);
 		throw new MCLException("max redirect count exceeded");
 	}
 

From 43a05c92be4c0076687a3e15abb1a327e3d96734 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 11:08:36 +0100
Subject: [PATCH 30/41] Move timeZoneSet and sizeHeaderEnabled into Callback

They're only used during the handshake
---
 src/main/java/org/monetdb/jdbc/MonetConnection.java | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java
index e6627c7..efd47db 100644
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -112,9 +112,6 @@ public class MonetConnection
 
 	/** The number of results we receive from the server at once */
 	private int curReplySize = 100;	// server default
-	private boolean sizeHeaderEnabled = false; // used during handshake
-	private boolean timeZoneSet = false; // used during handshake
-
 
 	/** A template to apply to each query (like pre and post fixes), filled in constructor */
 	// note: it is made public to the package as queryTempl[2] is used from MonetStatement
@@ -236,10 +233,10 @@ public class MonetConnection
 			if (autoCommit != target.isAutocommit()) {
 				setAutoCommit(target.isAutocommit());
 			}
-			if (!sizeHeaderEnabled) {
+			if (!callback.sizeHeaderEnabled) {
 				sendControlCommand("sizeheader 1");
 			}
-			if (!timeZoneSet) {
+			if (!callback.timeZoneSet) {
 				setTimezone(target.getTimezone());
 			}
 		}
@@ -3774,6 +3771,9 @@ enum SqlOption {
 
 	private class SqlOptionsCallback extends MapiSocket.OptionsCallback {
 		private int level;
+		boolean sizeHeaderEnabled = false; // used during handshake
+		boolean timeZoneSet = false; // used during handshake
+
 
 		@Override
 		public void addOptions(String lang, int level) {

From e740edf43b02695c72a9f49c3c4ba402f6d7df9c Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 13:00:49 +0100
Subject: [PATCH 31/41] Debug and clarify pasword hashing code

---
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 75 ++++++++++++-------
 1 file changed, 50 insertions(+), 25 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 616e104..d06cd1c 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -460,7 +460,7 @@ private String challengeResponse(
 		String parts[] = challengeLine.split(":");
 		if (parts.length < 3)
 			throw new MCLException("Invalid challenge: expect at least 3 fields");
-		String challengePart = parts[0];
+		String saltPart = parts[0];
 		String serverTypePart = parts[1];
 		String versionPart = parts[2];
 		int version;
@@ -484,8 +484,6 @@ private String challengeResponse(
 		} else {
 			userResponse = target.getUser();
 		}
-		String passwordResponse = hashPassword(challengePart, password, passwordHashPart, validated.getHash(), serverHashesPart);
-
 		String optionsResponse = handleOptions(callback, optionsPart);
 
 		// Response looks like this:
@@ -495,7 +493,8 @@ private String challengeResponse(
 		StringBuilder response = new StringBuilder(80);
 		response.append("BIG:");
 		response.append(userResponse).append(":");
-		response.append(passwordResponse).append(":");
+		hashPassword(response, saltPart, password, passwordHashPart, validated.getHash(), serverHashesPart);
+		response.append(":");
 		response.append(validated.getLanguage()).append(":");
 		response.append(validated.getDatabase()).append(":");
 		response.append("FILETRANS:");
@@ -505,32 +504,53 @@ private String challengeResponse(
 	}
 
 	// challengePart, passwordHashPart, supportedHashesPart, target.getPassword()
-	private String hashPassword(String challenge, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException {
-		int maxHashLength = 512;
-
-		StringBuilder output = new StringBuilder(10 + maxHashLength / 4);
-		MessageDigest passwordDigest = pickBestAlgorithm(Collections.singleton(passwordAlgo), output);
-
+	private String hashPassword(String salt, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException {
+		// First determine which hash algorithms we can choose from for the challenge response.
+		// This defaults to whatever the server offers but may be restricted by the user.
 		Set algoSet  = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(",")));
 		if (!configuredHashes.isEmpty()) {
-			Set keep = new HashSet<>(Arrays.asList(configuredHashes.toUpperCase().split("[, ]")));
-			algoSet.retainAll(keep);
+			String[] allowedList = configuredHashes.toUpperCase().split("[, ]");
+			Set allowedSet = new HashSet<>(Arrays.asList(allowedList));
+			algoSet.retainAll(allowedSet);
 			if (algoSet.isEmpty()) {
-				throw new MCLException("None of the hash algorithms <" + configuredHashes + "> are supported, server only supports <" + serverSupportedAlgos + ">");
+				throw new MCLException("None of the hash algorithms in <" + configuredHashes + "> are supported, server only supports <" + serverSupportedAlgos + ">");
 			}
 		}
-		MessageDigest challengeDigest = pickBestAlgorithm(algoSet, null);
 
-		// First we use the password algo to hash the password.
-		// Then we use the challenge algo to hash the combination of the resulting hash digits and the challenge.
-		StringBuilder intermediate = new StringBuilder(maxHashLength / 4 + challenge.length());
+		int maxHashDigits = 512 / 4;
+
+		// We'll collect the result in the responseBuffer.
+		// It will start with '{' HASHNAME '}' followed by hexdigits
+		responseBuffer.append('{');
+
+		// This is where we accumulate what will eventually be hashed into the hexdigits above.
+		// It consists of the hexadecimal pre-hash of the password,
+		// followed by the salt from the server
+		StringBuilder intermediate = new StringBuilder(maxHashDigits + salt.length());
+
+		MessageDigest passwordDigest = pickBestAlgorithm(Collections.singleton(passwordAlgo), null);
+		// Here's the password..
 		hexhash(intermediate, passwordDigest, password);
-		intermediate.append(challenge);
-		hexhash(output, challengeDigest, intermediate.toString());
-		return output.toString();
+		// .. and here's the salt
+		intermediate.append(salt);
+
+		MessageDigest responseDigest = pickBestAlgorithm(algoSet, responseBuffer);
+		responseBuffer.append('}');
+		// pickBestAlgorithm has appended HASHNAME, buffer now contains '{' HASHNAME '}'
+		hexhash(responseBuffer, responseDigest, intermediate.toString());
+		// response buffer now contains '{' HASHNAME '}' HEX_DIGITS_OF_INTERMEDIATE_BUFFER
+
+		return responseBuffer.toString();
 	}
 
-	private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendPrefixHere) throws MCLException {
+	/**
+	 * Pick the most preferred digest algorithm and return a MessageDigest instance for that.
+	 * @param algos the MAPI names of permitted algorithms
+	 * @param appendMapiName if not null, append MAPI name of chose algorithm here
+	 * @return instance of the chosen digester
+	 * @throws MCLException if none of the options is supported
+	 */
+	private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendMapiName) throws MCLException {
 		for (String[] choice: KNOWN_ALGORITHMS) {
             String mapiName = choice[0];
             String algoName = choice[1];
@@ -543,10 +563,8 @@ private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendP
                 continue;
             }
             // we found a match
-            if (appendPrefixHere != null) {
-                appendPrefixHere.append('{');
-                appendPrefixHere.append(mapiName);
-                appendPrefixHere.append('}');
+            if (appendMapiName != null) {
+                appendMapiName.append(mapiName);
             }
             return digest;
         }
@@ -554,6 +572,13 @@ private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendP
 		throw new MCLException("No supported hash algorithm: " + algoNames);
 	}
 
+	/**
+	 * Hash the text into the digest and append the hexadecimal form of the
+	 * resulting digest to buffer.
+	 * @param buffer where the hex digits are appended
+	 * @param digest where the hex digits come from after the text has been digested
+	 * @param text text to digest
+	 */
 	private void hexhash(StringBuilder buffer, MessageDigest digest, String text) {
 		byte[] bytes = text.getBytes(StandardCharsets.UTF_8);
 		digest.update(bytes);

From 6c04469528207de8dd1d48dc548bdc0d4a6a1c32 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 16:13:01 +0100
Subject: [PATCH 32/41] fix comments

---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index d06cd1c..d1104aa 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -503,8 +503,7 @@ private String challengeResponse(
 		return response.toString();
 	}
 
-	// challengePart, passwordHashPart, supportedHashesPart, target.getPassword()
-	private String hashPassword(String salt, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException {
+	private String hashPassword(StringBuilder responseBuffer, String salt, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException {
 		// First determine which hash algorithms we can choose from for the challenge response.
 		// This defaults to whatever the server offers but may be restricted by the user.
 		Set algoSet  = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(",")));
@@ -1508,8 +1507,8 @@ public int read(final byte[] dest, int off, int len) throws IOException {
 	 * Newer MonetDB versions allow setting some options during the handshake.
 	 * The options are language-specific and each has a 'level'. The server
 	 * advertises up to which level options are supported for a given language
-	 * and for each language/option combination, {@link addOptions} will be invoked.
-	 * It should call {@link contribute} for each option it wants to set.
+	 * and for each language/option combination, {@link #addOptions} will be invoked.
+	 * It should call {@link #contribute} for each option it wants to set.
 	 * 
 	 * At the time of writing, only the 'sql' language supports options,
 	 * they are listed in enum mapi_handshake_options_levels in mapi.h.
@@ -1519,7 +1518,7 @@ public static abstract class OptionsCallback {
 
 		/**
 		 * Callback called for each language/level combination supported by the
-		 * server. May call {@link contribute} for options with a level STRICTLY
+		 * server. May call {@link #contribute} for options with a level STRICTLY
 		 * LOWER than the level passed as a parameter.
 		 * @param lang language advertised by the server
 		 * @param level one higher than the maximum supported option

From cbdff4580328d9d7d2f50ac96299f9c34ab21ce2 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 15 Dec 2023 16:36:49 +0100
Subject: [PATCH 33/41] Improve error messages

---
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 39 ++++++++++++-------
 .../org/monetdb/mcl/net/SecureSocket.java     |  2 +-
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index d1104aa..ab7b742 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -23,6 +23,8 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.*;
 
+import javax.net.ssl.SSLException;
+
 import org.monetdb.mcl.MCLException;
 import org.monetdb.mcl.io.BufferedMCLReader;
 import org.monetdb.mcl.io.BufferedMCLWriter;
@@ -356,22 +358,29 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx
 			throw new MCLException("Unix domain sockets are not supported, only TCP");
 		}
 		int port = validated.connectPort();
-		Socket sock = new Socket(tcpHost, port);
-		sock.setSoTimeout(validated.getSoTimeout());
-		sock.setTcpNoDelay(true);
-		sock.setKeepAlive(true);
-
-		sock = wrapTLS(sock, validated);
-
-		fromMonet = new BlockInputStream(sock.getInputStream());
-		toMonet = new BlockOutputStream(sock.getOutputStream());
-		reader = new BufferedMCLReader(fromMonet, StandardCharsets.UTF_8);
-		writer = new BufferedMCLWriter(toMonet, StandardCharsets.UTF_8);
-		writer.registerReader(reader);
-		reader.advance();
+		Socket sock = null;
+		try {
+			sock = new Socket(tcpHost, port);
+			sock.setSoTimeout(validated.getSoTimeout());
+			sock.setTcpNoDelay(true);
+			sock.setKeepAlive(true);
+
+			sock = wrapTLS(sock, validated);
+
+			fromMonet = new BlockInputStream(sock.getInputStream());
+			toMonet = new BlockOutputStream(sock.getOutputStream());
+			reader = new BufferedMCLReader(fromMonet, StandardCharsets.UTF_8);
+			writer = new BufferedMCLWriter(toMonet, StandardCharsets.UTF_8);
+			writer.registerReader(reader);
+			reader.advance();
 
-		// Only assign to sock when everything went ok so far
-		con = sock;
+			// Only assign to sock when everything went ok so far
+			con = sock;
+		} catch (SSLException e) {
+			throw new MCLException("SSL error: " + e.getMessage(), e);
+		} catch (IOException e) {
+			throw new MCLException("Could not connect to " + tcpHost + ":" + port + ": " + e.getMessage(), e);
+		}
 	}
 
 	private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException {
diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index 9fcb7b7..01cb363 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -38,7 +38,7 @@ public static Socket wrap(Target.Validated validated, Socket inner) throws IOExc
 			}
 			return wrapSocket(inner, validated, socketFactory, checkName);
 		} catch (CertificateException e) {
-			throw new SSLException(e.getMessage(), e);
+			throw new SSLException("TLS certificate rejected", e);
 		}
 	}
 

From f3f08fcea5b4e0c8c911a239f7495a064f3cd6e1 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Wed, 3 Jan 2024 11:05:14 +0100
Subject: [PATCH 34/41] comment

---
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index ab7b742..9437267 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -388,13 +388,18 @@ private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOExcepti
             return SecureSocket.wrap(validated, sock);
         else {
 			// Send an even number of NUL bytes.
-			// We expect the server to speak MAPI and in that case, it's a NOP.
-			// If we're accidentally connecting to a TLS server, the bytes are
-			// invalid as a Client Hello message and most TLS implementations
-			// drop the connection.
-			// This is nice because otherwise we would hang, as the TLS server
-			// is waiting for us to send a TLS CLient Hello, and we are waiting
-			// for a MAPI server to send a server challenge.
+			// We expect the server to speak the MAPI protocol and in MAPI,
+			// NUL NUL is a no-op.
+			// However, if we're accidentally connecting to a TLS-protected
+			// server, that server expects a TLS 'Client Hello' message and
+			// the NULs will hopefully force an error.
+			// The error is useful because otherwise we end up in a deadlock:
+			// - the MAPI client is waiting for the server to send a MAPI challenge,
+			// - the TLS server is waiting fot the client to send a Client Hello.
+			// Unfortunately, the number of NULs needed to force an error
+			// varies between implementations. Some TLS servers abort after
+			// the first NUL, others need lots of them.
+			// For now we standardize on 8.
 			sock.getOutputStream().write(NUL_BYTES);
 		}
         return sock;

From 4bbfa1fe0afc511493728fdced64fdbcbd193438 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Wed, 3 Jan 2024 11:05:56 +0100
Subject: [PATCH 35/41] Cache the system trust roots between invocations

Loading them is expensive, it easily takes 100-200 milliseconds.
---
 .../java/org/monetdb/mcl/net/SecureSocket.java     | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/monetdb/mcl/net/SecureSocket.java b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
index 01cb363..f9eac14 100644
--- a/src/main/java/org/monetdb/mcl/net/SecureSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/SecureSocket.java
@@ -16,6 +16,18 @@ public class SecureSocket {
 	private static final String[] ENABLED_PROTOCOLS = {"TLSv1.3"};
 	private static final String[] APPLICATION_PROTOCOLS = {"mapi/9"};
 
+	// Cache for the default SSL factory. It must load all trust roots
+	// so it's worthwhile to cache.
+	// Only access this through #getDefaultSocketFactory()
+	private static SSLSocketFactory vanillaFactory = null;
+
+	private static synchronized SSLSocketFactory getDefaultSocketFactory() {
+		if (vanillaFactory == null) {
+			vanillaFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
+		}
+		return vanillaFactory;
+	}
+
 	public static Socket wrap(Target.Validated validated, Socket inner) throws IOException {
 		Target.Verify verify = validated.connectVerify();
 		SSLSocketFactory socketFactory;
@@ -23,7 +35,7 @@ public static Socket wrap(Target.Validated validated, Socket inner) throws IOExc
 		try {
 			switch (verify) {
 				case System:
-					socketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault();
+					socketFactory = getDefaultSocketFactory();
 					break;
 				case Cert:
 					KeyStore keyStore = keyStoreForCert(validated.getCert());

From 9a4095a960dbc3b1a7fafc2f25b5b4f93b4211d0 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 10:17:34 +0100
Subject: [PATCH 36/41] convert space indents to tabs

---
 src/main/java/org/monetdb/jdbc/MonetBlob.java |  4 +-
 .../org/monetdb/jdbc/MonetConnection.java     |  2 +-
 .../java/org/monetdb/jdbc/MonetStatement.java |  2 +-
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 90 +++++++++----------
 .../java/org/monetdb/util/CmdLineOpts.java    |  6 +-
 5 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/src/main/java/org/monetdb/jdbc/MonetBlob.java b/src/main/java/org/monetdb/jdbc/MonetBlob.java
index d6cc7f5..f0ab255 100644
--- a/src/main/java/org/monetdb/jdbc/MonetBlob.java
+++ b/src/main/java/org/monetdb/jdbc/MonetBlob.java
@@ -299,8 +299,8 @@ public int setBytes(final long pos, final byte[] bytes, int offset, final int le
 		try {
 			offset--;
 			/* transactions? what are you talking about? */
-            if (len - (int) pos >= 0)
-                System.arraycopy(bytes, offset + (int) pos, buf, (int) pos, len - (int) pos);
+			if (len - (int) pos >= 0)
+				System.arraycopy(bytes, offset + (int) pos, buf, (int) pos, len - (int) pos);
 		} catch (IndexOutOfBoundsException e) {
 			throw new SQLException(e.getMessage(), "M0M10");
 		}
diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java
index a9c39e3..76887f0 100644
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -2761,7 +2761,7 @@ public void complete() throws SQLException {
 		@Override
 		public void close() {
 			// feed all rows to the garbage collector
-            Arrays.fill(data, null);
+			Arrays.fill(data, null);
 		}
 
 		/**
diff --git a/src/main/java/org/monetdb/jdbc/MonetStatement.java b/src/main/java/org/monetdb/jdbc/MonetStatement.java
index b4a376a..8d76ae7 100644
--- a/src/main/java/org/monetdb/jdbc/MonetStatement.java
+++ b/src/main/java/org/monetdb/jdbc/MonetStatement.java
@@ -1571,7 +1571,7 @@ else if (row > tupleCount + 1)
 		if (row < 1 || row > tupleCount)
 			return false;
 
-        System.arraycopy(results[row - 1], 0, tlp.values, 0, results[row - 1].length);
+		System.arraycopy(results[row - 1], 0, tlp.values, 0, results[row - 1].length);
 
 		return true;
 	}
diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 51a1c7a..66eaaff 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -347,9 +347,9 @@ private boolean tryConnect(OptionsCallback callback, ArrayList warningBu
 			throw e;
 		} catch (ValidationError e) {
 			close();
-            throw new MCLException(e.getMessage());
-        }
-    }
+			throw new MCLException(e.getMessage());
+		}
+	}
 
 	private void connectSocket(Target.Validated validated) throws MCLException, IOException {
 		// This method performs steps 2-6 of the procedure outlined in the URL spec
@@ -384,9 +384,9 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx
 	}
 
 	private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOException {
-        if (validated.getTls())
-            return SecureSocket.wrap(validated, sock);
-        else {
+		if (validated.getTls())
+			return SecureSocket.wrap(validated, sock);
+		else {
 			// Send an even number of NUL bytes.
 			// We expect the server to speak the MAPI protocol and in MAPI,
 			// NUL NUL is a no-op.
@@ -402,8 +402,8 @@ private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOExcepti
 			// For now we standardize on 8.
 			sock.getOutputStream().write(NUL_BYTES);
 		}
-        return sock;
-    }
+		return sock;
+	}
 
 	private boolean handshake(Target.Validated validated, OptionsCallback callback, ArrayList warnings) throws IOException, MCLException {
 		String challenge = reader.getLine();
@@ -449,17 +449,17 @@ private boolean handshake(Target.Validated validated, OptionsCallback callback,
 		} catch (URISyntaxException | ValidationError e) {
 			throw new MCLException("While processing redirect " + redirect + ": " + e.getMessage(), e);
 		}
-        if (redirect.startsWith("mapi:merovingian://proxy")) {
+		if (redirect.startsWith("mapi:merovingian://proxy")) {
 			// The reader is stuck at LineType.PROMPT but actually the
 			// next challenge is already there.
-            reader.resetLineType();
+			reader.resetLineType();
 			reader.advance();
-        } else {
-            close();
-        }
+		} else {
+			close();
+		}
 
-        return false;   // we need another go
-    }
+		return false;   // we need another go
+	}
 
 	private String challengeResponse(
 			Target.Validated validated, final String challengeLine,
@@ -565,22 +565,22 @@ private String hashPassword(StringBuilder responseBuffer, String salt, String pa
 	 */
 	private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendMapiName) throws MCLException {
 		for (String[] choice: KNOWN_ALGORITHMS) {
-            String mapiName = choice[0];
-            String algoName = choice[1];
-            MessageDigest digest;
-            if (!algos.contains(mapiName))
-                continue;
-            try {
-                digest = MessageDigest.getInstance(algoName);
-            } catch (NoSuchAlgorithmException e) {
-                continue;
-            }
-            // we found a match
-            if (appendMapiName != null) {
-                appendMapiName.append(mapiName);
-            }
-            return digest;
-        }
+			String mapiName = choice[0];
+			String algoName = choice[1];
+			MessageDigest digest;
+			if (!algos.contains(mapiName))
+				continue;
+			try {
+				digest = MessageDigest.getInstance(algoName);
+			} catch (NoSuchAlgorithmException e) {
+				continue;
+			}
+			// we found a match
+			if (appendMapiName != null) {
+				appendMapiName.append(mapiName);
+			}
+			return digest;
+		}
 		String algoNames = String.join(",", algos);
 		throw new MCLException("No supported hash algorithm: " + algoNames);
 	}
@@ -610,22 +610,22 @@ private String handleOptions(OptionsCallback callback, String optionsPart) throw
 
 		StringBuilder buffer = new StringBuilder();
 		callback.setBuffer(buffer);
-        for (String optlevel: optionsPart.split(",")) {
-            int eqindex = optlevel.indexOf('=');
-            if (eqindex < 0)
-                throw new MCLException("Invalid options part in server challenge: " + optionsPart);
-            String lang = optlevel.substring(0, eqindex);
-            int level;
-            try {
-                level = Integer.parseInt(optlevel.substring(eqindex + 1));
-            } catch (NumberFormatException e) {
-                throw new MCLException("Invalid option level in server challenge: " + optlevel);
-            }
-            callback.addOptions(lang, level);
-        }
+		for (String optlevel: optionsPart.split(",")) {
+			int eqindex = optlevel.indexOf('=');
+			if (eqindex < 0)
+				throw new MCLException("Invalid options part in server challenge: " + optionsPart);
+			String lang = optlevel.substring(0, eqindex);
+			int level;
+			try {
+				level = Integer.parseInt(optlevel.substring(eqindex + 1));
+			} catch (NumberFormatException e) {
+				throw new MCLException("Invalid option level in server challenge: " + optlevel);
+			}
+			callback.addOptions(lang, level);
+		}
 
 		return buffer.toString();
-    }
+	}
 
 	/**
 	 * Returns an InputStream that reads from this open connection on
diff --git a/src/main/java/org/monetdb/util/CmdLineOpts.java b/src/main/java/org/monetdb/util/CmdLineOpts.java
index 3216223..0b41fe0 100644
--- a/src/main/java/org/monetdb/util/CmdLineOpts.java
+++ b/src/main/java/org/monetdb/util/CmdLineOpts.java
@@ -96,9 +96,9 @@ public void processFile(final java.io.File file) throws OptionsException {
 					option.addArgument(prop.getProperty(key));
 				} else if (!ignoredInFile.contains(key)) {
 					// ignore unknown options (it used to throw an OptionsException)
-                    System.out.println("Info: Ignoring unknown/unsupported option (in " + file.getAbsolutePath() + "): " + key);
-                }
-            }
+					System.out.println("Info: Ignoring unknown/unsupported option (in " + file.getAbsolutePath() + "): " + key);
+				}
+			}
 		} catch (java.io.IOException e) {
 			throw new OptionsException("File IO Exception: " + e);
 		}

From f07446509f5bd1d86559b603e20cfa437202a363 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 10:41:24 +0100
Subject: [PATCH 37/41] formatting

---
 .../org/monetdb/jdbc/MonetConnection.java     |  1 +
 .../org/monetdb/mcl/io/BufferedMCLReader.java |  1 +
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 29 +++++++++----------
 .../java/org/monetdb/mcl/net/Parameter.java   |  2 ++
 .../org/monetdb/mcl/net/ParameterType.java    |  5 +++-
 tests/javaspecific.md                         |  1 -
 6 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/monetdb/jdbc/MonetConnection.java b/src/main/java/org/monetdb/jdbc/MonetConnection.java
index 76887f0..95c64eb 100644
--- a/src/main/java/org/monetdb/jdbc/MonetConnection.java
+++ b/src/main/java/org/monetdb/jdbc/MonetConnection.java
@@ -1245,6 +1245,7 @@ public boolean isValid(final int timeout) throws SQLException {
 	 * Passing this to {@link DriverManager.getConnection()} together
 	 * with the URL "jdbc:monetdb:" will create a new connection identical to
 	 * the current one.
+	 *
 	 * @return
 	 */
 
diff --git a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
index 6137851..cda6f97 100644
--- a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
+++ b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
@@ -96,6 +96,7 @@ public String getLine() {
 
 	/**
 	 * Return a substring of the current line, or null if we're at the end or before the beginning.
+	 *
 	 * @return the current line or null
 	 */
 	public String getLine(int start) {
diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 66eaaff..88b3904 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -83,7 +83,7 @@
  */
 public final class MapiSocket {
 	/* an even number of NUL bytes used during the handshake */
-	private static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0, };
+	private static final byte[] NUL_BYTES = new byte[]{ 0, 0, 0, 0, 0, 0, 0, 0 };
 
 	/* A mapping between hash algorithm names as used in the MAPI
 	 * protocol, and the names by which the Java runtime knows them.
@@ -273,15 +273,15 @@ public List connect(String url, Properties props) throws URISyntaxExcept
 		return connect(new Target(url, props), null);
 	}
 
-		/**
+	/**
 	 * Connect according to the settings in the 'target' parameter.
 	 * If followRedirect is false, a RedirectionException is
 	 * thrown when a redirect is encountered.
-	 * 
+	 *
 	 * Some settings, such as the initial reply size, can already be configured
 	 * during the handshake, saving a command round-trip later on.
 	 * To do so, create and pass a subclass of {@link MapiSocket.OptionsCallback}.
-	 * 
+	 *
 	 * @param target the connection settings
 	 * @param callback will be called if the server allows options to be set during the
 	 * initial handshake
@@ -461,17 +461,14 @@ private boolean handshake(Target.Validated validated, OptionsCallback callback,
 		return false;   // we need another go
 	}
 
-	private String challengeResponse(
-			Target.Validated validated, final String challengeLine,
-			OptionsCallback callback
-	) throws MCLException {
+	private String challengeResponse(Target.Validated validated, final String challengeLine, OptionsCallback callback) throws MCLException {
 		// The challengeLine looks like this:
 		//
 		// 45IYyVyRnbgEnK92ad:merovingian:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512:
 		// WgHIibSyH:mserver:9:RIPEMD160,SHA512,SHA384,SHA256,SHA224,SHA1:LIT:SHA512:sql=6:BINARY=1:
 		// 0         1       2 3                                          4   5      6     7
 
-		String parts[] = challengeLine.split(":");
+		String[] parts = challengeLine.split(":");
 		if (parts.length < 3)
 			throw new MCLException("Invalid challenge: expect at least 3 fields");
 		String saltPart = parts[0];
@@ -520,7 +517,7 @@ private String challengeResponse(
 	private String hashPassword(StringBuilder responseBuffer, String salt, String password, String passwordAlgo, String configuredHashes, String serverSupportedAlgos) throws MCLException {
 		// First determine which hash algorithms we can choose from for the challenge response.
 		// This defaults to whatever the server offers but may be restricted by the user.
-		Set algoSet  = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(",")));
+		Set algoSet = new HashSet<>(Arrays.asList(serverSupportedAlgos.split(",")));
 		if (!configuredHashes.isEmpty()) {
 			String[] allowedList = configuredHashes.toUpperCase().split("[, ]");
 			Set allowedSet = new HashSet<>(Arrays.asList(allowedList));
@@ -558,13 +555,14 @@ private String hashPassword(StringBuilder responseBuffer, String salt, String pa
 
 	/**
 	 * Pick the most preferred digest algorithm and return a MessageDigest instance for that.
-	 * @param algos the MAPI names of permitted algorithms
+	 *
+	 * @param algos          the MAPI names of permitted algorithms
 	 * @param appendMapiName if not null, append MAPI name of chose algorithm here
 	 * @return instance of the chosen digester
 	 * @throws MCLException if none of the options is supported
 	 */
 	private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendMapiName) throws MCLException {
-		for (String[] choice: KNOWN_ALGORITHMS) {
+		for (String[] choice : KNOWN_ALGORITHMS) {
 			String mapiName = choice[0];
 			String algoName = choice[1];
 			MessageDigest digest;
@@ -588,15 +586,16 @@ private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendM
 	/**
 	 * Hash the text into the digest and append the hexadecimal form of the
 	 * resulting digest to buffer.
+	 *
 	 * @param buffer where the hex digits are appended
 	 * @param digest where the hex digits come from after the text has been digested
-	 * @param text text to digest
+	 * @param text   text to digest
 	 */
 	private void hexhash(StringBuilder buffer, MessageDigest digest, String text) {
 		byte[] bytes = text.getBytes(StandardCharsets.UTF_8);
 		digest.update(bytes);
 		byte[] output = digest.digest();
-		for (byte b: output) {
+		for (byte b : output) {
 			int hi = (b & 0xF0) >> 4;
 			int lo = b & 0x0F;
 			buffer.append(HEXDIGITS[hi]);
@@ -610,7 +609,7 @@ private String handleOptions(OptionsCallback callback, String optionsPart) throw
 
 		StringBuilder buffer = new StringBuilder();
 		callback.setBuffer(buffer);
-		for (String optlevel: optionsPart.split(",")) {
+		for (String optlevel : optionsPart.split(",")) {
 			int eqindex = optlevel.indexOf('=');
 			if (eqindex < 0)
 				throw new MCLException("Invalid options part in server challenge: " + optionsPart);
diff --git a/src/main/java/org/monetdb/mcl/net/Parameter.java b/src/main/java/org/monetdb/mcl/net/Parameter.java
index e1bbf68..1c42606 100644
--- a/src/main/java/org/monetdb/mcl/net/Parameter.java
+++ b/src/main/java/org/monetdb/mcl/net/Parameter.java
@@ -115,6 +115,7 @@ public static Parameter forName(String name) {
 	 * The ground rule is that if we encounter an unknown setting
 	 * without an underscore in the name, it is an error. If it has
 	 * an underscore in its name, it can be ignored.
+	 *
 	 * @param name the name of the setting to check
 	 * @return true if it can safely be ignored
 	 */
@@ -128,6 +129,7 @@ public static boolean isIgnored(String name) {
 	 * Return a default value for the given setting, as an Object of the appropriate type.
 	 * Note that the value returned for TIMEZONE may change if the system time zone
 	 * is changed or if Daylight Saving Time starts or ends.
+	 *
 	 * @return
 	 */
 	public Object getDefault() {
diff --git a/src/main/java/org/monetdb/mcl/net/ParameterType.java b/src/main/java/org/monetdb/mcl/net/ParameterType.java
index e0c5191..49034b8 100644
--- a/src/main/java/org/monetdb/mcl/net/ParameterType.java
+++ b/src/main/java/org/monetdb/mcl/net/ParameterType.java
@@ -26,6 +26,7 @@ public enum ParameterType {
 	 * Convert a string to a boolean, accepting true/false/yes/no/on/off.
 	 * 
 	 * Uppercase is also accepted.
+	 *
 	 * @param value text to be parsed
 	 * @return boolean interpretation of the text
 	 */
@@ -55,7 +56,8 @@ public static boolean parseBool(String value) {
 
 	/**
 	 * Convert text into an Object of the appropriate type
-	 * @param name name of the setting for use in error messages
+	 *
+	 * @param name  name of the setting for use in error messages
 	 * @param value text to be converted
 	 * @return Object representation of the text
 	 * @throws ValidationError if the text cannot be converted
@@ -84,6 +86,7 @@ public Object parse(String name, String value) throws ValidationError {
 
 	/**
 	 * Represent the object as a string.
+	 *
 	 * @param value, must be of the appropriate type
 	 * @return textual representation
 	 */
diff --git a/tests/javaspecific.md b/tests/javaspecific.md
index f1739fc..5aced13 100644
--- a/tests/javaspecific.md
+++ b/tests/javaspecific.md
@@ -1,4 +1,3 @@
-
 # Java-specific tests
 
 Test settings that are only in monetdb-java.

From 8e400526c9a6b8343cf30b5b4ce28dab92cb6d52 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 11:29:40 +0100
Subject: [PATCH 38/41] comments

---
 .../java/org/monetdb/mcl/net/MapiSocket.java  | 42 +++++++++----------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 88b3904..0595014 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -387,19 +387,17 @@ private Socket wrapTLS(Socket sock, Target.Validated validated) throws IOExcepti
 		if (validated.getTls())
 			return SecureSocket.wrap(validated, sock);
 		else {
-			// Send an even number of NUL bytes.
-			// We expect the server to speak the MAPI protocol and in MAPI,
-			// NUL NUL is a no-op.
-			// However, if we're accidentally connecting to a TLS-protected
-			// server, that server expects a TLS 'Client Hello' message and
-			// the NULs will hopefully force an error.
-			// The error is useful because otherwise we end up in a deadlock:
-			// - the MAPI client is waiting for the server to send a MAPI challenge,
-			// - the TLS server is waiting fot the client to send a Client Hello.
-			// Unfortunately, the number of NULs needed to force an error
-			// varies between implementations. Some TLS servers abort after
-			// the first NUL, others need lots of them.
-			// For now we standardize on 8.
+			// Send an even number of NUL bytes to avoid a deadlock if
+			// we're accidentally connecting to a TLS-protected server.
+			// The cause of the deadlock is that we speak MAPI and we wait
+			// for the server to send a MAPI challenge.
+			// However, if the server is trying to set up TLS, it will be
+			// waiting for us to send a TLS 'Client Hello' packet.
+			// Hence, deadlock.
+			// NUL NUL is a no-op in MAPI and will hopefully force an error
+			// in the TLS server. This does not always work, some
+			// TLS implementations abort on the first NUL, some need more NULs
+			// than we are prepared to send here. 8 seems to be a good number.
 			sock.getOutputStream().write(NUL_BYTES);
 		}
 		return sock;
@@ -531,7 +529,6 @@ private String hashPassword(StringBuilder responseBuffer, String salt, String pa
 
 		// We'll collect the result in the responseBuffer.
 		// It will start with '{' HASHNAME '}' followed by hexdigits
-		responseBuffer.append('{');
 
 		// This is where we accumulate what will eventually be hashed into the hexdigits above.
 		// It consists of the hexadecimal pre-hash of the password,
@@ -544,7 +541,9 @@ private String hashPassword(StringBuilder responseBuffer, String salt, String pa
 		// .. and here's the salt
 		intermediate.append(salt);
 
+		responseBuffer.append('{');
 		MessageDigest responseDigest = pickBestAlgorithm(algoSet, responseBuffer);
+		// the call above has appended the HASHNAME, now add '}'
 		responseBuffer.append('}');
 		// pickBestAlgorithm has appended HASHNAME, buffer now contains '{' HASHNAME '}'
 		hexhash(responseBuffer, responseDigest, intermediate.toString());
@@ -557,7 +556,7 @@ private String hashPassword(StringBuilder responseBuffer, String salt, String pa
 	 * Pick the most preferred digest algorithm and return a MessageDigest instance for that.
 	 *
 	 * @param algos          the MAPI names of permitted algorithms
-	 * @param appendMapiName if not null, append MAPI name of chose algorithm here
+	 * @param appendMapiName if not null, append MAPI name of chose algorithm to this buffer
 	 * @return instance of the chosen digester
 	 * @throws MCLException if none of the options is supported
 	 */
@@ -584,7 +583,7 @@ private MessageDigest pickBestAlgorithm(Set algos, StringBuilder appendM
 	}
 
 	/**
-	 * Hash the text into the digest and append the hexadecimal form of the
+	 * Hash the text into the MessageDigest and append the hexadecimal form of the
 	 * resulting digest to buffer.
 	 *
 	 * @param buffer where the hex digits are appended
@@ -1516,13 +1515,14 @@ public int read(final byte[] dest, int off, int len) throws IOException {
 
 	/**
 	 * Callback used during the initial MAPI handshake.
-	 * 
+	 *
 	 * Newer MonetDB versions allow setting some options during the handshake.
 	 * The options are language-specific and each has a 'level'. The server
-	 * advertises up to which level options are supported for a given language
-	 * and for each language/option combination, {@link #addOptions} will be invoked.
-	 * It should call {@link #contribute} for each option it wants to set.
-	 * 
+	 * advertises up to which level options are supported for a given language.
+	 * For each language/option combination, {@link #addOptions} will be invoked
+	 * during the handshake. This method should call {@link #contribute} for each
+	 * option it wants to set.
+	 *
 	 * At the time of writing, only the 'sql' language supports options,
 	 * they are listed in enum mapi_handshake_options_levels in mapi.h.
 	 */

From eed4cb46573a0a99e5218f3ac48ebed624f42faf Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 11:30:09 +0100
Subject: [PATCH 39/41] Handle null case

---
 src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
index cda6f97..cc59f7e 100644
--- a/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
+++ b/src/main/java/org/monetdb/mcl/io/BufferedMCLReader.java
@@ -100,7 +100,10 @@ public String getLine() {
 	 * @return the current line or null
 	 */
 	public String getLine(int start) {
-		return getLine().substring(start);
+		String line = getLine();
+		if (line != null)
+			line = line.substring(start);
+		return line;
 	}
 
 	/**

From f6c155013c8e86c393c044d4d1d2a2e8d6a19c3a Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 11:33:49 +0100
Subject: [PATCH 40/41] Ensure sock is closed on errors

---
 src/main/java/org/monetdb/mcl/net/MapiSocket.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/java/org/monetdb/mcl/net/MapiSocket.java b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
index 0595014..060d7ef 100644
--- a/src/main/java/org/monetdb/mcl/net/MapiSocket.java
+++ b/src/main/java/org/monetdb/mcl/net/MapiSocket.java
@@ -376,10 +376,18 @@ private void connectSocket(Target.Validated validated) throws MCLException, IOEx
 
 			// Only assign to sock when everything went ok so far
 			con = sock;
+			sock = null;
 		} catch (SSLException e) {
 			throw new MCLException("SSL error: " + e.getMessage(), e);
 		} catch (IOException e) {
 			throw new MCLException("Could not connect to " + tcpHost + ":" + port + ": " + e.getMessage(), e);
+		} finally {
+			if (sock != null)
+				try {
+					sock.close();
+				} catch (IOException e) {
+					// ignore
+				}
 		}
 	}
 

From 87a15fa5ff93138382d0618423d625ce5feb42b2 Mon Sep 17 00:00:00 2001
From: Joeri van Ruth 
Date: Fri, 5 Jan 2024 11:56:03 +0100
Subject: [PATCH 41/41] relax wording of expected exception

---
 tests/TLSTester.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/TLSTester.java b/tests/TLSTester.java
index 5410042..628b4dc 100644
--- a/tests/TLSTester.java
+++ b/tests/TLSTester.java
@@ -201,7 +201,7 @@ private void test_fail_tls_to_plain() throws IOException, SQLException {
 	}
 
 	private void test_fail_plain_to_tls() throws IOException, SQLException {
-		attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect");
+		attempt("fail_plain_to_tls", "server1").with(Parameter.TLS, false).expectFailure("Cannot connect", "Could not connect");
 	}
 
 	private void test_connect_server_name() throws IOException, SQLException {
@@ -305,7 +305,7 @@ public void expectFailure(String... expectedMessages) throws SQLException {
 					if (e.getMessage().contains(expected))
 						return;
 				String message = "Test " + testName + " threw the wrong exception: " + e.getMessage() + '\n' + "Expected:\n        <" + String.join(">\n        <", expectedMessages) + ">";
-				throw new RuntimeException(message);
+				throw new RuntimeException(message, e);
 
 			}
 		}