diff --git a/project.clj b/project.clj index 31a1ae80..28fe4831 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ (defproject aleph (or (System/getenv "PROJECT_VERSION") "0.5.0") :description "A framework for asynchronous communication" - :repositories {"jboss" "https://repository.jboss.org/nexus/content/groups/public/" + :repositories {"jboss" "https://repository.jboss.org/nexus/content/groups/public/" "sonatype-oss-public" "https://oss.sonatype.org/content/groups/public/"} :url "https://github.com/clj-commons/aleph" :license {:name "MIT License"} @@ -21,39 +21,51 @@ [io.netty/netty-handler-proxy ~netty-version] [io.netty/netty-resolver ~netty-version] [io.netty/netty-resolver-dns ~netty-version]] - :profiles {:dev {:dependencies [[org.clojure/clojure "1.10.3"] - [criterium "0.4.6"] - [cheshire "5.10.0"] - [org.slf4j/slf4j-simple "1.7.30"] - [com.cognitect/transit-clj "1.0.324"] - [spootnik/signal "0.2.4"] - [me.mourjo/dynamic-redef "0.1.0"]]} + :profiles {:dev {:dependencies [[org.clojure/clojure "1.11.1"] + [criterium "0.4.6"] + [cheshire "5.10.0"] + [org.slf4j/slf4j-simple "1.7.30"] + [com.cognitect/transit-clj "1.0.324"] + [spootnik/signal "0.2.4"] + [me.mourjo/dynamic-redef "0.1.0"] + + ;; for testing clj-http parity + [clj-http "3.12.3"] + [ring/ring-jetty-adapter "1.9.3"] + [org.apache.logging.log4j/log4j-api "2.17.1"] + [org.apache.logging.log4j/log4j-core "2.17.1"] + [org.apache.logging.log4j/log4j-1.2-api "2.17.1"]]} :lein-to-deps {:source-paths ["deps"]} ;; This is for self-generating certs for testing ONLY: - :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"] - [org.bouncycastle/bcpkix-jdk15on "1.69"]] - :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}} - :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/" + :test {:dependencies [[org.bouncycastle/bcprov-jdk15on "1.69"] + [org.bouncycastle/bcpkix-jdk15on "1.69"]] + :javac-options ^:replace ["--release" "12"] ; necessary for some tests + :jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}} + :codox {:src-dir-uri "https://github.com/ztellman/aleph/tree/master/" :src-linenum-anchor-prefix "L" - :defaults {:doc/format :markdown} - :include [aleph.tcp - aleph.udp - aleph.http - aleph.flow] - :output-dir "doc"} + :defaults {:doc/format :markdown} + :include [aleph.tcp + aleph.udp + aleph.http + aleph.flow] + :output-dir "doc"} :plugins [[lein-codox "0.10.7"] - [lein-jammin "0.1.1"] [lein-marginalia "0.9.1"] [lein-pprint "1.3.2"] [ztellman/lein-cljfmt "0.1.10"]] :java-source-paths ["src/aleph/utils"] :cljfmt {:indents {#".*" [[:inner 0]]}} - :test-selectors {:default #(not - (some #{:benchmark :stress} - (cons (:tag %) (keys %)))) - :benchmark :benchmark - :stress :stress - :all (constantly true)} + :test-selectors {:default #(not + (some #{:benchmark :stress :integration :ignore} + (cons (:tag %) (keys %)))) + :benchmark :benchmark + :integration :integration + :stress :stress + :clj-http [(fn clj-http-ns-pred [namespc & _] + (.contains (str namespc) "clj-http")) + (fn clj-http-test-pred [m & _] + (not (:ignore m)))] + :all (constantly true)} :jvm-opts ^:replace ["-server" "-Xmx2g" "-XX:+HeapDumpOnOutOfMemoryError" diff --git a/src/aleph/http.clj b/src/aleph/http.clj index 060f58f1..44ba97e8 100644 --- a/src/aleph/http.clj +++ b/src/aleph/http.clj @@ -102,7 +102,7 @@ | `max-queue-size` | the maximum number of pending acquires from the pool that are allowed before `acquire` will start to throw a `java.util.concurrent.RejectedExecutionException`, defaults to `65536` | `control-period` | the interval, in milliseconds, between use of the controller to adjust the size of the pool, defaults to `60000` | `dns-options` | an optional map with async DNS resolver settings, for more information check `aleph.netty/dns-resolver-group`. When set, ignores `name-resolver` setting from `connection-options` in favor of shared DNS resolver instance - | `middleware` | a function to modify request before sending, defaults to `aleph.http.client-middleware/wrap-request` + | `middleware` | a function to modify clients/requests before sending, defaults to `aleph.http.client-middleware/wrap-request` | `pool-builder-fn` | an optional one arity function which returns a `io.aleph.dirigiste.IPool` from a map containing the following keys: `generate`, `destroy`, `control-period`, `max-queue-length` and `stats-callback`. | `pool-controller-builder-fn` | an optional zero arity function which returns a `io.aleph.dirigiste.IPool$Controller`. diff --git a/src/aleph/http/client_middleware.clj b/src/aleph/http/client_middleware.clj index f9082de6..4a4c1c34 100644 --- a/src/aleph/http/client_middleware.clj +++ b/src/aleph/http/client_middleware.clj @@ -155,6 +155,7 @@ {:scheme (keyword (.getProtocol url-parsed)) :server-name (.getHost url-parsed) :server-port (when-pos (.getPort url-parsed)) + :url url :uri (url-encode-illegal-characters (.getPath url-parsed)) :user-info (when-let [user-info (.getUserInfo url-parsed)] (URLDecoder/decode user-info)) @@ -243,12 +244,13 @@ (if (unexceptional-status? status) rsp (cond - (false? (opt req :throw-exceptions)) rsp (instance? InputStream body) - (d/chain' (d/future (bs/to-byte-array body)) + (d/chain' + (d/future + (bs/to-byte-array body)) (fn [body] (d/error-deferred (ex-info @@ -563,9 +565,8 @@ [req] (if-let [url (:url req)] (-> req - (dissoc :url) - (assoc :request-url url) - (merge (parse-url url))) + (assoc :request-url url) + (merge (parse-url url))) req)) (defn wrap-request-timing @@ -918,7 +919,13 @@ (opt req :save-request) (assoc :aleph/request req')))) +(def default-client-middleware + "Default middleware that takes a client fn" + [wrap-exceptions + wrap-request-timing]) + (def default-middleware + "Default middleware that takes a request map" [wrap-method wrap-url wrap-nested-params @@ -937,22 +944,24 @@ "Returns a batteries-included HTTP request function corresponding to the given core client. See default-middleware for the middleware wrappers that are used by default" - [client] - (let [client' (-> client - wrap-exceptions - wrap-request-timing)] - (fn [req] - (let [executor (ex/executor)] - (if (:aleph.http.client/close req) - (client req) - - (let [req' (reduce #(%2 %1) req default-middleware)] - (d/chain' (client' req') - - ;; coerce the response body - (fn [{:keys [body] :as rsp}] - (let [rsp' (handle-response-debug req' rsp)] - (if (and (some? body) (some? (:as req'))) - (d/future-with (or executor (ex/wait-pool)) - (coerce-response-body req' rsp')) - rsp')))))))))) + ([client] + (wrap-request client default-client-middleware default-middleware)) + ([client client-middleware middleware] + (let [client' (reduce #(%2 %1) + client + client-middleware)] + (fn [req] + (let [executor (ex/executor)] + (if (:aleph.http.client/close req) + (client req) + + (let [req' (reduce #(%2 %1) req middleware)] + (d/chain' (client' req') + + ;; coerce the response body + (fn [{:keys [body] :as rsp}] + (let [rsp' (handle-response-debug req' rsp)] + (if (and (some? body) (some? (:as req'))) + (d/future-with (or executor (ex/wait-pool)) + (coerce-response-body req' rsp')) + rsp'))))))))))) diff --git a/src/aleph/http/core.clj b/src/aleph/http/core.clj index 493a41bb..bba08e27 100644 --- a/src/aleph/http/core.clj +++ b/src/aleph/http/core.clj @@ -224,9 +224,10 @@ :remote-addr (netty/channel-remote-address ch)) (p/def-derived-map NettyResponse [^HttpResponse rsp complete body] - :status (-> rsp .status .code) + :status (-> rsp (.status) (.code)) + :reason-phrase (-> rsp (.status) (.reasonPhrase)) :aleph/keep-alive? (HttpUtil/isKeepAlive rsp) - :headers (-> rsp .headers headers->map) + :headers (-> rsp (.headers) headers->map) :aleph/complete complete :body body) diff --git a/src/aleph/http/multipart.clj b/src/aleph/http/multipart.clj index 0f615385..4025fe57 100644 --- a/src/aleph/http/multipart.clj +++ b/src/aleph/http/multipart.clj @@ -201,7 +201,7 @@ (defn decode-request "Takes a ring request and returns a manifold stream which yields - parts of the mutlipart/form-data encoded body. In case the size of + parts of the multipart/form-data encoded body. In case the size of a part content exceeds `:memory-limit` limit (16KB by default), corresponding payload would be written to a temp file. Check `:memory?` flag to know whether content might be read directly from `:content` or diff --git a/test-resources/big_array_json.json b/test-resources/big_array_json.json new file mode 100644 index 00000000..51ccef7d --- /dev/null +++ b/test-resources/big_array_json.json @@ -0,0 +1,102 @@ +[ + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]}, + {"foo": "bar", "baz": "qux", "values": [1, 2, 3, 4, 5]} +] diff --git a/test-resources/keystore b/test-resources/keystore new file mode 100644 index 00000000..2944ca21 Binary files /dev/null and b/test-resources/keystore differ diff --git a/test-resources/m.txt b/test-resources/m.txt new file mode 100644 index 00000000..8e6ce25e --- /dev/null +++ b/test-resources/m.txt @@ -0,0 +1,4 @@ +this +is +some +file. diff --git a/test/aleph/http/clj_http/client_test.clj b/test/aleph/http/clj_http/client_test.clj new file mode 100644 index 00000000..ef7d54aa --- /dev/null +++ b/test/aleph/http/clj_http/client_test.clj @@ -0,0 +1,1755 @@ +(ns aleph.http.clj-http.client-test + (:require [aleph.http.clj-http.core-test :refer [run-server]] + [aleph.http.clj-http.util :refer [make-request]] + [cheshire.core :as json] + [clj-http.client :as client] + [clj-http.conn-mgr :as conn] + [clj-http.util :as util] + [clojure.java.io :refer [resource]] + [clojure.string :as str] + [clojure.test :refer :all] + [cognitect.transit :as transit] + [ring.middleware.nested-params :refer [parse-nested-keys]] + [ring.util.codec :refer [form-decode-str]] + [slingshot.slingshot :refer [try+]]) + (:import java.io.ByteArrayInputStream + java.net.UnknownHostException + org.apache.http.HttpEntity + org.apache.logging.log4j.LogManager)) + +(set! *warn-on-reflection* false) + +(defonce logger (LogManager/getLogger "clj-http.test.client-test")) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + +(def request (make-request client/request {:using-middleware? true})) + +(defn parse-form-params [s] + (->> (str/split (form-decode-str s) #"&") + (map #(str/split % #"=")) + (map #(vector + (map keyword (parse-nested-keys (first %))) + (second %))) + (reduce (fn [m [ks v]] + (assoc-in m ks v)) {}))) + +(deftest ^:integration roundtrip + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (request {:uri "/get" :method :get})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp)))) + ;; roundtrip with scheme as a string + (let [resp (request {:uri "/get" :method :get + :scheme "http"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp)))) + (let [params {:a "1" :b "2"}] + (doseq [[content-type read-fn] + [[nil (comp parse-form-params slurp)] + [:x-www-form-urlencoded (comp parse-form-params slurp)] + [:edn (comp read-string slurp)] + [:transit+json #(client/parse-transit % :json)] + [:transit+msgpack #(client/parse-transit % :msgpack)]]] + (let [resp (request {:uri "/post" + :as :stream + :method :post + :content-type content-type + :form-params params})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= params (read-fn (:body resp))) + (str "failed with content-type [" content-type "]")))))) + +(deftest ^:integration roundtrip-async + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (promise) + exception (promise) + _ (request {:uri "/get" :method :get + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= "get" (:body @resp))) + (is (not (realized? exception)))) + ;; roundtrip with scheme as a string + (let [resp (promise) + exception (promise) + _ (request {:uri "/get" :method :get + :scheme "http" + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= "get" (:body @resp))) + (is (not (realized? exception)))) + + (let [params {:a "1" :b "2"}] + (doseq [[content-type read-fn] + [[nil (comp parse-form-params slurp)] + [:x-www-form-urlencoded (comp parse-form-params slurp)] + [:edn (comp read-string slurp)] + [:transit+json #(client/parse-transit % :json)] + [:transit+msgpack #(client/parse-transit % :msgpack)]]] + (let [resp (promise) + exception (promise) + _ (request {:uri "/post" + :as :stream + :method :post + :content-type content-type + :flatten-nested-keys [] + :form-params params + :async? true} resp exception)] + (is (= 200 (:status @resp))) + (is (= "close" (get-in @resp [:headers "connection"]))) + (is (= params (read-fn (:body @resp)))) + (is (not (realized? exception))))))) + +(def ^:dynamic *test-dynamic-var* nil) + +(deftest ^:integration async-preserves-dynamic-variable-bindings + (run-server) + (let [expected-var "cat"] + (binding [*test-dynamic-var* expected-var] + (let [test-fn (fn [uri success-p fail-p] + (request {:uri uri + :method :get + :scheme "http" + :async? true} + (fn [_] + (deliver success-p *test-dynamic-var*) + (deliver fail-p :success)) + (fn [_] + (deliver success-p :fail) + (deliver fail-p *test-dynamic-var*))))] + (testing "dynamic variables on success responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/get" success-p fail-p) + (is (= @success-p expected-var *test-dynamic-var*)) + (is (= @fail-p :success) + "Verify that we went through the success path, not the failure"))) + + (testing "dynamic variables on fail responses" + (let [success-p (promise) + fail-p (promise)] + (test-fn "/json-bad" success-p fail-p) + (is (= @success-p :fail) + "Verify that we went through the failure path, not the success") + (is (= @fail-p expected-var *test-dynamic-var*)))))))) + +(deftest ^:integration multipart-async + (run-server) + (let [resp (promise) + exception (promise) + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "title" :content "some-file"} + {:name "Content/Type" :content "text/plain"} + {:name "file" + :content (clojure.java.io/file + "test-resources/m.txt")}]} + resp + exception)] + + (is (= 200 (:status @resp))) + (is (not (realized? exception))) + #_(when (realized? exception) (prn @exception))) + + ;; Regression Testing https://github.com/dakrone/clj-http/issues/560 + (testing "multipart uploads larger than 25kb" + (let [resp (promise) + exception (promise) + ;; assumption: file > 5kb + file (clojure.java.io/file "test-resources/big_array_json.json") + + _ (request {:uri "/post" :method :post + :async? true + :multipart [{:name "part-1" :content file} + {:name "part-2" :content file} + {:name "part-3" :content file} + {:name "part-4" :content file} + {:name "part-5" :content file}]} + resp + exception)] + (is (= 200 (:status (deref resp 500 :failed)))) + (is (not (realized? exception)))))) + +(deftest ^:integration nil-input + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/get nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/post nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/put nil))) + (is (thrown-with-msg? Exception #"Host URL cannot be nil" + (client/delete nil)))) + +(defn async-identity-client + "A async client which simply respond the request" + [request respond raise] + (respond request)) + +(defn is-passed [middleware req] + (let [client (middleware identity)] + (is (= req (client req))))) + +(defn is-passed-async [middleware req] + (let [client (middleware async-identity-client) + resp (promise) + exception (promise) + _ (client req resp exception)] + (is (= req @resp)) + (is (not (realized? exception))))) + +(defn is-applied [middleware req-in req-out] + (let [client (middleware identity)] + (is (= req-out (client req-in))))) + +(defn is-applied-async [middleware req-in req-out] + (let [client (middleware async-identity-client) + resp (promise) + exception (promise) + _ (client req-in resp exception)] + (is (= req-out @resp)) + (is (not (realized? exception))))) + +(deftest redirect-on-get + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))))) + +(deftest redirect-on-get-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception))))) + +(deftest relative-redirect-on-get + (let [client (fn [req] + (if (:redirects-count req) + {:status 200 + :req req} + {:status 302 + :headers {"location" "/bat"}})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))))) + +(deftest relative-redirect-on-get-async + (let [client (fn [req respond raise] + (respond (if (:redirects-count req) + {:status 200 + :req req} + {:status 302 + :headers {"location" "/bat"}}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception))))) + +(deftest trace-redirects-using-uri + (let [client (fn [req] {:status 200 :req req}) + r-client (-> client client/wrap-redirects) + resp (r-client {:scheme :http :server-name "example.com" :uri "/" + :request-method :get})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= [] (:trace-redirects resp))))) + +(deftest trace-redirects-using-uri-async + (let [client (fn [req respond raise] (respond {:status 200 :req req})) + r-client (-> client client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:scheme :http :server-name "example.com" :uri "/" + :request-method :get} resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= [] (:trace-redirects @resp))) + (is (not (realized? exception))))) + +(deftest redirect-without-location-header + (let [client (fn [req] + {:status 302 :body "no redirection here"}) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get})] + (is (= 302 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "no redirection here" (:body resp))))) + +(deftest redirect-without-location-header-async + (let [client (fn [req respond raise] + (respond {:status 302 :body "no redirection here"})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get} resp exception)] + (is (= 302 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "no redirection here" (:body @resp))) + (is (not (realized? exception))))) + +(deftest redirect-with-query-string + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat?x=y"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :query-params {:x "z"}})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat?x=y"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp)))) + (is (= "x=y" (:query-string (:req resp)))) + (is (nil? (:query-params (:req resp)))))) + +(deftest redirect-with-query-string-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat?x=y"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :query-params {:x "z"}} + resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat?x=y"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (= "x=y" (:query-string (:req @resp)))) + (is (nil? (:query-params (:req @resp)))) + (is (not (realized? exception))))) + +(deftest max-redirects + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :max-redirects 0})] + (is (= 302 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "http://example.net/bat" (get (:headers resp) "location"))))) + +(deftest max-redirects-async + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 302 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method :get :max-redirects 0} + resp exception)] + (is (= 302 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "http://example.net/bat" (get (:headers @resp) "location"))) + (is (not (realized? exception))))) + +(deftest redirect-303-to-get-on-any-method + (doseq [method [:get :head :post :delete :put :option]] + (let [client (fn [req] + (if (= "example.com" (:server-name req)) + {:status 303 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req})) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (r-client {:server-name "example.com" :url "http://example.com" + :request-method method})] + (is (= 200 (:status resp))) + (is (= :get (:request-method (:req resp)))) + (is (= :http (:scheme (:req resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects resp))) + (is (= "/bat" (:uri (:req resp))))))) + +(deftest redirect-303-to-get-on-any-method-async + (doseq [method [:get :head :post :delete :put :option]] + (let [client (fn [req respond raise] + (respond (if (= "example.com" (:server-name req)) + {:status 303 + :headers {"location" "http://example.net/bat"}} + {:status 200 + :req req}))) + r-client (-> client client/wrap-url client/wrap-redirects) + resp (promise) + exception (promise) + _ (r-client {:server-name "example.com" :url "http://example.com" + :request-method method} + resp exception)] + (is (= 200 (:status @resp))) + (is (= :get (:request-method (:req @resp)))) + (is (= :http (:scheme (:req @resp)))) + (is (= ["http://example.com" "http://example.net/bat"] + (:trace-redirects @resp))) + (is (= "/bat" (:uri (:req @resp)))) + (is (not (realized? exception)))))) + +(deftest pass-on-non-redirect + (let [client (fn [req] {:status 200 :body (:body req)}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com"})] + (is (= 200 (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= "ok" (:body resp))))) + +(deftest pass-on-non-redirect-async + (let [client (fn [req respond raise] + (respond {:status 200 :body (:body req)})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com"} resp exception)] + (is (= 200 (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= "ok" (:body @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-non-redirectable-methods + (doseq [method [:put :post :delete] + status [301 302 307 308]] + (let [client (fn [req] {:status status :body (:body req) + :headers {"location" "http://example.com/bat"}}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com" + :request-method method})] + (is (= status (:status resp))) + (is (= ["http://example.com"] (:trace-redirects resp))) + (is (= {"location" "http://example.com/bat"} (:headers resp))) + (is (= "ok" (:body resp)))))) + +(deftest pass-on-non-redirectable-methods-async + (doseq [method [:put :post :delete] + status [301 302 307 308]] + (let [client (fn [req respond raise] + (respond {:status status :body (:body req) + :headers {"location" "http://example.com/bat"}})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com" + :request-method method} resp exception)] + (is (= status (:status @resp))) + (is (= ["http://example.com"] (:trace-redirects @resp))) + (is (= {"location" "http://example.com/bat"} (:headers @resp))) + (is (= "ok" (:body @resp))) + (is (not (realized? exception)))))) + +(deftest force-redirects-on-non-redirectable-methods + (doseq [method [:put :post :delete] + [status expected-method] [[301 :get] [302 :get] [307 method]]] + (let [client (fn [{:keys [trace-redirects body] :as req}] + (if trace-redirects + {:status 200 :body body :trace-redirects trace-redirects + :req req} + {:status status :body body :req req + :headers {"location" "http://example.com/bat"}})) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :url "http://example.com" + :request-method method + :force-redirects true})] + (is (= 200 (:status resp))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects resp))) + (is (= "ok" (:body resp))) + (is (= expected-method (:request-method (:req resp))))))) + +(deftest force-redirects-on-non-redirectable-methods-async + (doseq [method [:put :post :delete] + [status expected-method] [[301 :get] [302 :get] [307 method]]] + (let [client (fn [{:keys [trace-redirects body] :as req} respond raise] + (respond (if trace-redirects + {:status 200 :body body + :trace-redirects trace-redirects + :req req} + {:status status :body body :req req + :headers {"location" + "http://example.com/bat"}}))) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :url "http://example.com" + :request-method method + :force-redirects true} resp exception)] + (is (= 200 (:status @resp))) + (is (= ["http://example.com" "http://example.com/bat"] + (:trace-redirects @resp))) + (is (= "ok" (:body @resp))) + (is (= expected-method (:request-method (:req @resp)))) + (is (not (realized? exception)))))) + +(deftest pass-on-follow-redirects-false + (let [client (fn [req] {:status 302 :body (:body req)}) + r-client (client/wrap-redirects client) + resp (r-client {:body "ok" :follow-redirects false})] + (is (= 302 (:status resp))) + (is (= "ok" (:body resp))) + (is (nil? (:trace-redirects resp))))) + +(deftest pass-on-follow-redirects-false-async + (let [client (fn [req respond raise] + (respond {:status 302 :body (:body req)})) + r-client (client/wrap-redirects client) + resp (promise) + exception (promise) + _ (r-client {:body "ok" :follow-redirects false} resp exception)] + (is (= 302 (:status @resp))) + (is (= "ok" (:body @resp))) + (is (nil? (:trace-redirects @resp))) + (is (not (realized? exception))))) + +(deftest throw-on-exceptional + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #"500" + (e-client {})))) + (let [client (fn [req] {:status 500 :body "foo"}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #":body" + (e-client {:throw-entire-message? true}))))) + +(deftest throw-on-custom-exceptional + (let [client (fn [req] {:status 201}) + e-client (client/wrap-exceptions client)] + (is (thrown-with-msg? Exception #"201" + (e-client {:unexceptional-status #{200}}))))) + +(deftest throw-type-field + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client)] + (try+ + (e-client {}) + (catch [:type :clj-http.client/unexceptional-status] _ + (is true)) + (catch Object _ + (is false ":type selector was not caught."))))) + +(deftest throw-on-exceptional-async + (let [client (fn [req respond raise] + (try + (respond {:status 500}) + (catch Throwable ex + (raise ex)))) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {} resp exception)] + (is (thrown-with-msg? Exception #"500" + (throw @exception)))) + (let [client (fn [req respond raise] + (try + (respond {:status 500 :body "foo"}) + (catch Throwable ex + (raise ex)))) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {:throw-entire-message? true} resp exception)] + (is (thrown-with-msg? Exception #":body" + (throw @exception))))) + +(deftest pass-on-non-exceptional + (let [client (fn [req] {:status 200}) + e-client (client/wrap-exceptions client) + resp (e-client {})] + (is (= 200 (:status resp))))) + +(deftest pass-on-custom-non-exceptional + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client) + resp (e-client {:unexceptional-status #{200 500}})] + (is (= 500 (:status resp))))) + +(deftest pass-on-non-exceptional-async + (let [client (fn [req respond raise] (respond {:status 200})) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {} resp exception)] + (is (= 200 (:status @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-exceptional-when-surpressed + (let [client (fn [req] {:status 500}) + e-client (client/wrap-exceptions client) + resp (e-client {:throw-exceptions false})] + (is (= 500 (:status resp))))) + +(deftest pass-on-exceptional-when-surpressed-async + (let [client (fn [req respond raise] (respond {:status 500})) + e-client (client/wrap-exceptions client) + resp (promise) + exception (promise) + _ (e-client {:throw-exceptions false} resp exception)] + (is (= 500 (:status @resp))) + (is (not (realized? exception))))) + +(deftest apply-on-compressed + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/gzip (util/utf8-bytes "foofoofoo")) + :headers {"content-encoding" "gzip"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "foofoofoo" (util/utf8-string (:body resp)))) + (is (= "gzip" (:orig-content-encoding resp))) + (is (= nil (get-in resp [:headers "content-encoding"]))))) + +(deftest apply-on-compressed-async + (let [client (fn [req respond raise] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + (respond {:body (util/gzip (util/utf8-bytes "foofoofoo")) + :headers {"content-encoding" "gzip"}})) + c-client (client/wrap-decompression client) + resp (promise) + exception (promise) + _ (c-client {} resp exception)] + (is (= "foofoofoo" (util/utf8-string (:body @resp)))) + (is (= "gzip" (:orig-content-encoding @resp))) + (is (= nil (get-in @resp [:headers "content-encoding"]))))) + +(deftest apply-on-deflated + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "barbarbar" (-> resp :body util/force-byte-array util/utf8-string)) + "string correctly inflated") + (is (= "deflate" (:orig-content-encoding resp))) + (is (= nil (get-in resp [:headers "content-encoding"]))))) + +(deftest apply-on-deflated-async + (let [client (fn [req respond raise] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + (respond {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}})) + c-client (client/wrap-decompression client) + resp (promise) + exception (promise) + _ (c-client {} resp exception)] + (is (= "barbarbar" (-> @resp :body util/force-byte-array util/utf8-string)) + "string correctly inflated") + (is (= "deflate" (:orig-content-encoding @resp))) + (is (= nil (get-in @resp [:headers "content-encoding"]))))) + +(deftest t-disabled-body-decompression + (let [client (fn [req] + (is (not= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/deflate (util/utf8-bytes "barbarbar")) + :headers {"content-encoding" "deflate"}}) + c-client (client/wrap-decompression client) + resp (c-client {:decompress-body false})] + (is (= (slurp (util/inflate (util/deflate (util/utf8-bytes "barbarbar")))) + (slurp (util/inflate (-> resp :body util/force-byte-array)))) + "string not inflated") + (is (= nil (:orig-content-encoding resp))) + (is (= "deflate" (get-in resp [:headers "content-encoding"]))))) + +(deftest t-weird-non-known-compression + (let [client (fn [req] + (is (= "gzip, deflate" + (get-in req [:headers "accept-encoding"]))) + {:body (util/utf8-bytes "foofoofoo") + :headers {"content-encoding" "pig-latin"}}) + c-client (client/wrap-decompression client) + resp (c-client {})] + (is (= "foofoofoo" (util/utf8-string (:body resp)))) + (is (= "pig-latin" (:orig-content-encoding resp))) + (is (= "pig-latin" (get-in resp [:headers "content-encoding"]))))) + +(deftest pass-on-non-compressed + (let [c-client (client/wrap-decompression (fn [req] {:body "foo"})) + resp (c-client {:uri "/foo"})] + (is (= "foo" (:body resp))))) + +(deftest apply-on-accept + (is-applied client/wrap-accept + {:accept :json} + {:headers {"accept" "application/json"}}) + (is-applied client/wrap-accept + {:accept :transit+json} + {:headers {"accept" "application/transit+json"}}) + (is-applied client/wrap-accept + {:accept :transit+msgpack} + {:headers {"accept" "application/transit+msgpack"}})) + +(deftest apply-on-accept-async + (is-applied-async client/wrap-accept + {:accept :json} + {:headers {"accept" "application/json"}}) + (is-applied-async client/wrap-accept + {:accept :transit+json} + {:headers {"accept" "application/transit+json"}}) + (is-applied-async client/wrap-accept + {:accept :transit+msgpack} + {:headers {"accept" "application/transit+msgpack"}})) + +(deftest pass-on-no-accept + (is-passed client/wrap-accept + {:uri "/foo"})) + +(deftest pass-on-no-accept-async + (is-passed-async client/wrap-accept + {:uri "/foo"})) + +(deftest apply-on-accept-encoding + (is-applied client/wrap-accept-encoding + {:accept-encoding [:identity :gzip]} + {:headers {"accept-encoding" "identity, gzip"}})) + +(deftest apply-custom-accept-encoding + (testing "no custom encodings to accept" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {} + {:headers {"accept-encoding" "gzip, deflate"} + :orig-content-encoding nil})) + (testing "accept some custom encodings, but still include gzip and deflate" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {:accept-encoding [:foo :bar]} + {:headers {"accept-encoding" "foo, bar, gzip, deflate"} + :orig-content-encoding nil})) + (testing "accept some custom encodings, but exclude gzip and deflate" + (is-applied (comp client/wrap-accept-encoding + client/wrap-decompression) + {:accept-encoding [:foo :bar] :decompress-body false} + {:headers {"accept-encoding" "foo, bar"} + :decompress-body false}))) + +(deftest pass-on-no-accept-encoding + (is-passed client/wrap-accept-encoding + {:uri "/foo"})) + +(deftest apply-on-output-coercion + (let [client (fn [req] {:body (util/utf8-bytes "foo")}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (= "foo" (:body resp))))) + +(deftest apply-on-output-coercion-async + (let [client (fn [req respond raise] + (respond {:body (util/utf8-bytes "foo")})) + o-client (client/wrap-output-coercion client) + resp (promise) + exception (promise) + _ (o-client {:uri "/foo"} resp exception)] + (is (= "foo" (:body @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-no-output-coercion + (let [client (fn [req] {:body nil}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (nil? (:body resp)))) + (let [the-stream (ByteArrayInputStream. (byte-array [])) + client (fn [req] {:body the-stream}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :stream})] + (is (= the-stream (:body resp)))) + (let [client (fn [req] {:body :thebytes}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :byte-array})] + (is (= :thebytes (:body resp))))) + +(deftest pass-on-no-output-coercion-async + (let [client (fn [req] {:body nil}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo"})] + (is (nil? (:body resp)))) + (let [the-stream (ByteArrayInputStream. (byte-array [])) + client (fn [req] {:body the-stream}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :stream})] + (is (= the-stream (:body resp)))) + (let [client (fn [req] {:body :thebytes}) + o-client (client/wrap-output-coercion client) + resp (o-client {:uri "/foo" :as :byte-array})] + (is (= :thebytes (:body resp))))) + +(deftest apply-on-input-coercion + (let [i-client (client/wrap-input-coercion identity) + resp (i-client {:body "foo"}) + resp2 (i-client {:body "foo2" :body-encoding "ASCII"}) + data (slurp (.getContent ^HttpEntity (:body resp))) + data2 (slurp (.getContent ^HttpEntity (:body resp2)))] + (is (= "UTF-8" (:character-encoding resp))) + (is (= "foo" data)) + (is (= "ASCII" (:character-encoding resp2))) + (is (= "foo2" data2)))) + +(deftest apply-on-input-coercion-async + (let [i-client (client/wrap-input-coercion (fn [request respond raise] + (respond request))) + resp (promise) + _ (i-client {:body "foo"} resp nil) + resp2 (promise) + _ (i-client {:body "foo2" :body-encoding "ASCII"} resp2 nil) + data (slurp (.getContent ^HttpEntity (:body @resp))) + data2 (slurp (.getContent ^HttpEntity (:body @resp2)))] + (is (= "UTF-8" (:character-encoding @resp))) + (is (= "foo" data)) + (is (= "ASCII" (:character-encoding @resp2))) + (is (= "foo2" data2)))) + +(deftest pass-on-no-input-coercion + (is-passed client/wrap-input-coercion + {:body nil})) + +(deftest pass-on-no-input-coercion + (is-passed-async client/wrap-input-coercion + {:body nil})) + +(deftest no-length-for-input-stream + (let [i-client (client/wrap-input-coercion identity) + resp1 (i-client {:body (ByteArrayInputStream. (util/utf8-bytes "foo"))}) + resp2 (i-client {:body (ByteArrayInputStream. (util/utf8-bytes "foo")) + :length 3}) + ^HttpEntity body1 (:body resp1) + ^HttpEntity body2 (:body resp2)] + (is (= -1 (.getContentLength body1))) + (is (= 3 (.getContentLength body2))))) + +(deftest apply-on-content-type + (is-applied client/wrap-content-type + {:content-type :json} + {:headers {"content-type" "application/json"} + :content-type :json}) + (is-applied client/wrap-content-type + {:content-type :json :character-encoding "UTF-8"} + {:headers {"content-type" "application/json; charset=UTF-8"} + :content-type :json :character-encoding "UTF-8"}) + (is-applied client/wrap-content-type + {:content-type :transit+json} + {:headers {"content-type" "application/transit+json"} + :content-type :transit+json}) + (is-applied client/wrap-content-type + {:content-type :transit+msgpack} + {:headers {"content-type" "application/transit+msgpack"} + :content-type :transit+msgpack})) + +(deftest apply-on-content-type-async + (is-applied-async client/wrap-content-type + {:content-type :json} + {:headers {"content-type" "application/json"} + :content-type :json}) + (is-applied-async client/wrap-content-type + {:content-type :json :character-encoding "UTF-8"} + {:headers {"content-type" "application/json; charset=UTF-8"} + :content-type :json :character-encoding "UTF-8"}) + (is-applied-async client/wrap-content-type + {:content-type :transit+json} + {:headers {"content-type" "application/transit+json"} + :content-type :transit+json}) + (is-applied-async client/wrap-content-type + {:content-type :transit+msgpack} + {:headers {"content-type" "application/transit+msgpack"} + :content-type :transit+msgpack})) + +(deftest pass-on-no-content-type + (is-passed client/wrap-content-type + {:uri "/foo"})) + +(deftest apply-on-query-params + (is-applied client/wrap-query-params + {:query-params {"foo" "bar" "dir" "<<"}} + {:query-string "foo=bar&dir=%3C%3C"}) + (is-applied client/wrap-query-params + {:query-string "foo=1" + :query-params {"foo" ["2" "3"]}} + {:query-string "foo=1&foo=2&foo=3"})) + +(deftest apply-on-query-params-async + (is-applied-async client/wrap-query-params + {:query-params {"foo" "bar" "dir" "<<"}} + {:query-string "foo=bar&dir=%3C%3C"}) + (is-applied-async client/wrap-query-params + {:query-string "foo=1" + :query-params {"foo" ["2" "3"]}} + {:query-string "foo=1&foo=2&foo=3"})) + +(deftest pass-on-no-query-params + (is-passed client/wrap-query-params + {:uri "/foo"})) + +(deftest apply-on-basic-auth + (is-applied client/wrap-basic-auth + {:basic-auth ["Aladdin" "open sesame"]} + {:headers {"authorization" + "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}})) + +(deftest apply-on-basic-auth-async + (is-applied-async client/wrap-basic-auth + {:basic-auth ["Aladdin" "open sesame"]} + {:headers {"authorization" + "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="}})) + +(deftest pass-on-no-basic-auth + (is-passed client/wrap-basic-auth + {:uri "/foo"})) + +(deftest apply-on-oauth + (is-applied client/wrap-oauth + {:oauth-token "my-token"} + {:headers {"authorization" + "Bearer my-token"}})) + +(deftest apply-on-oauth-async + (is-applied-async client/wrap-oauth + {:oauth-token "my-token"} + {:headers {"authorization" + "Bearer my-token"}})) + +(deftest pass-on-no-oauth + (is-passed client/wrap-oauth + {:uri "/foo"})) + +(deftest apply-on-method + (let [m-client (client/wrap-method identity) + echo (m-client {:key :val :method :post})] + (is (= :val (:key echo))) + (is (= :post (:request-method echo))) + (is (not (:method echo))))) + +(deftest apply-on-method-async + (let [m-client (client/wrap-method async-identity-client) + echo (promise) + exception (promise) + _ (m-client {:key :val :method :post} echo exception)] + (is (= :val (:key @echo))) + (is (= :post (:request-method @echo))) + (is (not (:method @echo))))) + +(deftest pass-on-no-method + (let [m-client (client/wrap-method identity) + echo (m-client {:key :val})] + (is (= :val (:key echo))) + (is (not (:request-method echo))))) + +(deftest apply-on-url + (let [u-client (client/wrap-url identity) + resp (u-client {:url "http://google.com:8080/baz foo?bar=bat bit?"})] + (is (= :http (:scheme resp))) + (is (= "google.com" (:server-name resp))) + (is (= 8080 (:server-port resp))) + (is (= "/baz%20foo" (:uri resp))) + (is (= "bar=bat%20bit?" (:query-string resp))))) + +(deftest apply-on-url + (let [u-client (client/wrap-url async-identity-client) + resp (promise) + exception (promise) + _ (u-client {:url "http://google.com:8080/baz foo?bar=bat bit?"} + resp exception)] + (is (= :http (:scheme @resp))) + (is (= "google.com" (:server-name @resp))) + (is (= 8080 (:server-port @resp))) + (is (= "/baz%20foo" (:uri @resp))) + (is (= "bar=bat%20bit?" (:query-string @resp))) + (is (not (realized? exception))))) + +(deftest pass-on-no-url + (let [u-client (client/wrap-url identity) + resp (u-client {:uri "/foo"})] + (is (= "/foo" (:uri resp))))) + +(deftest provide-default-port + (is (= nil (-> "http://example.com/" client/parse-url :server-port))) + (is (= 8080 (-> "http://example.com:8080/" client/parse-url :server-port))) + (is (= nil (-> "https://example.com/" client/parse-url :server-port))) + (is (= 8443 (-> "https://example.com:8443/" client/parse-url :server-port))) + (is (= "https://example.com:8443/" + (-> "https://example.com:8443/" client/parse-url :url)))) + +(deftest decode-credentials-from-url + (is (= "fred's diner:fred's password" + (-> "http://fred%27s%20diner:fred%27s%20password@example.com/foo" + client/parse-url + :user-info)))) + +(deftest unparse-url + (is (= "http://fred's diner:fred's password@example.com/foo" + (-> "http://fred%27s%20diner:fred%27s%20password@example.com/foo" + client/parse-url client/unparse-url))) + (is (= "https://foo:bar@example.org:8080" + (-> "https://foo:bar@example.org:8080" + client/parse-url client/unparse-url))) + (is (= "ftp://example.org?foo" + (-> "ftp://example.org?foo" + client/parse-url client/unparse-url)))) + +(defrecord Point [x y]) + +(def write-point + "Write a point in Transit format." + (transit/write-handler + (constantly "point") + (fn [point] [(:x point) (:y point)]) + (constantly nil))) + +(def read-point + "Read a point in Transit format." + (transit/read-handler + (fn [[x y]] + (->Point x y)))) + +(def transit-opts + "Transit read and write options." + {:encode {:handlers {Point write-point}} + :decode {:handlers {"point" read-point}}}) + +(def transit-opts-deprecated + "Deprecated Transit read and write options." + {:handlers {Point write-point "point" read-point}}) + +(deftest apply-on-form-params + (testing "With form params" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :post + :form-params (sorted-map :param1 "value1" + :param2 "value2")})] + (is (= "param1=value1¶m2=value2" (:body resp))) + (is (= "application/x-www-form-urlencoded" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :put + :form-params (sorted-map :param1 "value1" + :param2 "value2")})] + (is (= "param1=value1¶m2=value2" (:body resp))) + (is (= "application/x-www-form-urlencoded" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "With json form params" + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :post + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :put + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method :patch + :content-type :json + :form-params params})] + (is (= (json/encode params) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params)))) + (let [param-client (client/wrap-form-params identity) + params {:param1 (java.util.Date. (long 0))} + resp (param-client {:request-method :put + :content-type :json + :form-params params + :json-opts {:date-format "yyyy-MM-dd"}})] + (is (= (json/encode params {:date-format "yyyy-MM-dd"}) (:body resp))) + (is (= "application/json" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "With EDN form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method method + :content-type :edn + :form-params params})] + (is (= (pr-str params) (:body resp))) + (is (= "application/edn" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/JSON form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method method + :content-type :transit+json + :form-params params + :transit-opts transit-opts})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :json transit-opts))) + (is (= "application/transit+json" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/MessagePack form params" + (doseq [method [:post :put :patch]] + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 "value2"} + resp (param-client {:request-method method + :content-type :transit+msgpack + :form-params params + :transit-opts transit-opts})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :msgpack transit-opts))) + (is (= "application/transit+msgpack" (:content-type resp))) + (is (not (contains? resp :form-params)))))) + + (testing "With Transit/JSON form params and deprecated options" + (let [param-client (client/wrap-form-params identity) + params {:param1 "value1" :param2 (Point. 1 2)} + resp (param-client {:request-method :post + :content-type :transit+json + :form-params params + :transit-opts transit-opts-deprecated})] + (is (= params (client/parse-transit + (ByteArrayInputStream. (:body resp)) + :json transit-opts-deprecated))) + (is (= "application/transit+json" (:content-type resp))) + (is (not (contains? resp :form-params))))) + + (testing "Ensure it does not affect GET requests" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:request-method :get + :body "untouched" + :form-params {:param1 "value1" + :param2 "value2"}})] + (is (= "untouched" (:body resp))) + (is (not (contains? resp :content-type))))) + + (testing "with no form params" + (let [param-client (client/wrap-form-params identity) + resp (param-client {:body "untouched"})] + (is (= "untouched" (:body resp))) + (is (not (contains? resp :content-type)))))) + +(deftest apply-on-form-params-async + (testing "With form params" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :post + :form-params (sorted-map :param1 "value1" + :param2 "value2")} + resp exception)] + (is (= "param1=value1¶m2=value2" (:body @resp))) + (is (= "application/x-www-form-urlencoded" (:content-type @resp))) + (is (not (contains? @resp :form-params))) + (is (not (realized? exception)))) + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :put + :form-params (sorted-map :param1 "value1" + :param2 "value2")} + resp exception)] + (is (= "param1=value1¶m2=value2" (:body @resp))) + (is (= "application/x-www-form-urlencoded" (:content-type @resp))) + (is (not (contains? @resp :form-params))) + (is (not (realized? exception))))) + + (testing "Ensure it does not affect GET requests" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:request-method :get + :body "untouched" + :form-params {:param1 "value1" + :param2 "value2"}} + resp exception)] + (is (= "untouched" (:body @resp))) + (is (not (contains? @resp :content-type))) + (is (not (realized? exception))))) + + (testing "with no form params" + (let [param-client (client/wrap-form-params async-identity-client) + resp (promise) + exception (promise) + _ (param-client {:body "untouched"} resp exception)] + (is (= "untouched" (:body @resp))) + (is (not (contains? @resp :content-type))) + (is (not (realized? exception)))))) + +(deftest apply-on-nested-params + (testing "nested parameter maps" + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"foo" "bar"} + :form-params {"foo" "bar"} + :flatten-nested-keys [:query-params :form-params]} + {:query-params {"foo" "bar"} + :form-params {"foo" "bar"} + :flatten-nested-keys [:query-params :form-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"x" {"y" "z"}} + :form-params {"x" {"y" "z"}} + :flatten-nested-keys [:query-params]} + {:query-params {"x[y]" "z"} + :form-params {"x" {"y" "z"}} + :flatten-nested-keys [:query-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a" {"b" {"c" "d"}}} + :flatten-nested-keys [:form-params]} + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a[b][c]" "d"} + :flatten-nested-keys [:form-params]}) + (is-applied (comp client/wrap-form-params + client/wrap-nested-params) + {:query-params {"a" {"b" {"c" "d"}}} + :form-params {"a" {"b" {"c" "d"}}} + :flatten-nested-keys [:query-params :form-params]} + {:query-params {"a[b][c]" "d"} + :form-params {"a[b][c]" "d"} + :flatten-nested-keys [:query-params :form-params]})) + + (testing "not creating empty param maps" + (is-applied client/wrap-query-params {} {}))) + +(deftest t-ignore-unknown-host + (is (thrown? UnknownHostException (client/get "http://example.invalid"))) + (is (nil? (client/get "http://example.invalid" + {:ignore-unknown-host? true})))) + +(deftest t-ignore-unknown-host-async + (let [resp (promise) exception (promise)] + (client/get "http://example.invalid" + {:async? true} resp exception) + (is (thrown? UnknownHostException (throw @exception)))) + (let [resp (promise) exception (promise)] + (client/get "http://example.invalid" + {:ignore-unknown-host? true + :async? true} resp exception) + (is (nil? @resp)))) + +(deftest test-status-predicates + (testing "2xx statuses" + (doseq [s (range 200 299)] + (is (client/success? {:status s})) + (is (not (client/redirect? {:status s}))) + (is (not (client/client-error? {:status s}))) + (is (not (client/server-error? {:status s}))))) + (testing "3xx statuses" + (doseq [s (range 300 399)] + (is (not (client/success? {:status s}))) + (is (client/redirect? {:status s})) + (is (not (client/client-error? {:status s}))) + (is (not (client/server-error? {:status s}))))) + (testing "4xx statuses" + (doseq [s (range 400 499)] + (is (not (client/success? {:status s}))) + (is (not (client/redirect? {:status s}))) + (is (client/client-error? {:status s})) + (is (not (client/server-error? {:status s}))))) + (testing "5xx statuses" + (doseq [s (range 500 599)] + (is (not (client/success? {:status s}))) + (is (not (client/redirect? {:status s}))) + (is (not (client/client-error? {:status s}))) + (is (client/server-error? {:status s})))) + (testing "409 Conflict" + (is (client/conflict? {:status 409})) + (is (not (client/conflict? {:status 201}))) + (is (not (client/conflict? {:status 404}))))) + +(deftest test-wrap-lower-case-headers + (is (= {:status 404} ((client/wrap-lower-case-headers + (fn [r] r)) {:status 404}))) + (is (= {:headers {"content-type" "application/json"}} + ((client/wrap-lower-case-headers + #(do (is (= {:headers {"accept" "application/json"}} %1)) + {:headers {"Content-Type" "application/json"}})) + {:headers {"Accept" "application/json"}})))) + +(deftest t-request-timing + (is (pos? (:request-time ((client/wrap-request-timing + (fn [r] (Thread/sleep 15) r)) {}))))) + +(deftest t-wrap-additional-header-parsing + (let [^String text (slurp (resource "header-test.html")) + client (fn [req] {:body (.getBytes text)}) + new-client (client/wrap-additional-header-parsing client) + resp (new-client {:decode-body-headers true}) + resp2 (new-client {:decode-body-headers false}) + resp3 ((client/wrap-additional-header-parsing + (fn [req] {:body nil})) {:decode-body-headers true}) + resp4 ((client/wrap-additional-header-parsing + (fn [req] {:headers {"content-type" "application/pdf"} + :body (.getBytes text)})) + {:decode-body-headers true})] + (is (= {"content-type" "text/html; charset=Shift_JIS" + "content-style-type" "text/css" + "content-script-type" "text/javascript"} + (:headers resp))) + (is (nil? (:headers resp2))) + (is (nil? (:headers resp3))) + (is (= {"content-type" "application/pdf"} (:headers resp4))))) + +(deftest t-wrap-additional-header-parsing-html5 + (let [^String text (slurp (resource "header-html5-test.html")) + client (fn [req] {:body (.getBytes text)}) + new-client (client/wrap-additional-header-parsing client) + resp (new-client {:decode-body-headers true})] + (is (= {"content-type" "text/html; charset=UTF-8"} + (:headers resp))))) + +(deftest ^:integration t-request-without-url-set + (run-server) + ;; roundtrip with scheme as a keyword + (let [resp (request {:uri "/redirect-to-get" + :method :get})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "get" (:body resp))))) + +(deftest ^:integration t-reusable-conn-mgrs + (run-server) + (let [cm (conn/make-reusable-conn-manager {:timeout 10 :insecure? false}) + resp1 (request {:uri "/redirect-to-get" + :method :get + :connection-manager cm}) + resp2 (request {:uri "/redirect-to-get" + :method :get})] + (is (= 200 (:status resp1) (:status resp2))) + (is (nil? (get-in resp1 [:headers "connection"])) + "connection should remain open") + (is (= "close" (get-in resp2 [:headers "connection"])) + "connection should be closed") + (.shutdown cm))) + +(deftest ^:integration t-reusable-async-conn-mgrs + (run-server) + (let [cm (conn/make-reuseable-async-conn-manager {:timeout 10 :insecure? false}) + resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/redirect-to-get" :method :get :connection-manager cm} + resp1 + exce1) + (request {:async? true :uri "/redirect-to-get" :method :get} + resp2 + exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (nil? (get-in @resp1 [:headers "connection"])) + "connection should remain open") + (is (= "close" (get-in @resp2 [:headers "connection"])) + "connection should be closed") + (is (not (realized? exce2))) + (is (not (realized? exce1))) + (.shutdown cm))) + +(deftest ^:integration t-with-async-pool + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} resp1 exce1) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-with-async-pool-sleep + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} resp1 exce1) + (Thread/sleep 500) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-async-pool-wrap-exception + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise) count (atom 2)] + (request {:async? true :uri "/error" :method :get} resp1 exce1) + (Thread/sleep 500) + (request {:async? true :uri "/get" :method :get} resp2 exce2) + (is (realized? exce1)) + (is (not (realized? exce2))) + (is (= 200 (:status @resp2)))))) + +(deftest ^:integration t-async-pool-exception-when-start + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise) + middleware (fn [client] + (fn [req resp raise] (throw (Exception.))))] + (client/with-additional-middleware + [middleware] + (try (request {:async? true :uri "/error" :method :get} resp1 exce1) + (catch Throwable ex)) + (Thread/sleep 500) + (try (request {:async? true :uri "/get" :method :get} resp2 exce2) + (catch Throwable ex)) + (is (not (realized? exce1))) + (is (not (realized? exce2))) + (is (not (realized? resp1))) + (is (not (realized? resp2))))))) + +(deftest ^:integration t-reuse-async-pool + (run-server) + (client/with-async-connection-pool {} + (let [resp1 (promise) resp2 (promise) + exce1 (promise) exce2 (promise)] + (request {:async? true :uri "/get" :method :get} + (fn [resp] + (resp1 resp) + (request (client/reuse-pool + {:async? true + :uri "/get" + :method :get} + resp) + resp2 + exce2)) + exce1) + (is (= 200 (:status @resp1) (:status @resp2))) + (is (not (realized? exce2))) + (is (not (realized? exce1)))))) + +(deftest ^:integration t-async-pool-redirect-to-get + (run-server) + (client/with-async-connection-pool {} + (let [resp (promise) + exce (promise)] + (request {:async? true :uri "/redirect-to-get" + :method :get :redirect-strategy :default} resp exce) + (is (= 200 (:status @resp))) + (is (not (realized? exce)))))) + +(deftest ^:integration t-async-pool-max-redirect + (run-server) + (client/with-async-connection-pool {} + (let [resp (promise) + exce (promise)] + (request {:async? true :uri "/redirect" :method :get + :redirect-strategy :default + :throw-exceptions true} resp exce) + (is @exce) + (is (not (realized? resp)))))) + +(deftest test-url-encode-path + (is (= (client/url-encode-illegal-characters "?foo bar+baz[]75") + "?foo%20bar+baz%5B%5D75")) + (is (= {:uri (str "/:@-._~!$&'()*+,=" + ";" + ":@-._~!$&'()*+," + "=" + ":@-._~!$&'()*+,==") + :query-string (str "/?:@-._~!$'()*+,;" + "=" + "/?:@-._~!$'()*+,;==")} + ;; This URL sucks, yes, it's actually a valid URL + (select-keys (client/parse-url + (str "http://example.com/:@-._~!$&'()*+,=;:@-._~!$&'()*+" + ",=:@-._~!$&'()*+,==?/?:@-._~!$'()*+,;=/?:@-._~!$'(" + ")*+,;==#/?:@-._~!$&'()*+,;=")) + [:uri :query-string]))) + (let [all-chars (apply str (map char (range 256))) + all-legal (client/url-encode-illegal-characters all-chars)] + (is (= all-legal + (client/url-encode-illegal-characters all-legal))))) + +(defmethod client/coerce-response-body :json+ms949 + [req resp] + (client/coerce-json-body req resp true "MS949")) + +(deftest t-coercion-methods + (let [json-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) + json-ms949-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"안뇽\"}" "MS949")) + auto-body (ByteArrayInputStream. (.getBytes "{\"foo\":\"bar\"}")) + edn-body (ByteArrayInputStream. (.getBytes "{:foo \"bar\"}")) + transit-json-body (ByteArrayInputStream. + (.getBytes "[\"^ \",\"~:foo\",\"bar\"]")) + transit-msgpack-body (->> (map byte [-127 -91 126 58 102 111 + 111 -93 98 97 114]) + (byte-array 11) + (ByteArrayInputStream.)) + www-form-urlencoded-body (ByteArrayInputStream. (.getBytes "foo=bar")) + auto-www-form-urlencoded-body + (ByteArrayInputStream. (.getBytes "foo=bar")) + json-resp {:body json-body :status 200 + :headers {"content-type" "application/json"}} + json-ms949-resp {:body json-ms949-body :status 200 + :headers {"content-type" "application/json; charset=ms949"}} + auto-resp {:body auto-body :status 200 + :headers {"content-type" "application/json"}} + edn-resp {:body edn-body :status 200 + :headers {"content-type" "application/edn"}} + transit-json-resp {:body transit-json-body :status 200 + :headers {"content-type" "application/transit-json"}} + transit-msgpack-resp {:body transit-msgpack-body :status 200 + :headers {"content-type" + "application/transit-msgpack"}} + www-form-urlencoded-resp + {:body www-form-urlencoded-body :status 200 + :headers {"content-type" + "application/x-www-form-urlencoded"}} + auto-www-form-urlencoded-resp + {:body auto-www-form-urlencoded-body :status 200 + :headers {"content-type" + "application/x-www-form-urlencoded"}}] + (is (= {:foo "bar"} + (:body (client/coerce-response-body {:as :json} json-resp)) + (:body (client/coerce-response-body {:as :clojure} edn-resp)) + (:body (client/coerce-response-body {:as :auto} auto-resp)) + (:body (client/coerce-response-body {:as :transit+json} + transit-json-resp)) + (:body (client/coerce-response-body {:as :transit+msgpack} + transit-msgpack-resp)) + (:body (client/coerce-response-body {:as :auto} + auto-www-form-urlencoded-resp)) + (:body (client/coerce-response-body {:as :x-www-form-urlencoded} + www-form-urlencoded-resp)))) + (is (= {:foo "안뇽"} + (:body (client/coerce-response-body {:as :json+ms949} json-ms949-resp)))) + + (testing "throws AssertionError when optional libraries are not loaded" + (with-redefs [client/json-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :json} json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} json-resp)))) + (with-redefs [client/transit-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+json} transit-json-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :transit+msgpack} transit-msgpack-resp)))) + (with-redefs [client/ring-codec-enabled? false] + (is (thrown? AssertionError (client/coerce-response-body {:as :x-www-form-urlencoded} www-form-urlencoded-resp))) + (is (thrown? AssertionError (client/coerce-response-body {:as :auto} auto-www-form-urlencoded-resp))))))) + + +(deftest t-reader-coercion + (let [read-lines (fn [reader] (vec (take-while not-empty (repeatedly #(.readLine reader))))) + reader-body (ByteArrayInputStream. (.getBytes "foo\nbar\n")) + reader-resp {:body reader-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}} + encoded-body (ByteArrayInputStream. (byte-array [0xA9])) + encoded-resp {:body encoded-body :status 200 :headers {"content-type" "text/plain; charset=iso-8859-1"}} + utf8-body (ByteArrayInputStream. (byte-array [0xC2 0xA9])) + utf8-resp {:body utf8-body :status 200 :headers {"content-type" "text/plain; charset=utf-8"}}] + (is (= ["foo" "bar"] + (read-lines (:body (client/coerce-response-body {:as :reader} reader-resp))))) + + (is (= "©" + (.readLine (:body (client/coerce-response-body {:as :reader} encoded-resp))) + (.readLine (:body (client/coerce-response-body {:as :reader} utf8-resp))))))) + +(deftest ^:integration t-with-middleware + (run-server) + (is (:request-time (request {:uri "/get" :method :get}))) + (is (= client/*current-middleware* client/default-middleware)) + (client/with-middleware [client/wrap-url + client/wrap-method + #'client/wrap-request-timing] + (is (:request-time (request {:uri "/get" :method :get}))) + (is (= client/*current-middleware* [client/wrap-url + client/wrap-method + #'client/wrap-request-timing]))) + (client/with-middleware (->> client/default-middleware + (remove #{client/wrap-request-timing})) + (is (not (:request-time (request {:uri "/get" :method :get})))) + (is (not (contains? (set client/*current-middleware*) + client/wrap-request-timing))) + (is (contains? (set client/default-middleware) + client/wrap-request-timing)))) + +(deftest t-detect-charset-by-content-type + (is (= "UTF-8" (client/detect-charset nil))) + (is (= "UTF-8"(client/detect-charset "application/json"))) + (is (= "UTF-8"(client/detect-charset "text/html"))) + (is (= "GBK"(client/detect-charset "application/json; charset=GBK"))) + (is (= "ISO-8859-1" (client/detect-charset + "application/json; charset=ISO-8859-1"))) + (is (= "ISO-8859-1" (client/detect-charset + "application/json; charset = ISO-8859-1"))) + (is (= "GB2312" (client/detect-charset "text/html; Charset=GB2312")))) + +(deftest ^:integration customMethodTest + (run-server) + (let [resp (request {:uri "/propfind" :method "PROPFIND"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "propfind" (:body resp)))) + (let [resp (request {:uri "/propfind-with-body" + :method "PROPFIND" + :body "propfindbody"})] + (is (= 200 (:status resp))) + (is (= "close" (get-in resp [:headers "connection"]))) + (is (= "propfindbody" (:body resp))))) + +(deftest ^:integration status-line-parsing + (run-server) + (let [resp (request {:uri "/get" :method :get}) + protocol-version (:protocol-version resp)] + (is (= 200 (:status resp))) + (is (= "HTTP" (:name protocol-version))) + (is (= 1 (:major protocol-version))) + (is (= 1 (:minor protocol-version))) + (is (= "OK" (:reason-phrase resp))))) + +(deftest ^:integration multi-valued-query-params + (run-server) + (testing "default (repeating) multi-valued query params" + (let [resp (request {:uri "/query-string" + :method :get + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a=1&a=2&a=3") query-string) + (is (.contains query-string "b=x&b=y&b=z") query-string))) + + (testing "multi-valued query params in indexed-style" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :indexed + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a[0]=1&a[1]=2&a[2]=3") query-string) + (is (.contains query-string "b[0]=x&b[1]=y&b[2]=z") query-string))) + + (testing "multi-valued query params in array-style" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :array + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a[]=1&a[]=2&a[]=3") query-string) + (is (.contains query-string "b[]=x&b[]=y&b[]=z") query-string))) + (testing "multi-valued query params in comma-separated" + (let [resp (request {:uri "/query-string" + :method :get + :multi-param-style :comma-separated + :query-params {:a [1 2 3] + :b ["x" "y" "z"]}}) + query-string (-> resp :body form-decode-str)] + (is (= 200 (:status resp))) + (is (.contains query-string "a=1,2,3") query-string) + (is (.contains query-string "b=x,y,z") query-string)))) + +(deftest t-wrap-flatten-nested-params + (is-applied client/wrap-flatten-nested-params + {} + {:flatten-nested-keys [:query-params]}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-keys []} + {:flatten-nested-keys []}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-keys [:foo]} + {:flatten-nested-keys [:foo]}) + (is-applied client/wrap-flatten-nested-params + {:ignore-nested-query-string true} + {:ignore-nested-query-string true + :flatten-nested-keys []}) + (is-applied client/wrap-flatten-nested-params + {} + {:flatten-nested-keys '(:query-params)}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-form-params true} + {:flatten-nested-form-params true + :flatten-nested-keys '(:query-params :form-params)}) + (is-applied client/wrap-flatten-nested-params + {:flatten-nested-form-params true + :ignore-nested-query-string true} + {:ignore-nested-query-string true + :flatten-nested-form-params true + :flatten-nested-keys '(:form-params)}) + (try + ((client/wrap-flatten-nested-params identity) + {:flatten-nested-form-params true + :ignore-nested-query-string true + :flatten-nested-keys [:thing :bar]}) + (is false "should have thrown exception") + (catch IllegalArgumentException e + (is (= (.getMessage e) + (str "only :flatten-nested-keys or :ignore-nested-query-string/" + ":flatten-nested-keys may be specified, not both"))))) + (try + ((client/wrap-flatten-nested-params identity) + {:ignore-nested-query-string true + :flatten-nested-keys [:thing :bar]}) + (is false "should have thrown exception") + (catch IllegalArgumentException e + (is (= (.getMessage e) + (str "only :flatten-nested-keys or :ignore-nested-query-string/" + ":flatten-nested-keys may be specified, not both")))))) diff --git a/test/aleph/http/clj_http/core_test.clj b/test/aleph/http/clj_http/core_test.clj new file mode 100644 index 00000000..c0920026 --- /dev/null +++ b/test/aleph/http/clj_http/core_test.clj @@ -0,0 +1,1003 @@ +(ns aleph.http.clj-http.core-test + (:require [aleph.http.clj-http.util :refer [make-request]] + [cheshire.core :as json] + [clj-http.client :as client] + [clj-http.conn-mgr :as conn] + [clj-http.core :as core] + [clj-http.util :as util] + [clojure.java.io :refer [file]] + [clojure.test :refer :all] + [ring.adapter.jetty :as ring]) + (:import (java.io ByteArrayInputStream ByteArrayOutputStream FilterInputStream InputStream) + (java.net InetAddress SocketTimeoutException) + (java.util.concurrent TimeoutException TimeUnit) + (org.apache.http HttpConnection HttpInetConnection HttpRequest HttpResponse ProtocolException) + org.apache.http.client.config.RequestConfig + org.apache.http.client.params.ClientPNames + org.apache.http.client.protocol.HttpClientContext + org.apache.http.impl.conn.InMemoryDnsResolver + org.apache.http.impl.cookie.RFC6265CookieSpecProvider + [org.apache.http.message BasicHeader BasicHeaderIterator] + [org.apache.http.params CoreConnectionPNames CoreProtocolPNames] + [org.apache.http.protocol ExecutionContext HttpContext] + org.apache.logging.log4j.LogManager + sun.security.provider.certpath.SunCertPathBuilderException)) + +(set! *warn-on-reflection* false) + +(defonce logger (LogManager/getLogger "clj-http.test.core-test")) + +(defn handler [req] + (condp = [(:request-method req) (:uri req)] + [:get "/get"] + {:status 200 :body "get"} + [:get "/dont-cache"] + {:status 200 :body "nocache" + :headers {"cache-control" "private"}} + [:get "/empty"] + {:status 200 :body nil} + [:get "/empty-gzip"] + {:status 200 :body nil + :headers {"content-encoding" "gzip"}} + [:get "/clojure"] + {:status 200 :body "{:foo \"bar\" :baz 7M :eggplant {:quux #{1 2 3}}}" + :headers {"content-type" "application/clojure"}} + [:get "/edn"] + {:status 200 :body "{:foo \"bar\" :baz 7M :eggplant {:quux #{1 2 3}}}" + :headers {"content-type" "application/edn"}} + [:get "/clojure-bad"] + {:status 200 :body "{:foo \"bar\" :baz #=(+ 1 1)}" + :headers {"content-type" "application/clojure"}} + [:get "/json"] + {:status 200 :body "{\"foo\":\"bar\"}" + :headers {"content-type" "application/json"}} + [:get "/json-array"] + {:status 200 :body "[\"foo\", \"bar\"]" + :headers {"content-type" "application/json"}} + [:get "/json-large-array"] + {:status 200 :body (file "test-resources/big_array_json.json") + :headers {"content-type" "application/json"}} + [:get "/json-bad"] + {:status 400 :body "{\"foo\":\"bar\"}"} + [:get "/redirect"] + {:status 302 + :headers {"location" "http://localhost:18080/redirect"}} + [:get "/bad-redirect"] + {:status 301 :headers {"location" "https:///"}} + [:get "/redirect-to-get"] + {:status 302 + :headers {"location" "http://localhost:18080/get"}} + [:get "/unmodified-resource"] + {:status 304} + [:get "/transit-json"] + {:status 200 :body (str "[\"^ \",\"~:eggplant\",[\"^ \",\"~:quux\"," + "[\"~#set\",[1,3,2]]],\"~:baz\",\"~f7\"," + "\"~:foo\",\"bar\"]") + :headers {"content-type" "application/transit+json"}} + [:get "/transit-json-bad"] + {:status 400 :body "[\"^ \", \"~:foo\",\"bar\"]"} + [:get "/transit-json-empty"] + {:status 200 + :headers {"content-type" "application/transit+json"}} + [:get "/transit-msgpack"] + {:status 200 + :body (->> [-125 -86 126 58 101 103 103 112 108 97 110 116 -127 -90 126 + 58 113 117 117 120 -110 -91 126 35 115 101 116 -109 1 3 2 + -91 126 58 98 97 122 -93 126 102 55 -91 126 58 102 111 111 + -93 98 97 114] + (map byte) + (byte-array) + (ByteArrayInputStream.)) + :headers {"content-type" "application/transit+msgpack"}} + [:head "/head"] + {:status 200} + [:get "/content-type"] + {:status 200 :body (:content-type req)} + [:get "/header"] + {:status 200 :body (get-in req [:headers "x-my-header"])} + [:post "/post"] + {:status 200 :body (:body req)} + [:get "/error"] + {:status 500 :body "o noes"} + [:get "/timeout"] + (do + (Thread/sleep 10) + {:status 200 :body "timeout"}) + [:delete "/delete-with-body"] + {:status 200 :body "delete-with-body"} + [:post "/multipart"] + {:status 200 :body (:body req) + :headers {"x-original-content-type" (get-in req [:headers "content-type"] "not found")}} + [:get "/get-with-body"] + {:status 200 :body (:body req)} + [:options "/options"] + {:status 200 :body "options"} + [:copy "/copy"] + {:status 200 :body "copy"} + [:move "/move"] + {:status 200 :body "move"} + [:patch "/patch"] + {:status 200 :body "patch"} + [:get "/headers"] + {:status 200 :body (json/encode (:headers req))} + [:propfind "/propfind"] + {:status 200 :body "propfind"} + [:propfind "/propfind-with-body"] + {:status 200 :body (:body req)} + [:get "/query-string"] + {:status 200 :body (:query-string req)} + [:get "/cookie"] + {:status 200 :body "yay" :headers {"Set-Cookie" "foo=bar"}} + [:get "/bad-cookie"] + {:status 200 :body "yay" + :headers + {"Set-Cookie" + (str "DD-PSHARD=3; expires=\"Thu, 12-Apr-2018 06:40:25 GMT\"; " + "Max-Age=604800; Path=/; secure; HttpOnly")}})) + +(defn add-headers-if-requested [client] + (fn [req] + (let [resp (client req) + add-all (-> req :headers (get "add-headers")) + headers (:headers resp)] + (if add-all + (assoc resp :headers (assoc headers "got" (pr-str (:headers req)))) + resp)))) + +(defn run-server + [] + (defonce ^org.eclipse.jetty.server.Server server + (ring/run-jetty (add-headers-if-requested #'handler) {:port 18080 :join? false}))) + +(defn localhost [path] + (str "http://localhost:18080" path)) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + + +(def request (make-request core/request {:using-middleware? false})) + +(use-fixtures :once + (fn [f] + (binding [client/request (make-request client/request {:using-middleware? true})] + (f)))) + +(defn slurp-body [req] + (slurp (:body req))) + +(deftest ^:integration makes-get-request + (run-server) + (let [resp (request {:request-method :get :uri "/get"})] + (is (= 200 (:status resp))) + (is (= "get" (slurp-body resp))))) + +(deftest ^:ignore dns-resolver + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + resp (request {:request-method :get :uri "/get" + :server-name "foo.bar.com" + :dns-resolver custom-dns-resolver})] + (is (= 200 (:status resp))) + (is (= "get" (slurp-body resp))))) + +(deftest ^:ignore dns-resolver-unknown-host + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "foo.bar.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))])))] + (is (thrown? java.net.UnknownHostException (request {:request-method :get :uri "/get" + :server-name "www.google.com" + :dns-resolver custom-dns-resolver}))))) + +(deftest ^:ignore dns-resolver-reusable-connection-manager + (run-server) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "totallynonexistant.google.com" + (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reuseable-async-conn-manager {:dns-resolver custom-dns-resolver}) + hc (core/build-async-http-client {} cm)] + (client/get "http://totallynonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json + :async true} + (fn [resp] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp)))) + (fn [e] (is false (str "failed with " e))))) + (let [custom-dns-resolver (doto (InMemoryDnsResolver.) + (.add "nonexistant.google.com" (into-array[(InetAddress/getByAddress (byte-array [127 0 0 1]))]))) + cm (conn/make-reusable-conn-manager {:dns-resolver custom-dns-resolver}) + hc (:http-client (client/get "http://nonexistant.google.com:18080/get" + {:connection-manager cm})) + resp (client/get "http://nonexistant.google.com:18080/json" + {:connection-manager cm + :http-client hc + :as :json})] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))))) + +(deftest ^:integration save-request-option + (run-server) + (let [resp (request {:request-method :post + :uri "/post" + :body (util/utf8-bytes "contents") + :save-request? true})] + (is (= "/post" (-> resp :request :uri))))) + +(deftest ^:integration makes-head-request + (run-server) + (let [resp (request {:request-method :head :uri "/head"})] + (is (= 200 (:status resp))) + (is (nil? (:body resp))))) + +(deftest ^:integration sets-content-type-with-charset + (run-server) + (let [resp (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/content-type" + :content-type "text/plain" + :character-encoding "UTF-8"})] + (is (= "text/plain; charset=UTF-8" (:body resp))))) + +(deftest ^:integration sets-content-type-without-charset + (run-server) + (let [resp (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/content-type" + :content-type "text/plain"})] + (is (= "text/plain" (:body resp))))) + +(deftest ^:integration sets-arbitrary-headers + (run-server) + (let [resp (request {:request-method :get :uri "/header" + :headers {"x-my-header" "header-val"}})] + (is (= "header-val" (slurp-body resp))))) + +(deftest ^:integration sends-and-returns-byte-array-body + (run-server) + (let [resp (request {:request-method :post :uri "/post" + :body (util/utf8-bytes "contents")})] + (is (= 200 (:status resp))) + (is (= "contents" (slurp-body resp))))) + +(deftest ^:integration returns-arbitrary-headers + (run-server) + (let [resp (request {:request-method :get :uri "/get"})] + (is (string? (get-in resp [:headers "date"]))) + (is (nil? (get-in resp [:headers "Date"]))))) + +(deftest ^:integration returns-status-on-exceptional-responses + (run-server) + (let [resp (request {:request-method :get :uri "/error"})] + (is (= 500 (:status resp))))) + +(deftest ^:integration sets-socket-timeout + (run-server) + (try + (is (thrown? SocketTimeoutException + (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get :uri "/timeout" + :socket-timeout 1}))))) + +(deftest ^:integration delete-with-body + (run-server) + (let [resp (request {:request-method :delete :uri "/delete-with-body" + :body (.getBytes "foo bar")})] + (is (= 200 (:status resp))))) + +;; Module issue exporting SunCertPathBuilderException +#_ +(deftest ^:integration self-signed-ssl-get + (let [server (ring/run-jetty handler + {:port 8081 :ssl-port 18082 + :ssl? true + :join? false + :keystore "test-resources/keystore" + :key-password "keykey"})] + (try + (is (thrown? SunCertPathBuilderException + (client/request {:scheme :https + :server-name "localhost" + :server-port 18082 + :request-method :get :uri "/get"}))) + (let [resp (request {:request-method :get :uri "/get" :server-port 18082 + :scheme :https :insecure? true})] + (is (= 200 (:status resp))) + (is (= "get" (String. (util/force-byte-array (:body resp)))))) + (finally + (.stop server))))) + +(deftest ^:integration multipart-form-uploads + (run-server) + (let [bytes (util/utf8-bytes "byte-test") + stream (ByteArrayInputStream. bytes) + resp (request {:request-method :post :uri "/multipart" + :multipart [{:name "a" :content "testFINDMEtest" + :encoding "UTF-8" + :mime-type "application/text"} + {:name "b" :content bytes + :mime-type "application/json"} + {:name "d" + :content (file "test-resources/keystore") + :encoding "UTF-8" + :mime-type "application/binary"} + {:name "c" :content stream + :mime-type "application/json"} + {:name "e" :part-name "eggplant" + :content "content" + :mime-type "application/text"}]}) + resp-body (apply str (map #(try (char %) (catch Exception _ "")) + (util/force-byte-array (:body resp))))] + #_(println "clj-http resp-body:\n>>>>>>>>>>>\n" resp-body "\n>>>>>>>>>>\n") + (is (= 200 (:status resp))) + (is (re-find #"testFINDMEtest" resp-body)) + (is (re-find #"application/json" resp-body)) + (is (re-find #"application/text" resp-body)) + (is (re-find #"UTF-8" resp-body)) + (is (re-find #"byte-test" resp-body)) + (is (re-find #"name=\"c\"" resp-body)) + (is (re-find #"name=\"d\"" resp-body)) + (is (re-find #"name=\"eggplant\"" resp-body)) + (is (re-find #"content" resp-body)))) + +(deftest ^:integration multipart-inputstream-length + (run-server) + (let [bytes (util/utf8-bytes "byte-test") + stream (ByteArrayInputStream. bytes) + resp (request {:request-method :post :uri "/multipart" + :multipart [{:name "c" :content stream :length 9 + :mime-type "application/json"}]}) + resp-body (apply str (map #(try (char %) (catch Exception _ "")) + (util/force-byte-array (:body resp))))] + (is (= 200 (:status resp))) + (is (re-find #"byte-test" resp-body)))) + +(deftest parse-headers + (are [headers expected] + (let [iterator (BasicHeaderIterator. + (into-array BasicHeader + (map (fn [[name value]] + (BasicHeader. name value)) + headers)) nil)] + (is (= (core/parse-headers iterator) expected))) + + [] {} + + [["Set-Cookie" "one"]] {"set-cookie" "one"} + + [["Set-Cookie" "one"] ["set-COOKIE" "two"]] + {"set-cookie" ["one" "two"]} + + [["Set-Cookie" "one"] ["serVer" "some-server"] ["set-cookie" "two"]] + {"set-cookie" ["one" "two"] "server" "some-server"})) + +(deftest ^:integration t-streaming-response + (run-server) + (let [stream (:body (request {:request-method :get :uri "/get" :as :stream})) + body (slurp stream)] + (is (= "get" body)))) + + +(deftest ^:integration throw-on-too-many-redirects + (run-server) + (let [resp (client/get (localhost "/redirect") + {:max-redirects 2 :throw-exceptions false + :redirect-strategy :none + :allow-circular-redirects true})] + (is (= 302 (:status resp)))) + + (let [resp (client/get (localhost "/redirect") + {:max-redirects 3 + :redirect-strategy :graceful + :allow-circular-redirects true})] + (is (= 302 (:status resp))) + (is (= 3 (count (:trace-redirects resp)))) + (is (= ["http://localhost:18080/redirect" + "http://localhost:18080/redirect" + "http://localhost:18080/redirect"] + (:trace-redirects resp)))) + + (is (thrown-with-msg? Exception #"Maximum redirects \(2\) exceeded" + (client/get (localhost "/redirect") + {:max-redirects 2 + :throw-exceptions true + :allow-circular-redirects true}))) + (is (thrown-with-msg? Exception #"Maximum redirects \(50\) exceeded" + (client/get (localhost "/redirect") + {:throw-exceptions true + :allow-circular-redirects true})))) + +(deftest ^:integration get-with-body + (run-server) + (let [resp (request {:request-method :get :uri "/get-with-body" + :body (.getBytes "foo bar")})] + (is (= 200 (:status resp))) + (is (= "foo bar" (String. (util/force-byte-array (:body resp))))))) + +(deftest ^:integration head-with-body + (run-server) + (let [resp (request {:request-method :head :uri "/head" :body "foo"})] + (is (= 200 (:status resp))))) + +(deftest ^:integration t-clojure-output-coercion + (run-server) + (let [resp (client/get (localhost "/clojure") {:as :clojure})] + (is (= 200 (:status resp))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} (:body resp)))) + (let [clj-resp (client/get (localhost "/clojure") {:as :auto}) + edn-resp (client/get (localhost "/edn") {:as :auto})] + (is (= 200 (:status clj-resp) (:status edn-resp))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} + (:body clj-resp) + (:body edn-resp))))) + +(deftest ^:integration t-transit-output-coercion + (run-server) + (let [transit-json-resp (client/get (localhost "/transit-json") {:as :auto}) + transit-msgpack-resp (client/get (localhost "/transit-msgpack") + {:as :auto}) + bad-status-resp-default + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json}) + bad-status-resp-always + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json + :coerce :always}) + bad-status-resp-exceptional + (client/get (localhost "/transit-json-bad") + {:throw-exceptions false :as :transit+json + :coerce :exceptional}) + empty-resp (client/get (localhost "/transit-json-empty") + {:throw-exceptions false :as :transit+json})] + (is (= 200 + (:status transit-json-resp) + (:status transit-msgpack-resp) + (:status empty-resp))) + (is (= 400 + (:status bad-status-resp-default) + (:status bad-status-resp-always) + (:status bad-status-resp-exceptional))) + (is (= {:foo "bar" :baz 7M :eggplant {:quux #{1 2 3}}} + (:body transit-json-resp) + (:body transit-msgpack-resp))) + + (is (nil? (:body empty-resp))) + + (is (= "[\"^ \", \"~:foo\",\"bar\"]" + (:body bad-status-resp-default))) + (is (= {:foo "bar"} + (:body bad-status-resp-always))) + (is (= {:foo "bar"} + (:body bad-status-resp-exceptional))))) + +(deftest ^:integration t-json-output-coercion + (run-server) + (let [resp (client/get (localhost "/json") {:as :json}) + resp-array (client/get (localhost "/json-array") {:as :json}) + resp-array-strict (client/get (localhost "/json-array") {:as :json-strict}) + resp-large-array (client/get (localhost "/json-large-array") {:as :json}) + resp-large-array-strict (client/get (localhost "/json-large-array") {:as :json-strict}) + resp-str (client/get (localhost "/json") + {:as :json :coerce :exceptional}) + resp-str-keys (client/get (localhost "/json") {:as :json-string-keys}) + resp-strict-str-keys (client/get (localhost "/json") + {:as :json-strict-string-keys}) + resp-auto (client/get (localhost "/json") {:as :auto}) + bad-resp (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json}) + bad-resp-json (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json + :coerce :always}) + bad-resp-json2 (client/get (localhost "/json-bad") + {:throw-exceptions false :as :json + :coerce :unexceptional})] + (is (= 200 + (:status resp) + (:status resp-array) + (:status resp-array-strict) + (:status resp-large-array) + (:status resp-large-array-strict) + (:status resp-str) + (:status resp-str-keys) + (:status resp-strict-str-keys) + (:status resp-auto))) + (is (= {:foo "bar"} + (:body resp) + (:body resp-auto))) + (is (= ["foo", "bar"] + (:body resp-array))) + (is (= {"foo" "bar"} + (:body resp-strict-str-keys) + (:body resp-str-keys))) + ;; '("foo" "bar") and ["foo" "bar"] compare as equal with =. + (is (vector? (:body resp-array))) + (is (vector? (:body resp-array-strict))) + (is (= "{\"foo\":\"bar\"}" (:body resp-str))) + (is (= 400 + (:status bad-resp) + (:status bad-resp-json) + (:status bad-resp-json2))) + (is (= "{\"foo\":\"bar\"}" (:body bad-resp)) + "don't coerce on bad response status by default") + (is (= {:foo "bar"} (:body bad-resp-json))) + (is (= "{\"foo\":\"bar\"}" (:body bad-resp-json2))) + + (testing "lazily parsed stream completes parsing." + (is (= 100 (count (:body resp-large-array))))) + (is (= 100 (count (:body resp-large-array-strict)))))) + +(deftest ^:integration t-ipv6 + (run-server) + (let [resp (client/get "http://[::1]:18080/get")] + (is (= 200 (:status resp))) + (is (= "get" (:body resp))))) + +(deftest t-custom-retry-handler + (let [called? (atom false)] + (is (thrown? Exception + (client/post "http://localhost" + {:multipart [{:name "title" :content "Foo"} + {:name "Content/type" + :content "text/plain"} + {:name "file" + :content (file "/tmp/missingfile")}] + :retry-handler (fn [ex try-count http-context] + (reset! called? true) + false)}))) + (is @called?))) + +;; super-basic test for methods that aren't used that often +(deftest ^:integration t-copy-options-move + (run-server) + (let [resp1 (client/options (localhost "/options")) + resp2 (client/move (localhost "/move")) + resp3 (client/copy (localhost "/copy")) + resp4 (client/patch (localhost "/patch"))] + (is (= #{200} (set (map :status [resp1 resp2 resp3 resp4])))) + (is (= "options" (:body resp1))) + (is (= "move" (:body resp2))) + (is (= "copy" (:body resp3))) + (is (= "patch" (:body resp4))))) + +(deftest ^:integration t-json-encoded-form-params + (run-server) + (let [params {:param1 "value1" :param2 {:foo "bar"}} + resp (client/post (localhost "/post") {:content-type :json + :form-params params})] + (is (= 200 (:status resp))) + (is (= (json/encode params) (:body resp))))) + +(deftest ^:integration t-request-interceptor + (run-server) + (let [req-ctx (atom []) + {:keys [status trace-redirects] :as resp} + (client/get + (localhost "/get") + {:request-interceptor + (fn [^HttpRequest req ^HttpContext ctx] + (reset! req-ctx {:method (.getMethod req) :uri (.getURI req)}))})] + (is (= 200 status)) + (is (= "GET" (:method @req-ctx))) + (is (= "/get" (.getPath (:uri @req-ctx)))))) + +(deftest ^:integration t-response-interceptor + (run-server) + (let [saved-ctx (atom []) + {:keys [status trace-redirects] :as resp} + (client/get + (localhost "/redirect-to-get") + {:response-interceptor + (fn [^HttpResponse resp ^HttpContext ctx] + (let [^HttpInetConnection conn + (.getAttribute ctx ExecutionContext/HTTP_CONNECTION)] + (swap! saved-ctx conj {:remote-port (.getRemotePort conn) + :http-conn conn})))})] + (is (= 200 status)) + (is (= 2 (count @saved-ctx))) + #_(is (= (count trace-redirects) (count @saved-ctx))) + (is (every? #(= 18080 (:remote-port %)) @saved-ctx)) + (is (every? #(instance? HttpConnection (:http-conn %)) @saved-ctx)))) + +(deftest ^:integration t-send-input-stream-body + (run-server) + (let [b1 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. (.getBytes "foo")) + :length 3})) + b2 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. + (.getBytes "foo"))})) + b3 (:body (client/post "http://localhost:18080/post" + {:body (ByteArrayInputStream. + (.getBytes "apple")) + :length 2}))] + (is (= b1 "foo")) + (is (= b2 "foo")) + (is (= b3 "ap")))) + +;; (deftest t-add-client-params +;; (testing "Using add-client-params!" +;; (let [ps {"http.conn-manager.timeout" 100 +;; "http.socket.timeout" 250 +;; "http.protocol.allow-circular-redirects" false +;; "http.protocol.version" HttpVersion/HTTP_1_0 +;; "http.useragent" "clj-http"} +;; setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! ps)))] +;; (doseq [[k v] ps] +;; (is (= v (.getParameter setps k))))))) + +;; Regression, get notified if something changes +(deftest ^:integration t-known-client-params-are-unchanged + (let [params ["http.socket.timeout" CoreConnectionPNames/SO_TIMEOUT + "http.connection.timeout" + CoreConnectionPNames/CONNECTION_TIMEOUT + "http.protocol.version" CoreProtocolPNames/PROTOCOL_VERSION + "http.useragent" CoreProtocolPNames/USER_AGENT + "http.conn-manager.timeout" ClientPNames/CONN_MANAGER_TIMEOUT + "http.protocol.allow-circular-redirects" + ClientPNames/ALLOW_CIRCULAR_REDIRECTS]] + (doseq [[plaintext constant] (partition 2 params)] + (is (= plaintext constant))))) + +;; If you don't explicitly set a :cookie-policy, use +;; CookiePolicy/BROWSER_COMPATIBILITY +;; (deftest t-add-client-params-default-cookie-policy +;; (testing "Using add-client-params! to get a default cookie policy" +;; (let [setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! {})))] +;; (is (= CookiePolicy/BROWSER_COMPATIBILITY +;; (.getParameter setps ClientPNames/COOKIE_POLICY)))))) + +;; If you set a :cookie-policy, the name of the policy is registered +;; as (str (type cookie-policy)) +;; (deftest t-add-client-params-cookie-policy +;; (testing "Using add-client-params! to get an explicitly set :cookie-policy" +;; (let [setps (.getParams (doto (DefaultHttpClient.) +;; (core/add-client-params! +;; {:cookie-policy (constantly nil)})))] +;; (is (.startsWith ^String +;; (.getParameter setps ClientPNames/COOKIE_POLICY) +;; "class "))))) + + +;; This relies on connections to writequit.org being slower than 10ms, if this +;; fails, you must have very nice internet. +(deftest ^:integration sets-connection-timeout + (run-server) + (try + (is (thrown? SocketTimeoutException + (client/request {:scheme :http + :server-name "writequit.org" + :server-port 80 + :request-method :get :uri "/" + :connection-timeout 10}))))) + +(deftest ^:integration connection-pool-timeout + (run-server) + (client/with-connection-pool {:threads 1 :default-per-route 1} + (let [async-request #(future (client/request {:scheme :http + :server-name "localhost" + :server-port 18080 + :request-method :get + :connection-timeout 1 + :connection-request-timeout 1 + :uri "/timeout"})) + is-pool-timeout-error? + (fn [req-fut] + (instance? org.apache.http.conn.ConnectionPoolTimeoutException + (try @req-fut (catch Exception e (.getCause e))))) + req1 (async-request) + req2 (async-request) + timeout-error1 (is-pool-timeout-error? req1) + timeout-error2 (is-pool-timeout-error? req2)] + (is (or timeout-error1 timeout-error2))))) + +(deftest ^:integration t-header-collections + (run-server) + (let [headers (-> (client/get "http://localhost:18080/headers" + {:headers {"foo" ["bar" "baz"] + "eggplant" "quux"}}) + :body + json/decode)] + (is (= {"eggplant" "quux" "foo" "bar,baz"} + (select-keys headers ["foo" "eggplant"]))))) + +(deftest ^:integration t-clojure-no-read-eval + (run-server) + (is (thrown? Exception (client/get (localhost "/clojure-bad") {:as :clojure})) + "Should throw an exception when reading clojure eval components")) + +(deftest ^:integration t-numeric-headers + (run-server) + (client/request {:method :get :url (localhost "/get") :headers {"foo" 2}})) + +(deftest ^:integration t-empty-response-coercion + (run-server) + (let [resp (client/get (localhost "/empty") {:as :clojure})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty") {:as :json})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty-gzip") + {:as :clojure})] + (is (= (:body resp) nil))) + (let [resp (client/get (localhost "/empty-gzip") + {:as :json})] + (is (= (:body resp) nil)))) + +(deftest ^:integration t-trace-redirects + (run-server) + (let [resp-with-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get")}) + + resp-with-graceful-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get") + :redirect-strategy :graceful}) + + resp-without-redirects + (client/request {:method :get + :url (localhost "/redirect-to-get") + :redirect-strategy :none})] + + (is (= (:trace-redirects resp-with-redirects) + ["http://localhost:18080/get"])) + + (is (= (:trace-redirects resp-with-graceful-redirects) + ["http://localhost:18080/get"])) + + (is (= (:trace-redirects resp-without-redirects) [])))) + +(deftest t-request-config + (let [params {:conn-timeout 100 ;; deprecated + :connection-timeout 200 ;; takes precedence over `:conn-timeout` + :conn-request-timeout 300 ;; deprecated + :connection-request-timeout 400 ;; takes precedence over `:conn-request-timeout` + :socket-timeout 500 + :max-redirects 600 + :cookie-spec "foo" + :normalize-uri false} + request-config (core/request-config params)] + (is (= 200 (.getConnectTimeout request-config))) + (is (= 400 (.getConnectionRequestTimeout request-config))) + (is (= 500 (.getSocketTimeout request-config))) + (is (= 600 (.getMaxRedirects request-config))) + (is (= core/CUSTOM_COOKIE_POLICY (.getCookieSpec request-config))) + (is (false? (.isNormalizeUri request-config))))) + +(deftest ^:integration t-override-request-config + (run-server) + (let [called-args (atom []) + real-http-client core/build-http-client + http-context (HttpClientContext/create) + request-config (.build (RequestConfig/custom))] + (with-redefs + [core/build-http-client + (fn [& args] + (proxy [org.apache.http.impl.client.CloseableHttpClient] [] + (execute [http-req context] + (swap! called-args conj [http-req context]) + (.execute (apply real-http-client args) http-req context))))] + (client/request {:method :get + :url "http://localhost:18080/get" + :http-client-context http-context + :http-request-config request-config}) + + (let [context-for-request (last (last @called-args))] + (is (= http-context context-for-request)) + (is (= request-config (.getRequestConfig context-for-request))))))) + +(deftest ^:integration ^:ignore test-custom-http-builder-fns + (run-server) + (let [resp (client/get (localhost "/get") + {:headers {"add-headers" "true"} + :http-builder-fns + [(fn [builder req] + (.setDefaultHeaders builder (:hdrs req)))] + :hdrs [(BasicHeader. "foo" "bar")]})] + (is (= 200 (:status resp))) + (is (.contains (get-in resp [:headers "got"]) "\"foo\" \"bar\"") + "Headers should have included the new default headers")) + (let [resp (promise) + error (promise) + f (client/get (localhost "/get") + {:async true + :headers {"add-headers" "true"} + :async-http-builder-fns + [(fn [builder req] + (.setDefaultHeaders builder (:hdrs req)))] + :hdrs [(BasicHeader. "foo" "bar")]} + resp error)] + (.get f) + (is (= 200 (:status @resp))) + (is (.contains (get-in @resp [:headers "got"]) "\"foo\" \"bar\"") + "Headers should have included the new default headers") + (is (not (realized? error))))) + +(deftest ^:integration test-custom-http-client-builder + (run-server) + (let [methods (atom nil) + resp (client/get + (localhost "/get") + {:http-client-builder + (-> (org.apache.http.impl.client.HttpClientBuilder/create) + (.setRequestExecutor + (proxy [org.apache.http.protocol.HttpRequestExecutor] [] + (execute [request connection context] + (->> request + .getRequestLine + .getMethod + (swap! methods conj)) + (proxy-super execute request connection context)))))})] + (is (= ["GET"] @methods)))) + +(deftest ^:integration test-bad-redirects + (run-server) + (try + (client/get (localhost "/bad-redirect")) + (is false "should have thrown an exception") + (catch ProtocolException e + (is (.contains + (.getMessage e) + "Redirect URI does not specify a valid host name: https:///")))) + ;; async version + (let [e (atom nil) + latch (promise)] + (try + (.get + (client/get (localhost "/bad-redirect") {:async true} + (fn [resp] + (is false + (str "should not have been called but got" resp))) + (fn [err] + (reset! e err) + (deliver latch true) + nil))) + (catch Exception error + (is (.contains + (.getMessage error) + "Redirect URI does not specify a valid host name: https:///")))) + @latch + (is (.contains + (.getMessage @e) + "Redirect URI does not specify a valid host name: https:///"))) + (try + (.get (client/get + (localhost "/bad-redirect") + {:async true + :validate-redirects false} + (fn [resp] + (is false + (str "should not have been called but got" resp))) + (fn [err] + (is false + (str "should not have been called but got" err)))) + 1 TimeUnit/SECONDS) + (is false "should have thrown a timeout exception") + (catch TimeoutException te))) + +(deftest ^:integration ^:ignore test-reusable-http-client + (run-server) + (let [cm (conn/make-reuseable-async-conn-manager {}) + hc (core/build-async-http-client {} cm)] + (client/get (localhost "/json") + {:connection-manager cm + :http-client hc + :as :json + :async true} + (fn [resp] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused")) + (fn [e] (is false (str "failed with " e))))) + (let [cm (conn/make-reusable-conn-manager {}) + hc (:http-client (client/get (localhost "/get") + {:connection-manager cm})) + resp (client/get (localhost "/json") + {:connection-manager cm + :http-client hc + :as :json})] + (is (= 200 (:status resp))) + (is (= {:foo "bar"} (:body resp))) + (is (= hc (:http-client resp)) + "http-client is correctly reused"))) + +(deftest ^:integration t-cookies-spec + (run-server) + (try + (client/get (localhost "/bad-cookie")) + (is false "should have failed") + (catch org.apache.http.cookie.MalformedCookieException e)) + (client/get (localhost "/bad-cookie") {:decode-cookies false}) + (let [validated (atom false) + spec-provider (RFC6265CookieSpecProvider.) + resp (client/get (localhost "/cookie") + {:cookie-spec + (fn [http-context] + (proxy [org.apache.http.impl.cookie.CookieSpecBase] [] + ;; Version and version header + (getVersion [] 0) + (getVersionHeader [] nil) + ;; parse headers into cookie objects + (parse [header cookie-origin] + (.parse (.create spec-provider http-context) + header cookie-origin)) + ;; Validate a cookie, throwing MalformedCookieException if the + ;; cookies isn't valid + (validate [cookie cookie-origin] + (reset! validated true)) + ;; Determine if a cookie matches the target location + (match [cookie cookie-origin] true) + ;; Format a list of cookies into a list of headers + (formatCookies [cookies] (java.util.ArrayList.))))})] + (is (= @validated true)))) + + +(deftest ^:ignore t-cache-config + (let [cc (core/build-cache-config + {:cache-config {:allow-303-caching true + :asynchronous-worker-idle-lifetime-secs 10 + :asynchronous-workers-core 2 + :asynchronous-workers-max 3 + :heuristic-caching-enabled true + :heuristic-coefficient 1.5 + :heuristic-default-lifetime 12 + :max-cache-entries 100 + :max-object-size 123 + :max-update-retries 3 + :revalidation-queue-size 2 + :shared-cache false + :weak-etag-on-put-delete-allowed true}})] + (is (= true (.is303CachingEnabled cc))) + (is (= 10 (.getAsynchronousWorkerIdleLifetimeSecs cc))) + (is (= 2 (.getAsynchronousWorkersCore cc))) + (is (= 3 (.getAsynchronousWorkersMax cc))) + (is (= true (.isHeuristicCachingEnabled cc))) + (is (= 1.5 (.getHeuristicCoefficient cc))) + (is (= 12 (.getHeuristicDefaultLifetime cc))) + (is (= 100 (.getMaxCacheEntries cc))) + (is (= 123 (.getMaxObjectSize cc))) + (is (= 3 (.getMaxUpdateRetries cc))) + (is (= 2 (.getRevalidationQueueSize cc))) + (is (= false (.isSharedCache cc))) + (is (= true (.isWeakETagOnPutDeleteAllowed cc))))) + +(deftest ^:integration ^:ignore t-client-caching + (run-server) + (let [cm (conn/make-reusable-conn-manager {}) + r1 (client/get (localhost "/get") + {:connection-manager cm :cache true}) + client (:http-client r1) + r2 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true}) + r3 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true}) + r4 (client/get (localhost "/get") + {:connection-manager cm :http-client client :cache true})] + (is (= :CACHE_MISS (:cached r1))) + (is (= :VALIDATED (:cached r2))) + (is (= :VALIDATED (:cached r3))) + (is (= :VALIDATED (:cached r4)))) + (let [cm (conn/make-reusable-conn-manager {}) + r1 (client/get (localhost "/dont-cache") + {:connection-manager cm :cache true}) + client (:http-client r1) + r2 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true}) + r3 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true}) + r4 (client/get (localhost "/dont-cache") + {:connection-manager cm :http-client client :cache true})] + (is (= :CACHE_MISS (:cached r1))) + (is (= :CACHE_MISS (:cached r2))) + (is (= :CACHE_MISS (:cached r3))) + (is (= :CACHE_MISS (:cached r4))))) diff --git a/test/aleph/http/clj_http/util.clj b/test/aleph/http/clj_http/util.clj new file mode 100644 index 00000000..575f87d4 --- /dev/null +++ b/test/aleph/http/clj_http/util.clj @@ -0,0 +1,338 @@ +(ns aleph.http.clj-http.util + (:require + [aleph.http :as http] + [aleph.http.core :as http.core] + [aleph.http.client-middleware :as aleph.mid] + [clj-commons.byte-streams :as bs] + [clj-http.core :as clj-http] + [clj-http.client] + [clojure.set :as set] + [clojure.string :as str] + [clojure.test :refer :all] + [clojure.tools.logging :as log]) + (:import + (java.io ByteArrayInputStream + ByteArrayOutputStream + FilterInputStream + InputStream) + (java.util.regex Pattern))) + +;; turn off default middleware for the core tests +(def no-middleware-pool (http/connection-pool {:middleware identity})) + +(def base-req + {:scheme :http + :server-name "localhost" + :server-port 18080}) + +(def ignored-headers ["date" "connection" "server"]) +(def multipart-related-headers ["content-length" "x-original-content-type"]) + +(defn header-keys + "Returns a set of headers of interest" + [m] + (->> (apply dissoc m ignored-headers) + (keys) + (map str/lower-case) + (set))) + +(defn is-headers= + "Are the two header maps equal? + + Additional Aleph headers are ignored" + [clj-http-headers aleph-headers] + (let [clj-http-ks (header-keys clj-http-headers) + aleph-ks (header-keys aleph-headers)] + (is (set/superset? aleph-ks clj-http-ks)) + (let [ks-intersection (set/intersection aleph-ks clj-http-ks) + clj-http-common-headers (select-keys clj-http-headers ks-intersection) + aleph-common-headers (select-keys aleph-headers ks-intersection)] + (is (= clj-http-common-headers aleph-common-headers))))) + +(defn- tee-output-stream + "Return the byte array contents of a stream, and a new, unconsumed stream" + [^InputStream in] + (let [baos (ByteArrayOutputStream.)] + (.transferTo in baos) ; not avail until JDK 9 + + (let [in-bytes (.toByteArray baos)] + {:bytes in-bytes + :stream (proxy [FilterInputStream] + [^InputStream (ByteArrayInputStream. in-bytes)] + (close [] + (.close in) + (proxy-super close)))}))) + +(defn bodies= + "Are the two bodies equal? clj-http's client/request fn coerces to strings by default, + while the core/request leaves the body an InputStream. + Aleph, in keeping with its stream-based nature, leaves it as an InputStream by default. + + If an InputStream, returns a new ByteArrayInputStream based on the consumed original" + [clj-http-body ^InputStream aleph-body] + (if clj-http-body + (condp instance? clj-http-body + InputStream + (do + (is (some? aleph-body) "Why is aleph body nil? It should be an empty InputStream for now...") + (let [baos (ByteArrayOutputStream.)] + (.transferTo clj-http-body baos) ; not avail until JDK 9 + + (let [clj-http-body-bytes (.toByteArray baos)] + (is (= (count clj-http-body-bytes) + (.available aleph-body))) + (is (bs/bytes= clj-http-body-bytes aleph-body)) + + (proxy [FilterInputStream] + [^InputStream (ByteArrayInputStream. clj-http-body-bytes)] + (close [] + (.close clj-http-body) + (proxy-super close)))))) + + (try + (do + (is (bs/bytes= clj-http-body aleph-body)) + clj-http-body) + (catch Exception e + (println "clj-http body class: " (class clj-http-body)) + (prn clj-http-body) + (flush) + (throw e)))) + (do + (is (= clj-http-body aleph-body)) + clj-http-body))) + +(defn- parse-multipart-boundary + [s] + (->> s + (re-find #"boundary=([^ ;]*)") + (second))) + +;;(defn- decode-multipart-body +;; [req body] +;; (let [req' (http.core/ring-request->netty-request req) +;; factory (DefaultHttpDataFactory. (long 1e6)) +;; decoder (HttpPostRequestDecoder. factory req') +;; baos (ByteArrayOutputStream.)])) + +(defn multipart-resp= + "Compares multipart responses from /multipart, which echoes the orig multipart bodies. + + Splits based on boundaries, and compares the parts. Whole-byte comparison is impossible + since the boundary strings are chosen randomly. + + Does not compare part headers for now, since they differ in case and order, and clj-http + adds Content-Length headers, which are uncommon, can cause problems, and may be completely + unknown for streaming requests." + [clj-http-resp aleph-resp] + (let [clj-http-headers (:headers clj-http-resp) + aleph-headers (:headers aleph-resp) + clj-http-boundary (parse-multipart-boundary (get clj-http-headers "x-original-content-type")) + aleph-boundary (parse-multipart-boundary (get aleph-headers "x-original-content-type")) + {clj-http-bytes :bytes clj-http-stream :stream} (tee-output-stream (:body clj-http-resp)) + aleph-bytes (-> aleph-resp :body tee-output-stream :bytes) + + ;; unlikely to be a problem, but let's make the regex literal, just to be safe + clj-http-boundary-regex (Pattern/compile clj-http-boundary (bit-or Pattern/LITERAL Pattern/MULTILINE)) + aleph-boundary-regex (Pattern/compile aleph-boundary (bit-or Pattern/LITERAL Pattern/MULTILINE)) + + clj-http-contents (-> ^bytes clj-http-bytes + (String.) + (str/split clj-http-boundary-regex)) + aleph-contents (-> ^bytes aleph-bytes + (String.) + (str/split aleph-boundary-regex))] + #_(do + (println "aleph bytes") + (bs/print-bytes aleph-bytes) + + (println "clj-http bytes") + (bs/print-bytes clj-http-bytes)) + + (is (= (count clj-http-contents) (count aleph-contents)) + "Unequal number of parts found!") + (doseq [[^String clj-http-part ^String aleph-part] (partition 2 (interleave clj-http-contents aleph-contents))] + (let [[clj-http-part-headers clj-http-part-body] (str/split clj-http-part #"\r\n\r\n") + [aleph-part-headers aleph-part-body] (str/split aleph-part #"\r\n\r\n")] + #_ (println "headers:>>>>>>>>>>>>\n" clj-http-part-headers "\n>>>>>>>>>>>>>>>>\n" aleph-part-headers) + #_ (println ">>>>>>>>>>\nbodies:\n" clj-http-part-body "\n>>>>>>>>>>>>>>>>\n" aleph-part-body) + (is (or (and (nil? clj-http-part-body) (nil? aleph-part-body)) + (.equalsIgnoreCase clj-http-part-body aleph-part-body)) + (str "clj-part:\n>>>>>>>>>>\n" clj-http-part-body "\n>>>>>>>>>>\naleph-part:\n>>>>>>>>>>\n" aleph-part-body "\n>>>>>>>>>>\n")))) + + clj-http-stream)) + + +(defn- defined-middleware + "Returns a set of symbols beginning with `wrap-` in the ns" + [ns] + (->> (ns-publics ns) + keys + (map str) + (filter #(str/starts-with? % "wrap-")) + (map symbol) + set)) + +(defn- aleph-test-conn-pool + "clj-http middleware is traditional fn-based middleware, using a 3-arity version to handle async. + + Aleph usually uses a more async-friendly interceptor-style, where the middleware transforms the request maps, + but does nothing about calling the next fn in the chain. + + Unfortunately, a couple middleware cannot be converted to interceptor-style, complicating things." + [middleware-list] + (let [missing-midw (set/difference + (defined-middleware 'clj-http.client) + (defined-middleware 'aleph.http.client-middleware))] + (when-not (seq missing-midw) + (println "clj-http is using middleware that aleph lacks:") + (prn missing-midw) + (log/warn "clj-http is using middleware that aleph lacks" + :missing-middleware missing-midw))) + (let [non-interceptor-middleware (set aleph.mid/default-client-middleware) + client-middleware (cond-> [] + (some #{clj-http.client/wrap-exceptions} middleware-list) + (conj aleph.mid/wrap-exceptions) + + (some #{clj-http.client/wrap-request-timing} middleware-list) + (conj aleph.mid/wrap-request-timing)) + middleware-list' (->> middleware-list + (map (fn [midw] + (-> midw + class + str + (str/split #"\$") + peek + (str/replace "_" "-") + (->> (symbol "aleph.http.client-middleware")) + requiring-resolve))) + (filter some?) + (map var-get) + (remove non-interceptor-middleware) + vec)] + ;;(println "Client-based middleware:") + ;;(prn client-middleware) + ;;(println "Regular middleware:") + ;;(prn middleware-list') + (http/connection-pool {:middleware #(aleph.mid/wrap-request % client-middleware middleware-list')}))) + + +(defn- print-middleware-list + [middleware-list] + (prn (mapv (fn [midw] + (-> midw + class + str + (str/split #"\$") + peek + (str/replace "_" "-") + symbol)) + middleware-list))) + +(defn- bais-clone + "Clones a ByteArrayInputStream and resets the original's pos, so it can be read again" + [^ByteArrayInputStream bais] + (.mark bais 0) + (let [new-bais (ByteArrayInputStream. (.readAllBytes bais))] + (.reset bais) + new-bais)) + +(defn build-aleph-ring-map + "Constructs an aleph ring map, based on the clj-http ring map. + + Adds corresponding middleware, and copies request ByteArrayInputStreams, + since they can't be read more than once by default." + [clj-http-ring-map clj-http-middleware] + (let [clone-bais-val (fn [m k] + (if (= ByteArrayInputStream (-> m k class)) + (assoc m k (bais-clone (k m))) + m)) + middleware-ring-map (merge clj-http-ring-map {:pool (aleph-test-conn-pool clj-http-middleware)})] + (cond-> middleware-ring-map + + (contains? clj-http-ring-map :body) + (clone-bais-val :body) + + (contains? clj-http-ring-map :multipart) + (update-in [:multipart] + (fn [parts] + (into [] + (map #(clone-bais-val % :content) + #_(fn [part] + (if (= ByteArrayInputStream (-> part :content class)) + (assoc part :content (bais-clone (:content part))) + part))) + parts)))))) + +(defn make-request + "Need to switch between clj-http's core/request and client/request. + + Modified version of original request fns, that also sends requests + via Aleph, and tests the responses for equality." + [clj-http-request {:keys [using-middleware?]}] + (fn compare-request + ([req] + (compare-request req nil nil)) + ([req respond raise] + (if (or respond raise) + ;; do not attempt to compare when using async clj-http...for now + (let [ring-map (merge base-req req)] + (clj-http-request ring-map respond raise)) + + (let [clj-http-ring-map (merge base-req req) + ;;_ (prn clj-http-ring-map) + clj-http-middleware (if using-middleware? clj-http.client/*current-middleware* []) + ;;_ (print-middleware-list clj-http.client/*current-middleware*) + aleph-ring-map (build-aleph-ring-map clj-http-ring-map clj-http-middleware) + ;;_ (prn aleph-ring-map) + is-multipart (contains? clj-http-ring-map :multipart) + clj-http-resp (clj-http-request clj-http-ring-map) + aleph-resp @(http/request aleph-ring-map)] + (is (= (:status clj-http-resp) (:status aleph-resp))) + + + + #_(when (not= (:status clj-http-resp) (:status aleph-resp)) + (println "clj-http req:") + (prn clj-http-ring-map) + (println) + (println "clj-http resp:") + (prn clj-http-resp) + (println) + (println) + (println "aleph req:") + (prn aleph-ring-map) + (println) + (println "aleph resp:") + (prn aleph-resp)) + + (is (instance? InputStream (:body aleph-resp))) ; non-nil, for now... + + (if is-multipart + (do + ;;(println "multipart resps") + ;;(prn clj-http-resp) + ;;(prn aleph-resp) + ;;(println) + + ;;(do + ;; (println "clj-http req:") + ;; (prn clj-http-ring-map) + ;; (println) + ;; (println "clj-http resp:") + ;; (prn clj-http-resp) + ;; (println) + ;; (println) + ;; (println "aleph req:") + ;; (prn aleph-ring-map) + ;; (println) + ;; (println "aleph resp:") + ;; (prn aleph-resp)) + + (is-headers= (apply dissoc (:headers clj-http-resp) multipart-related-headers) + (apply dissoc (:headers aleph-resp) multipart-related-headers)) + (assoc clj-http-resp :body (multipart-resp= clj-http-resp aleph-resp))) + (do + (is-headers= (:headers clj-http-resp) (:headers aleph-resp)) + (let [new-clj-http-body (bodies= (:body clj-http-resp) (:body aleph-resp))] + (assoc clj-http-resp :body new-clj-http-body)))))))))