Skip to content

Commit 4d516d7

Browse files
KingMobKingMob
authored
Merge pull request #681 from clj-commons/feature/http2
Client HTTP/2 support
2 parents 0c94547 + da14def commit 4d516d7

32 files changed

+4221
-2256
lines changed

‎.clj-kondo/metosin/malli/config.edn

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{:lint-as {malli.experimental/defn schema.core/defn}
2+
:linters {:unresolved-symbol {:exclude [(malli.core/=>)]}}}

‎.clj-kondo/org.clj-commons/byte-streams/clj_commons/byte_streams.clj_kondo

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
(ns clj-commons.byte-streams)
2+
3+
;; TODO: propagate type info from src/dst
4+
(defmacro def-conversion
5+
"Kondo hook"
6+
[[src dst :as conversion] params & body]
7+
`(fn [~(first params)
8+
~(if-let [options (second params)]
9+
options
10+
`_#)]
11+
~@body))

‎.clj-kondo/org.clj-commons/byte-streams/config.edn

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{:lint-as {byte-streams.utils/defprotocol+ clojure.core/defprotocol
2+
byte-streams.utils/deftype+ clojure.core/deftype
3+
byte-streams.utils/defrecord+ clojure.core/defrecord
4+
byte-streams.utils/definterface+ clojure.core/definterface
5+
clj-commons.byte-streams.utils/defprotocol+ clojure.core/defprotocol
6+
clj-commons.byte-streams.utils/deftype+ clojure.core/deftype
7+
clj-commons.byte-streams.utils/defrecord+ clojure.core/defrecord
8+
clj-commons.byte-streams.utils/definterface+ clojure.core/definterface}
9+
:hooks {:macroexpand {clj-commons.byte-streams/def-conversion clj-commons.byte-streams/def-conversion}}}

‎.clj-kondo/potemkin/potemkin/config.edn

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{:lint-as {potemkin.collections/compile-if clojure.core/if
2+
potemkin.collections/reify-map-type clojure.core/reify
3+
potemkin.collections/def-map-type clj-kondo.lint-as/def-catch-all
4+
potemkin.collections/def-derived-map clj-kondo.lint-as/def-catch-all
5+
6+
potemkin.types/reify+ clojure.core/reify
7+
potemkin.types/defprotocol+ clojure.core/defprotocol
8+
potemkin.types/deftype+ clojure.core/deftype
9+
potemkin.types/defrecord+ clojure.core/defrecord
10+
potemkin.types/definterface+ clojure.core/defprotocol
11+
potemkin.types/extend-protocol+ clojure.core/extend-protocol
12+
potemkin.types/def-abstract-type clj-kondo.lint-as/def-catch-all
13+
14+
potemkin.utils/doit clojure.core/doseq
15+
potemkin.utils/doary clojure.core/doseq
16+
potemkin.utils/condp-case clojure.core/condp
17+
potemkin.utils/fast-bound-fn clojure.core/bound-fn
18+
19+
potemkin.walk/prewalk clojure.walk/prewalk
20+
potemkin.walk/postwalk clojure.walk/postwalk
21+
potemkin.walk/walk clojure.walk/walk
22+
23+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
24+
;;;; top-level from import-vars
25+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
26+
27+
;; Have hooks
28+
;;potemkin/import-fn potemkin.namespaces/import-fn
29+
;;potemkin/import-macro potemkin.namespaces/import-macro
30+
;;potemkin/import-def potemkin.namespaces/import-def
31+
32+
;; Internal, not transitive
33+
;;potemkin/unify-gensyms potemkin.macros/unify-gensyms
34+
;;potemkin/normalize-gensyms potemkin.macros/normalize-gensyms
35+
;;potemkin/equivalent? potemkin.macros/equivalent?
36+
37+
potemkin/condp-case clojure.core/condp
38+
potemkin/doit potemkin.utils/doit
39+
potemkin/doary potemkin.utils/doary
40+
41+
potemkin/def-abstract-type clj-kondo.lint-as/def-catch-all
42+
potemkin/reify+ clojure.core/reify
43+
potemkin/defprotocol+ clojure.core/defprotocol
44+
potemkin/deftype+ clojure.core/deftype
45+
potemkin/defrecord+ clojure.core/defrecord
46+
potemkin/definterface+ clojure.core/defprotocol
47+
potemkin/extend-protocol+ clojure.core/extend-protocol
48+
49+
potemkin/reify-map-type clojure.core/reify
50+
potemkin/def-derived-map clj-kondo.lint-as/def-catch-all
51+
potemkin/def-map-type clj-kondo.lint-as/def-catch-all}
52+
53+
;; leave import-vars alone, kondo special-cases it
54+
:hooks {:macroexpand {#_#_potemkin.namespaces/import-vars potemkin.namespaces/import-vars
55+
potemkin.namespaces/import-fn potemkin.namespaces/import-fn
56+
potemkin.namespaces/import-macro potemkin.namespaces/import-macro
57+
potemkin.namespaces/import-def potemkin.namespaces/import-def
58+
59+
#_#_potemkin/import-vars potemkin.namespaces/import-vars
60+
potemkin/import-fn potemkin.namespaces/import-fn
61+
potemkin/import-macro potemkin.namespaces/import-macro
62+
potemkin/import-def potemkin.namespaces/import-def}}}

‎.clj-kondo/potemkin/potemkin/potemkin/namespaces.clj

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
(ns potemkin.namespaces
2+
(:require [clj-kondo.hooks-api :as api]))
3+
4+
(defn import-macro*
5+
([sym]
6+
`(def ~(-> sym name symbol) ~sym))
7+
([sym name]
8+
`(def ~name ~sym)))
9+
10+
(defmacro import-fn
11+
([sym]
12+
(import-macro* sym))
13+
([sym name]
14+
(import-macro* sym name)))
15+
16+
(defmacro import-macro
17+
([sym]
18+
(import-macro* sym))
19+
([sym name]
20+
(import-macro* sym name)))
21+
22+
(defmacro import-def
23+
([sym]
24+
(import-macro* sym))
25+
([sym name]
26+
(import-macro* sym name)))
27+
28+
#_
29+
(defmacro import-vars
30+
"Imports a list of vars from other namespaces."
31+
[& syms]
32+
(let [unravel (fn unravel [x]
33+
(if (sequential? x)
34+
(->> x
35+
rest
36+
(mapcat unravel)
37+
(map
38+
#(symbol
39+
(str (first x)
40+
(when-let [n (namespace %)]
41+
(str "." n)))
42+
(name %))))
43+
[x]))
44+
syms (mapcat unravel syms)
45+
result `(do
46+
~@(map
47+
(fn [sym]
48+
(let [vr (resolve sym)
49+
m (meta vr)]
50+
(cond
51+
(nil? vr) `(throw (ex-info (format "`%s` does not exist" '~sym) {}))
52+
(:macro m) `(def ~(-> sym name symbol) ~sym)
53+
(:arglists m) `(def ~(-> sym name symbol) ~sym)
54+
:else `(def ~(-> sym name symbol) ~sym))))
55+
syms))]
56+
result))

‎adr/adr-001-http2.adoc

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
= ADR 001: HTTP/2 API
2+
3+
Date: 2023-04-10
4+
5+
6+
== Status
7+
8+
#Proposed#
9+
10+
A decision may be "proposed" if the project stakeholders haven't agreed with it
11+
yet, or "accepted" once it is agreed. If a later ADR changes or reverses a
12+
decision, it may be marked as "deprecated" or "superseded" with a reference to
13+
its replacement.
14+
15+
== Context
16+
17+
- https://datatracker.ietf.org/doc/html/rfc9113
18+
- https://web.dev/performance-http2/
19+
- https://www.rfc-editor.org/rfc/rfc8441
20+
21+
=== HTTP/2
22+
23+
HTTP/2 is eight years old now, and quite popular. It adds numerous advantages,
24+
some invisible, some requiring user coding. Adding support in both the client and
25+
server is obvious, but the installed base of Aleph is quite large (1.8m downloads),
26+
so we have a duty to maintain backwards compatibility while also exposing HTTP/2
27+
capabilities to those who want/need it.
28+
29+
The default API Aleph presents is a handler fn that receives a single Ring map
30+
with an `InputStream` body (potentially transformed by middleware), and returns
31+
a Ring response map, with a static or streamable body.
32+
33+
HTTP/2 offers new features that users may want to take advantage of: flow
34+
control, in particular. HTTP/2 introduced other features, in particular server
35+
push and prioritization, but they did not work out. Server push (the ability to
36+
send "responses" before receiving a request) has been criticized as difficult
37+
to use correctly, and effectively https://chromestatus.com/feature/6302414934114304[disabled by Chrome].
38+
Prioritization hasn't been disabled exactly, but it has various problems, and
39+
was replaced with a simpler scheme for HTTP/3.
40+
41+
Since the Aleph team is small, we have to carefully consider what to support.
42+
Full HTTP/2 support may not be advisable.
43+
44+
=== Prioritization
45+
46+
- https://datatracker.ietf.org/doc/html/rfc9218
47+
- https://calendar.perfplanet.com/2022/http-3-prioritization-demystified/
48+
- https://blog.cloudflare.com/better-http-2-prioritization-for-a-faster-web/
49+
- https://calendar.perfplanet.com/2018/http2-prioritization/
50+
- https://github.com/andydavies/http2-prioritization-issues
51+
52+
Prioritization in HTTP/2 is considered overly-complicated and was partially
53+
deprecated in RFC 9113. The browsers have different interpretations of how to
54+
arrange the priority tree, and Safari/Edge do not utilize it much at all. Many
55+
HTTP servers have broken or nonresponsive implementations. Many Linux servers
56+
have TCP buffers too large too effectively handle preempting lower-priority
57+
resources or change existing priorities.
58+
59+
Because of this RFC 9218 (Extensible Prioritization) was introduced. For HTTP/2,
60+
RFC 9218 is suggested as a backwards-compatible alternative, since it depends
61+
only on standard HTTP headers; it was adopted for HTTP/3 as the primary
62+
prioritization scheme.
63+
64+
=== Server Push
65+
66+
- https://docs.google.com/document/d/1K0NykTXBbbbTlv60t5MyJvXjqKGsCVNYHyLEXIxYMv0/edit
67+
- https://jakearchibald.com/2017/h2-push-tougher-than-i-thought/
68+
- https://developer.chrome.com/blog/removing-push/
69+
- https://chromestatus.com/feature/6302414934114304
70+
71+
HTTP/2 added the ability for servers to push unrequested data to clients, via
72+
the PUSH_PROMISE frame type. In theory, it can make better use of bandwidth by
73+
not waiting for browsers to discover and request necessary resources. In
74+
practice, it was found difficult to do so, accounting for buffer bloat, RTT, and
75+
browser caches, and not make things _worse_. Chrome effectively disabled it in
76+
Sept 2022.
77+
78+
=== Websockets
79+
80+
Websockets were built on top of HTTP/1.1-only fields, like the Upgrade and
81+
Connection headers and the 101 status code. RFC 8441 describes an HTTP/2 extension
82+
that allows websockets to be tunneled over an HTTP/2 stream. This allows efficient
83+
sharing of a single TCP connection for both HTTP/2 and websocket traffic, at the
84+
cost of a more complicated implementation. Netty does not have built-in support
85+
for it, though https://github.com/jauntsdn/netty-websocket-http2[user implementations]
86+
exist.
87+
88+
=== Proxies
89+
90+
HTTP/2 supports CONNECT proxying, but over a single HTTP/2 stream, not a whole TCP connection. Aleph has only ever supported using proxies from the client-side.
91+
92+
== Decision
93+
94+
=== Backwards-compatibility mode
95+
96+
We will maintain a backwards-compatible API for the majority of users, where
97+
their existing handlers will work as is. All streams will have equal priority,
98+
and flow control will ...
99+
100+
NOTE: What should the default flow control strategy be? An infinitely large
101+
window? The default 64kb window, that auto-replenishes as soon as bytes are read
102+
off? Netty offers both of the previous as default strategies available for use.
103+
104+
NOTE: What should a raw stream look like? A sequence of ByteBufs? Or a sequence
105+
of Frames, which is Netty's "raw" level for HTTP/2?
106+
107+
=== HTTP/2 connection mode
108+
We will NOT support server push, as our resources are too limited relative to
109+
the minimal potential benefit. Outside PRs may be considered, depending on the
110+
expected maintenance burden.
111+
112+
We will expose a connection-level handler API, where the handler can access the
113+
entire connection, all streams in the connection, and all frames in the streams.
114+
We will expose APIs to change connection and stream settings.
115+
116+
We will add APIs to get/set stream priorities and dependencies. We will NOT do
117+
anything with that knowledge. It will be up to the users to decide what to do
118+
with the priority/dependencies.
119+
120+
NOTE: Is there an obvious priority/dependency win we can offer? Maybe from Netty?
121+
122+
NOTE: Is there a way to assist in things like pausing or throttling other streams?
123+
Manifold's `throttle` doesn't support a function to change throttling behavior.
124+
And nothing supports pausing a stream at the moment.
125+
126+
We will add APIs to expose flow control for getting/setting both stream-level
127+
and connection-level window sizes.
128+
129+
NOTE: Do we need to do anything special to incorporate Manifold backpressure
130+
into this somehow?
131+
132+
=== Websockets
133+
134+
We will NOT add support for HTTP/2 websocket tunneling at this time. The
135+
cost/benefit ratio is too low. PRs may be considered, though.
136+
137+
=== Proxies
138+
139+
NOTE: TBD
140+
141+
== Consequences
142+
143+
This will improve web support for Clojure. I believe http-kit doesn't support
144+
HTTP/2, and Pedestal, if it supports it, only does so through Jetty, which is
145+
not very performant.
146+
147+
This will give us a chance to improve the clarity of the code.
148+
149+
As with any major change, this will add to the maintenance burden, and run the
150+
risk of breaking things, not just in HTTP/2 code, but also in pre-existing,
151+
overlapping HTTP/1.1 code.
152+
153+
''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
154+
155+
See https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions[Documenting architecture decisions - Michael Nygard]
156+
for the format and rationale of this document.

‎deps.edn

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
{:mvn/version "1.2.4", :exclusions [org.clojure/clojure]},
66
manifold/manifold
77
{:mvn/version "0.3.0", :exclusions [org.clojure/tools.logging]},
8-
org.clj-commons/byte-streams {:mvn/version "0.3.1"},
8+
org.clj-commons/byte-streams {:mvn/version "0.3.2"},
99
org.clj-commons/dirigiste {:mvn/version "1.0.3"},
1010
org.clj-commons/primitive-math {:mvn/version "1.0.0"},
11-
potemkin/potemkin {:mvn/version "0.4.5"},
11+
potemkin/potemkin {:mvn/version "0.4.6"},
1212
io.netty/netty-transport {:mvn/version "4.1.94.Final"},
1313
io.netty/netty-transport-native-epoll$linux-x86_64
1414
{:mvn/version "4.1.94.Final"},
@@ -22,6 +22,7 @@
2222
{:mvn/version "0.0.18.Final"},
2323
io.netty/netty-codec {:mvn/version "4.1.94.Final"},
2424
io.netty/netty-codec-http {:mvn/version "4.1.94.Final"},
25+
io.netty/netty-codec-http2 {:mvn/version "4.1.94.Final"},
2526
io.netty/netty-handler {:mvn/version "4.1.94.Final"},
2627
io.netty/netty-handler-proxy {:mvn/version "4.1.94.Final"},
2728
io.netty/netty-resolver {:mvn/version "4.1.94.Final"},
@@ -35,7 +36,7 @@
3536
{:git/sha "1bcf2fbbcbef611381e5e9ccdc77bec1e62ea5e5"}},
3637
:ns-default lein2deps.build,
3738
:lein2deps/compile-java
38-
{:src-dirs ["src/aleph/utils"],
39+
{:src-dirs ["src-java"],
3940
:class-dir "target/classes",
4041
:javac-opts ["-target" "1.8" "-source" "1.8"]}}},
4142
:deps/prep-lib

‎project.clj

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77
:license {:name "MIT License"}
88
:dependencies [[org.clojure/tools.logging "1.2.4" :exclusions [org.clojure/clojure]]
99
[manifold "0.3.0" :exclusions [org.clojure/tools.logging]]
10-
[org.clj-commons/byte-streams "0.3.1"]
10+
[org.clj-commons/byte-streams "0.3.2"]
1111
[org.clj-commons/dirigiste "1.0.3"]
1212
[org.clj-commons/primitive-math "1.0.0"]
13-
[potemkin "0.4.5"]
13+
[potemkin "0.4.6"]
1414
[io.netty/netty-transport ~netty-version]
1515
[io.netty/netty-transport-native-epoll ~netty-version :classifier "linux-x86_64"]
1616
[io.netty/netty-transport-native-epoll ~netty-version :classifier "linux-aarch_64"]
@@ -19,12 +19,13 @@
1919
[io.netty.incubator/netty-incubator-transport-native-io_uring "0.0.18.Final" :classifier "linux-aarch_64"]
2020
[io.netty/netty-codec ~netty-version]
2121
[io.netty/netty-codec-http ~netty-version]
22+
[io.netty/netty-codec-http2 ~netty-version]
2223
[io.netty/netty-handler ~netty-version]
2324
[io.netty/netty-handler-proxy ~netty-version]
2425
[io.netty/netty-resolver ~netty-version]
2526
[io.netty/netty-resolver-dns ~netty-version]
2627
[metosin/malli "0.10.4" :exclusions [org.clojure/clojure]]]
27-
:profiles {:dev {:dependencies [[org.clojure/clojure "1.11.0"]
28+
:profiles {:dev {:dependencies [[org.clojure/clojure "1.11.1"]
2829
[criterium "0.4.6"]
2930
[cheshire "5.10.0"]
3031
[org.slf4j/slf4j-simple "1.7.30"]
@@ -33,16 +34,23 @@
3334
;; This is for self-generating certs for testing ONLY:
3435
[org.bouncycastle/bcprov-jdk18on "1.72"]
3536
[org.bouncycastle/bcpkix-jdk18on "1.72"]]
36-
:jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=debug"]}
37+
:jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=debug"
38+
"-Dorg.slf4j.simpleLogger.showThreadName=false"
39+
"-Dorg.slf4j.simpleLogger.showThreadId=true"
40+
"-Dorg.slf4j.simpleLogger.showLogName=false"
41+
"-Dorg.slf4j.simpleLogger.showShortLogName=true"
42+
"-Dorg.slf4j.simpleLogger.showDateTime=true"]}
3743
:test {:jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=off"]}
38-
:pedantic {:pedantic? :abort}}
39-
:java-source-paths ["src/aleph/utils"]
40-
:test-selectors {:default #(not
41-
(some #{:benchmark :stress}
42-
(cons (:tag %) (keys %))))
44+
:leak-level-paranoid {:jvm-opts ["-Dio.netty.leakDetectionLevel=PARANOID"]}
45+
:pedantic {:pedantic? :abort}
46+
:trace {:jvm-opts ["-Dorg.slf4j.simpleLogger.defaultLogLevel=trace"]}}
47+
:java-source-paths ["src-java"]
48+
:test-selectors {:default #(not
49+
(some #{:benchmark :stress}
50+
(cons (:tag %) (keys %))))
4351
:benchmark :benchmark
44-
:stress :stress
45-
:all (constantly true)}
52+
:stress :stress
53+
:all (constantly true)}
4654
:jvm-opts ^:replace ["-server"
4755
"-Xmx2g"
4856
"-XX:+HeapDumpOnOutOfMemoryError"

0 commit comments

Comments
 (0)