diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13576e962..aecd61eda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,10 +32,6 @@ jobs: - windows-latest arch: - default - include: - - os: windows-latest - version: '1' - arch: x86 exclude: - os: macOS-latest version: 'min' # no apple silicon support for Julia 1.6 diff --git a/CHANGELOG.md b/CHANGELOG.md index b32569b11..04d34f787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## [v2.0.0] - 2025-02-01 +### Changed + + ## [v1.10.1] - 2023-11-28 ### Changed - Server errors are no longer serialized back to the client since this might leak sensitive diff --git a/Project.toml b/Project.toml index e485b3a45..030ae0288 100644 --- a/Project.toml +++ b/Project.toml @@ -1,47 +1,33 @@ name = "HTTP" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" authors = ["Jacob Quinn", "contributors: https://github.com/JuliaWeb/HTTP.jl/graphs/contributors"] -version = "1.10.15" +version = "2.0.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" -ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -ExceptionUnwrapping = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LibAwsCommon = "c6e421ba-b5f8-4792-a1c4-42948de3ed9d" +LibAwsHTTPFork = "d3f1d20b-921e-4930-8491-471e0be3121a" +LibAwsIO = "a5388770-19df-4151-b103-3d71de896ddf" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" -LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36" -MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d" -NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +Mmap = "a63ad114-7e13-5084-954f-fe012c677804" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -SimpleBufferStream = "777ac1f9-54b0-4bf8-805c-2214025038e7" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -CodecZlib = "0.7" -ConcurrentUtilities = "2.4" -ExceptionUnwrapping = "0.1" -LoggingExtras = "0.4.9,1" -MbedTLS = "0.6.8, 0.7, 1" -OpenSSL = "1.3" +JSON = "0.21.4" +LibAwsHTTPFork = "1.0.2" +LibAwsIO = "1.2.0" PrecompileTools = "1.2.1" -SimpleBufferStream = "1.1" -URIs = "1.3" -julia = "1.6" +julia = "1.10" [extras] -BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d" -Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" [targets] -test = ["BufferedStreams", "Deno_jll", "Distributed", "InteractiveUtils", "JSON", "Test", "Unitful", "NetworkOptions"] +test = ["JSON", "Test"] diff --git a/docs/examples/cors_server.jl b/docs/examples/cors_server.jl deleted file mode 100644 index 2275b3619..000000000 --- a/docs/examples/cors_server.jl +++ /dev/null @@ -1,152 +0,0 @@ -""" -Server example that takes after the simple server, however, -handles dealing with CORS preflight headers when dealing with more -than just a simple request. For CORS details, see e.g. https://cors-errors.info/ -""" - -using HTTP, JSON3, StructTypes, Sockets, UUIDs - -# modified Animal struct to associate with specific user -mutable struct Animal - id::Int - userId::UUID - type::String - name::String - Animal() = new() -end - -StructTypes.StructType(::Type{Animal}) = StructTypes.Mutable() - -# use a plain `Dict` as a "data store", outer Dict maps userId to user-specific Animals -const ANIMALS = Dict{UUID, Dict{Int, Animal}}() -const NEXT_ID = Ref(0) -function getNextId() - id = NEXT_ID[] - NEXT_ID[] += 1 - return id -end - -# CORS preflight headers that show what kinds of complex requests are allowed to API -const CORS_OPT_HEADERS = [ - "Access-Control-Allow-Origin" => "*", - "Access-Control-Allow-Headers" => "*", - "Access-Control-Allow-Methods" => "POST, GET, OPTIONS" -] - -# CORS response headers that set access right of the recepient -const CORS_RES_HEADERS = ["Access-Control-Allow-Origin" => "*"] - -#= -JSONMiddleware minimizes code by automatically converting the request body -to JSON to pass to the other service functions automatically. JSONMiddleware -recieves the body of the response from the other service funtions and sends -back a success response code -=# -function JSONMiddleware(handler) - # Middleware functions return *Handler* functions - return function(req::HTTP.Request) - # first check if there's any request body - if isempty(req.body) - # we slightly change the Handler interface here because we know - # our handler methods will either return nothing or an Animal instance - ret = handler(req) - else - # replace request body with parsed Animal instance - req.body = JSON3.read(req.body, Animal) - ret = handler(req) - end - # return a Response, if its a response already (from 404 and 405 handlers) - if ret isa HTTP.Response - return ret - else # otherwise serialize any Animal as json string and wrap it in Response - return HTTP.Response(200, CORS_RES_HEADERS, ret === nothing ? "" : JSON3.write(ret)) - end - end -end - -#= CorsMiddleware: handles preflight request with the OPTIONS flag -If a request was recieved with the correct headers, then a response will be -sent back with a 200 code, if the correct headers were not specified in the request, -then a CORS error will be recieved on the client side - -Since each request passes throught the CORS Handler, then if the request is -not a preflight request, it will simply go to the JSONMiddleware to be passed to the -correct service function =# -function CorsMiddleware(handler) - return function(req::HTTP.Request) - if HTTP.method(req)=="OPTIONS" - return HTTP.Response(200, CORS_OPT_HEADERS) - else - return handler(req) - end - end -end - -# **simplified** "service" functions -function createAnimal(req::HTTP.Request) - animal = req.body - animal.id = getNextId() - ANIMALS[animal.userId][animal.id] = animal - return animal -end - -function getAnimal(req::HTTP.Request) - # retrieve our matched path parameters from registered route - animalId = parse(Int, HTTP.getparams(req)["id"]) - userId = UUID(HTTP.getparams(req)["userId"]) - return ANIMALS[userId][animalId] -end - -function updateAnimal(req::HTTP.Request) - animal = req.body - ANIMALS[animal.userId][animal.id] = animal - return animal -end - -function deleteAnimal(req::HTTP.Request) - # retrieve our matched path parameters from registered route - animalId = parse(Int, HTTP.getparams(req)["id"]) - userId = UUID(HTTP.getparams(req)["userId"]) - delete!(ANIMALS[userId], animal.id) - return nothing -end - -function createUser(req::HTTP.Request) - userId = uuid4() - ANIMALS[userId] = Dict{Int, Animal}() - return userId -end - -# CORS handlers for error responses -cors404(::HTTP.Request) = HTTP.Response(404, CORS_RES_HEADERS, "") -cors405(::HTTP.Request) = HTTP.Response(405, CORS_RES_HEADERS, "") - -# add an additional endpoint for user creation -const ANIMAL_ROUTER = HTTP.Router(cors404, cors405) -HTTP.register!(ANIMAL_ROUTER, "POST", "/api/zoo/v1/users", createUser) -# modify service endpoints to have user pass UUID in -HTTP.register!(ANIMAL_ROUTER, "POST", "/api/zoo/v1/users/{userId}/animals", createAnimal) -HTTP.register!(ANIMAL_ROUTER, "GET", "/api/zoo/v1/users/{userId}/animals/{id}", getAnimal) -HTTP.register!(ANIMAL_ROUTER, "DELETE", "/api/zoo/v1/users/{userId}/animals/{id}", deleteAnimal) - -server = HTTP.serve!(ANIMAL_ROUTER |> JSONMiddleware |> CorsMiddleware, Sockets.localhost, 8080) - -# using our server -resp = HTTP.post("http://localhost:8080/api/zoo/v1/users") -userId = JSON3.read(resp.body, UUID) -x = Animal() -x.userId = userId -x.type = "cat" -x.name = "pete" -# create 1st animal -resp = HTTP.post("http://localhost:8080/api/zoo/v1/users/$(userId)/animals", [], JSON3.write(x)) -x2 = JSON3.read(resp.body, Animal) -# retrieve it back -resp = HTTP.get("http://localhost:8080/api/zoo/v1/users/$(userId)/animals/$(x2.id)") -x3 = JSON3.read(resp.body, Animal) -# try bad path -resp = HTTP.get("http://localhost:8080/api/zoo/v1/badpath") - -# close the server which will stop the HTTP server from listening -close(server) -@assert istaskdone(server.task) diff --git a/docs/examples/readme_examples.jl b/docs/examples/readme_examples.jl deleted file mode 100644 index 282453c5a..000000000 --- a/docs/examples/readme_examples.jl +++ /dev/null @@ -1,72 +0,0 @@ -#CLIENT - -#HTTP.request sends a HTTP Request Message and returns a Response Message. - -r = HTTP.request("GET", "http://httpbin.org/ip") -println(r.status) -println(String(r.body)) - -#HTTP.open sends a HTTP Request Message and opens an IO stream from which the Response can be read. -HTTP.open(:GET, "https://tinyurl.com/bach-cello-suite-1-ogg") do http - open(`vlc -q --play-and-exit --intf dummy -`, "w") do vlc - write(vlc, http) - end -end - -#SERVERS - -#Using HTTP.Servers.listen: -#The server will start listening on 127.0.0.1:8081 by default. - -using HTTP - -HTTP.listen() do http::HTTP.Stream - @show http.message - @show HTTP.header(http, "Content-Type") - while !eof(http) - println("body data: ", String(readavailable(http))) - end - HTTP.setstatus(http, 404) - HTTP.setheader(http, "Foo-Header" => "bar") - HTTP.startwrite(http) - write(http, "response body") - write(http, "more response body") -end - -#Using HTTP.Handlers.serve: - -using HTTP - -# HTTP.listen! and HTTP.serve! are the non-blocking versions of HTTP.listen/HTTP.serve -server = HTTP.serve!() do request::HTTP.Request - @show request - @show request.method - @show HTTP.header(request, "Content-Type") - @show request.body - try - return HTTP.Response("Hello") - catch e - return HTTP.Response(400, "Error: $e") - end - end - # HTTP.serve! returns an `HTTP.Server` object that we can close manually - close(server) - -#WebSocket Examples -using HTTP.WebSockets -server = WebSockets.listen!("127.0.0.1", 8081) do ws - for msg in ws - send(ws, msg) - end - end - -WebSockets.open("ws://127.0.0.1:8081") do ws - send(ws, "Hello") - s = receive(ws) - println(s) - end; -Hello -#Output: Hello - -close(server) - diff --git a/docs/examples/server_sent_events.jl b/docs/examples/server_sent_events.jl deleted file mode 100644 index fa3750af7..000000000 --- a/docs/examples/server_sent_events.jl +++ /dev/null @@ -1,106 +0,0 @@ -""" -Simple server that implements [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events), -loosely following [this tutorial](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events). - -### Example client code (JS): -```http - - - - Server-sent events demo - - -

Fetched items:

- - - - -``` - -### Example client code (Julia) -```julia -using HTTP, JSON - -HTTP.open("GET", "http://127.0.0.1:8080/api/events") do io - while !eof(io) - println(String(readavailable(io))) - end -end -``` - -### Server code: -""" -using HTTP, Sockets, JSON - -const ROUTER = HTTP.Router() - -function getItems(req::HTTP.Request) - headers = [ - "Access-Control-Allow-Origin" => "*", - "Access-Control-Allow-Methods" => "GET, OPTIONS" - ] - if HTTP.method(req) == "OPTIONS" - return HTTP.Response(200, headers) - end - return HTTP.Response(200, headers, JSON.json(rand(2))) -end - -function events(stream::HTTP.Stream) - HTTP.setheader(stream, "Access-Control-Allow-Origin" => "*") - HTTP.setheader(stream, "Access-Control-Allow-Methods" => "GET, OPTIONS") - HTTP.setheader(stream, "Content-Type" => "text/event-stream") - - if HTTP.method(stream.message) == "OPTIONS" - return nothing - end - - HTTP.setheader(stream, "Content-Type" => "text/event-stream") - HTTP.setheader(stream, "Cache-Control" => "no-cache") - while true - write(stream, "event: ping\ndata: $(round(Int, time()))\n\n") - if rand(Bool) - write(stream, "data: $(rand())\n\n") - end - sleep(1) - end - return nothing -end - -HTTP.register!(ROUTER, "GET", "/api/getItems", HTTP.streamhandler(getItems)) -HTTP.register!(ROUTER, "/api/events", events) - -server = HTTP.serve!(ROUTER, "127.0.0.1", 8080; stream=true) - -# Julia usage -resp = HTTP.get("http://localhost:8080/api/getItems") - -close = Ref(false) -@async HTTP.open("GET", "http://127.0.0.1:8080/api/events") do io - while !eof(io) && !close[] - println(String(readavailable(io))) - end -end - -# run the following to stop the streaming client request -close[] = true - -# close the server which will stop the HTTP server from listening -close(server) -@assert istaskdone(server.task) diff --git a/docs/examples/session.jl b/docs/examples/session.jl deleted file mode 100644 index 9b78a26ca..000000000 --- a/docs/examples/session.jl +++ /dev/null @@ -1,14 +0,0 @@ -""" -A simple example of creating a persistent session and logging into a web form. HTTP.jl does not have a distinct session object like requests.session() or rvest::html_session() but rather uses the `cookies` flag along with standard functions -""" -using HTTP - -#dummy site, any credentials work -url = "http://quotes.toscrape.com/login" -session = HTTP.get(url; cookies = true) - -credentials = Dict( - "Username" => "username", - "Password" => "password") - -response = HTTP.post(url, credentials) diff --git a/docs/examples/simple_server.jl b/docs/examples/simple_server.jl deleted file mode 100644 index d0fc37be1..000000000 --- a/docs/examples/simple_server.jl +++ /dev/null @@ -1,76 +0,0 @@ -""" -A simple example of creating a server with HTTP.jl. It handles creating, deleting, -updating, and retrieving Animals from a dictionary through 4 different routes -""" -using HTTP, JSON3, StructTypes, Sockets - -# modified Animal struct to associate with specific user -mutable struct Animal - id::Int - userId::Base.UUID - type::String - name::String - Animal() = new() -end - -StructTypes.StructType(::Type{Animal}) = StructTypes.Mutable() - -# use a plain `Dict` as a "data store" -const ANIMALS = Dict{Int, Animal}() -const NEXT_ID = Ref(0) -function getNextId() - id = NEXT_ID[] - NEXT_ID[] += 1 - return id -end - -# "service" functions to actually do the work -function createAnimal(req::HTTP.Request) - animal = JSON3.read(req.body, Animal) - animal.id = getNextId() - ANIMALS[animal.id] = animal - return HTTP.Response(200, JSON3.write(animal)) -end - -function getAnimal(req::HTTP.Request) - animalId = HTTP.getparams(req)["id"] - animal = ANIMALS[parse(Int, animalId)] - return HTTP.Response(200, JSON3.write(animal)) -end - -function updateAnimal(req::HTTP.Request) - animal = JSON3.read(req.body, Animal) - ANIMALS[animal.id] = animal - return HTTP.Response(200, JSON3.write(animal)) -end - -function deleteAnimal(req::HTTP.Request) - animalId = HTTP.getparams(req)["id"] - delete!(ANIMALS, animalId) - return HTTP.Response(200) -end - -# define REST endpoints to dispatch to "service" functions -const ANIMAL_ROUTER = HTTP.Router() -HTTP.register!(ANIMAL_ROUTER, "POST", "/api/zoo/v1/animals", createAnimal) -# note the use of `*` to capture the path segment "variable" animal id -HTTP.register!(ANIMAL_ROUTER, "GET", "/api/zoo/v1/animals/{id}", getAnimal) -HTTP.register!(ANIMAL_ROUTER, "PUT", "/api/zoo/v1/animals", updateAnimal) -HTTP.register!(ANIMAL_ROUTER, "DELETE", "/api/zoo/v1/animals/{id}", deleteAnimal) - -server = HTTP.serve!(ANIMAL_ROUTER, Sockets.localhost, 8080) - -# using our server -x = Animal() -x.type = "cat" -x.name = "pete" -# create 1st animal -resp = HTTP.post("http://localhost:8080/api/zoo/v1/animals", [], JSON3.write(x)) -x2 = JSON3.read(resp.body, Animal) -# retrieve it back -resp = HTTP.get("http://localhost:8080/api/zoo/v1/animals/$(x2.id)") -x3 = JSON3.read(resp.body, Animal) - -# close the server which will stop the HTTP server from listening -close(server) -@assert istaskdone(server.task) diff --git a/docs/examples/squaring_server_client.jl b/docs/examples/squaring_server_client.jl deleted file mode 100644 index 4f59138f2..000000000 --- a/docs/examples/squaring_server_client.jl +++ /dev/null @@ -1,73 +0,0 @@ -""" -Simple server in Julia and client code in JS. - -### Example client code (JS): -```http - - - - Squaring numbers - - - - -

Outputs

- - - - -``` - -### Server code: -""" -using HTTP - -const ROUTER = HTTP.Router() - -function square(req::HTTP.Request) - headers = [ - "Access-Control-Allow-Origin" => "*", - "Access-Control-Allow-Methods" => "POST, OPTIONS" - ] - # handle CORS requests - if HTTP.method(req) == "OPTIONS" - return HTTP.Response(200, headers) - end - body = parse(Float64, String(req.body)) - square = body^2 - HTTP.Response(200, headers, string(square)) -end - -HTTP.register!(ROUTER, "POST", "/api/square", square) - -server = HTTP.serve!(ROUTER, Sockets.localhost, 8080) - -# usage -resp = HTTP.post("http://localhost:8080/api/square"; body="3") -sq = parse(Float64, String(resp.body)) -@assert sq == 9.0 - -# close the server which will stop the HTTP server from listening -close(server) -@assert istaskdone(server.task) diff --git a/docs/make.jl b/docs/make.jl index 993a9b552..dbcdaf491 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,107 +1,28 @@ -using Documenter, HTTP, Sockets, URIs, Changelog - -# Generate Documenter-consumable changelog -Changelog.generate( - Changelog.Documenter(), - joinpath(@__DIR__, "../CHANGELOG.md"), - joinpath(@__DIR__, "src/changelog.md"); - repo = "JuliaWeb/HTTP.jl", -) - - -""" -Loops through the files in the examples folder and adds them (with any header comments) to the examples.md -markdown file. -""" -function generateExamples() - f = IOBuffer() - write( - f, - "```@meta - # NOTE this file is autogenerated, do not edit examples.md directly. To make an example, upload the .jl file to the examples folder. Header comments may be included at the top of the file using \"\"\" syntax -``` ", - ) - write(f, "\n") - write(f, "# Examples") - write( - f, - "\nSome examples that may prove potentially useful for those using -`HTTP.jl`. The code for these examples can also be found on Github - in the `docs/examples` folder.", - ) - for (root, dirs, files) in walkdir(joinpath(@__DIR__, "examples")) - #set order of files so simple is first, and Readme examples are last - temp = files[1] - files[findfirst(isequal("simple_server.jl"), files)] = temp - files[1] = "simple_server.jl" - temp = files[2] - files[findfirst(isequal("cors_server.jl"), files)] = temp - files[2] = "cors_server.jl" - temp = files[length(files)] - files[findfirst(isequal("readme_examples.jl"), files)] = temp - files[length(files)] = "readme_examples.jl" - - for file in files - println(file) - #extract title from example - write(f, "\n") - title = file - title = replace(title, "_" => " ") - title = replace(title, ".jl" => "") - title = titlecase(title) - title = "## " * title * "\n" - write(f, title) - #open each file and read contents - opened = open(joinpath(@__DIR__, "examples/") * file) - lines = readlines(opened, keep = true) - index = 1 - #find doc string intro if exists - if "\"\"\"\n" in lines - index = findall(isequal("\"\"\"\n"), lines)[2] - print(index) - for i = 2:index-1 - write(f, lines[i]) - end - lines = lines[index+1:end] - end - - write(f, "```julia") - write(f, "\n") - for line in lines - write(f, line) - end - write(f, "\n") - write(f, "```") - close(opened) - end - end - file = joinpath(@__DIR__, "src/examples.md") - current_content = isfile(file) ? read(file, String) : "" - updated_content = String(take!(f)) - # Only update content if something changed so that the file watcher in - # LiveServer.jl isn't triggering itself when running make.jl. - if updated_content != current_content - write(file, updated_content) - end -end - -generateExamples() +# docs/make.jl +using Documenter, HTTP makedocs( - # modules = [HTTP], - sitename = "HTTP.jl", + sitename = "HTTP.jl v$(HTTP.VERSION)", + format = Documenter.HTML(), + modules = [HTTP], pages = [ "Home" => "index.md", - "client.md", - "server.md", - "websockets.md", - "reference.md", - "examples.md", - "changelog.md", + "Manual" => [ + "Client Guide" => "manual/client.md", + "Server Guide" => "manual/server.md", + "Middleware Guide" => "manual/middleware.md", + "WebSockets" => "manual/websockets.md", + "Authentication" => "manual/authentication.md", + ], + "API Reference" => "api/reference.md" ], + clean = true, + strict = true, ) -deploydocs( - repo = "github.com/JuliaWeb/HTTP.jl.git", - push_preview = true, -) +if get(ENV, "DEPLOY_DOCS", "false") == "true" + deploydocs( + repo = "github.com/JuliaWeb/HTTP.jl.git", + push_preview = true, + ) +end \ No newline at end of file diff --git a/docs/src/api/reference.md b/docs/src/api/reference.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/src/client.md b/docs/src/client.md deleted file mode 100644 index f523e6167..000000000 --- a/docs/src/client.md +++ /dev/null @@ -1,294 +0,0 @@ -# Client - -HTTP.jl provides a wide range of HTTP client functionality, mostly exposed through the [`HTTP.request`](@ref) family of functions. This document aims to walk through the various ways requests can be configured to accomplish whatever your goal may be. - -## Basic Usage - -The standard form for making requests is: - -```julia -HTTP.request(method, url [, headers [, body]]; ]) -> HTTP.Response -``` - -First, let's walk through the positional arguments. - -### Method - -`method` refers to the HTTP method (sometimes known as "verb"), including GET, POST, PUT, DELETE, PATCH, TRACE, etc. It can be provided either as a `String` like `HTTP.request("GET", ...)`, or a `Symbol` like `HTTP.request(:GET, ...)`. There are also convenience methods for the most common methods: - * `HTTP.get(...)` - * `HTTP.post(...)` - * `HTTP.put(...)` - * `HTTP.delete(...)` - * `HTTP.patch(...)` - * `HTTP.head(...)` - -These methods operate identically to `HTTP.request`, except the `method` argument is "builtin" via function name. - -### Url - -For the request `url` argument, the [URIs.jl](https://github.com/JuliaWeb/URIs.jl) package is used to parse this `String` argument into a `URI` object, which detects the HTTP scheme, user info, host, port (if any), path, query parameters, fragment, etc. The host and port will be used to actually make a connection to the remote server and send data to and receive data from. Query parameters can be included in the `url` `String` itself, or passed as a separate [`query`](@ref) keyword argument to `HTTP.request`, which will be discussed in more detail later. - -### Headers - -The `headers` argument is an optional list of header name-value pairs to be included in the request. They can be provided as a `Vector` of `Pair`s, like `["header1" => "value1", "header2" => "value2"]`, or a `Dict`, like `Dict("header1" => "value1", "header2" => "value2")`. Header names do not have to be unique, so any argument passed will be converted to `Vector{Pair}`. By default, `HTTP.request` will include a few headers automatically, including `"Host" => url.host`, `"Accept" => "*/*"`, `"User-Agent" => "HTTP.jl/1.0"`, and `"Content-Length" => body_length`. These can be overwritten by providing them yourself, like `HTTP.get(url, ["Accept" => "application/json"])`, or you can prevent the header from being included by default by including it yourself with an empty string as value, like `HTTP.get(url, ["Accept" => ""])`, following the curl convention. There are keyword arguments that control the inclusion/setting of other headers, like `basicauth`, `detect_content_type`, `cookies`, etc. that will be discussed in more detail later. - -### Body - -The optional `body` argument makes up the "body" of the sent request and is only used for HTTP methods that expect a body, like POST and PUT. A variety of objects are supported: - * a `Dict` or `NamedTuple` to be serialized as the "application/x-www-form-urlencoded" content type, so `Dict("nm" => "val")` will be sent in the request body like `nm=val` - * any `AbstractString` or `AbstractVector{UInt8}` which will be sent "as is" for the request body - * a readable `IO` stream or any `IO`-like type `T` for which `eof(T)` and `readavailable(T)` are defined. This stream will be read and sent until `eof` is `true`. This object should support the `mark`/`reset` methods if request retires are desired (if not, no retries will be attempted). - * Any collection or iterable of the above (`Dict`, `AbstractString`, `AbstractVector{UInt8}`, or `IO`) which will result in a "Transfer-Encoding=chunked" request body, where each iterated element will be sent as a separate chunk - * a [`HTTP.Form`](@ref), which will be serialized as the "multipart/form-data" content-type - -### Response - -Upon successful completion, `HTTP.request` returns a [`HTTP.Response`](@ref) object. It includes the following fields: - * `status`: the HTTP status code of the response, e.g. `200` for normal response - * `headers`: a `Vector` of `String` `Pair`s for each name=value pair in the response headers. Convenience methods for working with headers include: `HTTP.hasheader(resp, key)` to check if header exists; `HTTP.header(resp, key)` retrieve the value of a header with name `key`; `HTTP.headers(resp, key)` retrieve a list of all the headers with name `key`, since headers can have duplicate names - * `body`: a `Vector{UInt8}` of the response body bytes. Alternatively, an `IO` object can be provided via the [`response_stream`](@ref) keyword argument to have the response body streamed as it is received. - -## Keyword Arguments - -A number of keyword arguments are provided to give fine-tuned control over the request process. - -### `query` - -Query parameters are included in the url of the request like `http://httpbin.org/anything?q1=v1&q2=v2`, where the string `?q1=v1&q2=v2` after the question mark represent the "query parameters". They are essentially a list of key-value pairs. Query parameters can be included in the url itself when calling `HTTP.request`, like `HTTP.request(:GET, "http://httpbin.org/anything?q1=v1&q2=v2")`, but oftentimes, it's convenient to generate and pass them programmatically. To do this, pass an object that iterates `String` `Pair`s to the `query` keyword argument, like the following examples: - * `HTTP.get(url; query=Dict("x1" => "y1", "x2" => "y2")` - * `HTTP.get(url; query=["x1" => "y1", "x1" => "y2"]`: this form allows duplicate key values - * `HTTP.get(url; query=[("x1", "y1)", ("x2", "y2")]` - -### `response_stream` - -By default, the `HTTP.Response` body is returned as a `Vector{UInt8}`. There may be scenarios, however, where more control is desired, like downloading large files, where it's preferable to stream the response body directly out to file or into some other `IO` object. By passing a writeable `IO` object to the `response_stream` keyword argument, the response body will not be fully materialized and will be written to as it is received from the remote connection. Note that in the presence of request redirects and retries, multiple requests end up being made in a single call to `HTTP.request` by default (configurable via the [`redirects`](@ref) and [`retry`](@ref) keyword arguments). If `response_stream` is provided and a request is redirected or retried, the `response_stream` is not written to until the *final* request is completed (either the redirect is successfully followed, or the request doesn't need to be retried, etc.). - -#### Examples - -Stream body to file: -```julia -io = open("get_data.txt", "w") -r = HTTP.request("GET", "http://httpbin.org/get", response_stream=io) -close(io) -println(read("get_data.txt", String)) -``` - -Stream body through buffer: -```julia -r = HTTP.get("http://httpbin.org/get", response_stream=IOBuffer()) -println(String(take!(r.body))) -``` - -### `verbose` - -HTTP requests can be a pain sometimes. We get it. It can be tricky to get the headers, query parameters, or expected body in just the right format. For convenience, the `verbose` keyword argument is provided to enable debug logging for the duration of the call to `HTTP.request`. This can be helpful to "peek under the hood" of what all goes on in the process of making request: are redirects returned and followed? Is the connection not getting made to the right host? Is some error causing the request to be retried unexpectedly? Currently, the `verbose` keyword argument supports passing increasing levels to enable more and more verbose logging, from `0` (no debug logging, the default), up to `3` (most verbose, probably too much for anyone but package developers). - -If you're running into a real head-scratcher, don't hesitate to [open an issue](https://github.com/JuliaWeb/HTTP.jl/issues/new) and include the problem you're running into; it's most helpful when you can include the output of passing `verbose=3`, so package maintainers can see a detailed view of what's going on. - -### Connection keyword arguments - -#### `connect_timeout` - -When a connection is attempted to a remote host, sometimes the connection is unable to be established for whatever reason. Passing a non-zero `connect_timetout` value will cause `HTTP.request` to wait that many seconds before giving up and throwing an error. - -#### `pool` - -Many remote web services/APIs have rate limits or throttling in place to avoid bad actors from abusing their service. They may prevent too many requests over a time period or they may prevent too many connections being simultaneously open from the same client. By default, when `HTTP.request` opens a remote connection, it remembers the exact host:port combination and will keep the connection open to be reused by subsequent requests to the same host:port. The `pool` keyword argument specifies a specific `HTTP.Pool` object to be used for controlling the maximum number of concurrent connections allowed to be happening across the pool. It's constructed via `HTTP.Pool(max::Int)`. Requests attempted when the maximum is already hit will block until previous requests finish. The `idle_timeout` keyword argument can be passed to `HTTP.request` to control how long it's been since a connection was lasted used in order to be considered 'valid'; otherwise, "stale" connections will be discarded. - -#### `readtimeout` - -After a connection is established and a request is sent, a response is expected. If a non-zero value is passed to the `readtimeout` keyword argument, `HTTP.request` will wait to receive a response that many seconds before throwing an error. Passing `readtimeout = 0` disables any timeout checking and is the default. - -### `status_exception` - -When a non-2XX HTTP status code is received in a response, this is meant to convey some error condition. 3XX responses typically deal with "redirects" where the request should actually try a different url (these are followed automatically by default in `HTTP.request`, though up to a limit; see [`redirect`](@ref)). 4XX status codes typically mean the remote server thinks something is wrong in how the request is made. 5XX typically mean something went wrong on the server-side when responding. By default, as mentioned previously, `HTTP.request` will attempt to follow redirect responses, and retry "retryable" requests (where the status code and original request method allow). If, after redirects/retries, a response still has a non-2XX response code, the default behavior is to throw an `HTTP.StatusError` exception to signal that the request didn't succeed. This behavior can be disabled by passing `status_exception=false`, where the `HTTP.Response` object will be returned with the non-2XX status code intact. - -### `logerrors` - -If `true`, `HTTP.StatusError`, `HTTP.TimeoutError`, `HTTP.IOError`, and `HTTP.ConnectError` will be logged via `@error` as they happen, regardless of whether the request is then retried or not. Useful for debugging or monitoring requests where there's worry of certain errors happening but ignored because of retries. - -### `logtag` - -If provided, will be used as the tag for error logging. Useful for debugging or monitoring requests. - -### `observelayers` - -If `true`, enables the `HTTP.observelayer` to wrap each client-side "layer" to track the amount of time spent in each layer as a request is processed. This can be useful for debugging performance issues. Note that when retries or redirects happen, the time spent in each layer is cumulative, as noted by the `[layer]_count`. The metrics are stored in the `Request.context` dictionary, and can be accessed like `HTTP.get(...).request.context`. - -### `basicauth` - -By default, if "user info" is detected in the request url, like `http://user:password@host`, the `Authorization: Basic` header will be added to the request headers before the request is sent. While not very common, some APIs use this form of authentication to verify requests. This automatic adding of the header can be disabled by passing `basicauth=false`. - -### `canonicalize_headers` - -In the HTTP specification, header names are case insensitive, yet it is sometimes desirable to send/receive headers in a more predictable format. Passing `canonicalize_headers=true` (false by default) will reformat all request *and* response headers to use the Canonical-Camel-Dash-Format. - -### `proxy` - -In certain network environments, connections to a proxy must first be made and external requests are then sent "through" the proxy. `HTTP.request` supports this workflow by allowing the passing of a proxy url via the `proxy` keyword argument. Alternatively, it's a common pattern for HTTP libraries to check for the `http_proxy`, `HTTP_PROXY`, `https_proxy`, `HTTPS_PROXY`, and `no_proxy` environment variables and, if present, be used for these kind of proxied requests. Note that environment variables typically should be set prior to starting the Julia process; alternatively, the `withenv` Julia function can be used to temporarily modify the current process environment variables, so it could be used like: - -```julia -resp = withenv("http_proxy" => proxy_url) do - HTTP.request(:GET, url) -end -``` - -The `no_proxy` argument is typically a comma-separated list of urls that should *not* use the proxy, and is parsed when the HTTP.jl package is loaded, and thus won't work with the `withenv` method mentioned. - -### `detect_content_type` - -By default, the `Content-Type` header is not included by `HTTP.request`. To automatically detect various content types (html, xml, pdf, images, zip, gzip, JSON) and set this header, pass `detect_content_type=true`. - -### `decompress` - -By default, when the response includes the `Content-Encoding: gzip` header, the response body will be decompressed. To avoid this behavior, pass `decompress=false`. This keyword also controls the automatic inclusion of the `Accept-Encoding: gzip` header in the request being sent. - -### Retry arguments - -#### `retry` - -Controls overall whether requests will be retried at all; pass `retry=false` to disable all retries. - -#### `retries` - -Controls the total number of retries that will be attempted. Can also disable all retries by passing `retries = 0`. Note that for a request to be retried, in addition to the `retry` and `retries` keyword arguments, must be "retryable", which includes the following requirements: - * Request body must be static (string or bytes) or an `IO` the supports the `mark`/`reset` interface - * The request method must be idempotent as defined by RFC-7231, which includes GET, HEAD, OPTIONS, TRACE, PUT, and DELETE (not POST or PATCH). - * If the method _isn't_ idempotent, can pass `retry_non_idempotent=true` keyword argument to retry idempotent requests - * The retry limit hasn't been reached, as specified by `retries` keyword argument - * The "failed" response must have one of the following status codes: 403, 408, 409, 429, 500, 502, 503, 504, 599. - -#### `retry_non_idempotent` - -By default, this keyword argument is `false`, which controls whether non-idempotent requests will be retried (POST or PATCH requests). - -#### `retry_delays` - -Allows providing a custom `ExponentialBackOff` object to control the delay between retries. -Default is `ExponentialBackOff(n = retries)`. - -#### `retry_check` - -Allows providing a custom function to control whether a retry should be attempted. -The function should accept 5 arguments: the delay state, exception, request, response (an `HTTP.Response` object *if* a request was successfully made, otherwise `nothing`), and `resp_body` response body (which may be `nothing` if there is no response yet, otherwise a `Vector{UInt8}`), and return `true` if a retry should be attempted. So in traditional nomenclature, the function would have the form `f(s, ex, req, resp, resp_body) -> Bool`. - -### Redirect Arguments - -#### `redirect` - -This keyword argument controls whether responses that specify a redirect (via 3XX status code + `Location` header) will be "followed" by issuing a follow up request to the specified `Location`. There are certain rules/logic that are followed when deciding whether to redirect and _how_ the redirected request will be made: - * The `Location` redirect url must be "valid" with an appropriate scheme and host; if the new location is relative to the original request url, the new url is resolved by calling `URIs.resolvereference` - * The response status code is one of: 301, 302, 307, or 308 - * The method of the redirected request may change: 307 or 308 status codes will _not_ change the method, 303 means only a GET request is allowed, otherwise, if the `redirect_method` keyword argument is provided, it will be used. If not provided, the redirected request will default to a GET request. - * We'll only make a redirect request if we haven't made too many redirect attempts already, as controlled by the `redirect_limit` keyword argument (default 3) - * The original request headers will, by default, be forwarded in the redirected request, unless the new url is an entirely new host, then `Cookie` and `Authorization` headers will be removed. - -#### `redirect_limit` - -Controls how many redirects will be "followed" by making additional requests in the case of a redirected response url recursively returning redirect responses and so on. In addition to `redirect=false`, passing `redirect_limit=0` will also disable any redirect behavior all together. - -#### `redirect_method` - -May control the method that will be used in the redirected request. For 307 or 308 status codes, the same method will be used by default. For 303, only GET requests are allowed. For other status codes (301, 302), this keyword argument will be used to determine what the redirect request method will be. Passing `redirect_method=:same` will result in the same method as the original request. Otherwise, passing `redirect_method=:GET`, or any other valid method as `String` or `Symbol`, will result in that method for the redirected request. - -#### `forwardheaders` - -Controls whether original request headers will be included in the redirected request. `true` by default. Pass `forwardheaders=false` to disable headers being used in redirected requests. By design, if the new redirect location url is a different host than the original host, "sensitive" headers will not be forwarded, including `Cookie` and `Authorization` headers. - -### SSL Arguments - -#### `require_ssl_verification` - -Controls whether the SSL configuration for a secure connection is verified in the handshake process. `true` by default. Should only be set to `false` if developing against a local server that can be completely trusted. - -#### `sslconfig` - -Allows specifying a custom `MbedTLS.SSLConfig` configuration to be used in the secure connection handshake process to verify the connection. A custom cert and key file can be passed to construct a custom `SSLConfig` like `MbedTLS.SSLConfig(cert_file, key_file)`. - -### Cookie Arguments - -#### `cookies` - -Controls if and how a request `Cookie` header is set for a request. By default, `cookies` is `true`, which means the `cookiejar` (an internal global by default) will be checked for previously received `Set-Cookie` headers from past responses for the request domain and, if found, will be included in the outgoing request. Passing `cookies=false` will disable any automatic cookie tracking/setting. For more granular control, the `Cookie` header can be set manually in the `headers` request argument and `cookies=false` is passed. Otherwise, a `Dict{String, String}` can be passed in the `cookies` argument, which should be cookie name-value pairs to be serialized into the `Cookie` header for the current request. If `cookies=false` or an empty `Dict` is passed in the `cookies` keyword argument, no `Set-Cookie` response headers will be parsed and stored for use in future requests. In the automatic case of including previously received cookies, verification is done to ensure the cookie hasn't expired, matches the correct domain, etc. - -#### `cookiejar` - -If cookies are "enabled" (either by passing `cookies=true` or passing a non-empty `Dict` via `cookies` keyword argument), this keyword argument specifies the `HTTP.CookieJar` object that should be used to store received `Set-Cookie` cookie name-value pairs from responses. Cookies are stored for the appropriate host/domain and will, by default, be included in future requests when the host/domain match and other conditions are met (cookie hasn't expired, etc.). By default, a single global `CookieJar` is used, which is threadsafe. To pass a custom `CookieJar`, first create one: `jar = HTTP.CookieJar()`, then pass like `HTTP.get(...; cookiejar=jar)`. - -## Streaming Requests - -### `HTTP.open` - -Allows a potentially more convenient API when the request and/or response bodies need to be streamed. Works like: - -```julia -HTTP.open(method, url, [, headers]; kw...) do io - write(io, body) - [startread(io) -> HTTP.Response] - while !eof(io) - readavailable(io) -> AbstractVector{UInt8} - end -end -> HTTP.Response -``` - -Where the `io` argument provided to the function body is an `HTTP.Stream` object, a custom `IO` that represents an open connection that is ready to be written to in order to send the request body, and/or read from to receive the response body. Note that `startread(io)` should be called before calling `readavailable` to ensure the response status line and headers are received and parsed appropriately. Calling `eof(io)` will return true until the response body has been completely received. Note that the returned `HTTP.Response` from `HTTP.open` will _not_ have a `.body` field since the body was read in the function body. - -### Download - -A [`download`](@ref) function is provided for similar functionality to `Downloads.download`. - -## Client-side Middleware (Layers) - -An `HTTP.Layer` is an abstract type to represent a client-side middleware. -A layer is any function of the form `f(::Handler) -> Handler`, where [`Handler`](@ref) is -a function of the form `f(::Request) -> Response`. Note that this `Handler` definition is the same from -the server-side documentation. It may also be apparent -that a `Layer` is the same as the [`Middleware`](@ref) interface from server-side, which is true, but -we define `Layer` to clarify the client-side distinction and its unique usage. - -Creating custom layers can be a convenient way to "enhance" the `HTTP.request` process with custom functionality. It might be a layer that computes a special authorization header, or modifies the body in some way, or treats the response specially. Oftentimes, layers are application or domain-specific, where certain domain knowledge can be used to improve or simplify the request process. Layers can also be used to enforce the usage of certain keyword arguments if desired. - -Custom layers can be deployed in one of two ways: - * [`HTTP.@client`](@ref): Create a custom "client" with shorthand verb definitions, but which - include custom layers; only these new verb methods will use the custom layers. - * [`HTTP.pushlayer!`](@ref)/[`HTTP.poplayer!`](@ref): Allows globally adding and removing - layers from the default HTTP.jl layer stack; *all* http requests will then use the custom layers - -### Quick Examples -```julia -module Auth - -using HTTP - -function auth_layer(handler) - # returns a `Handler` function; check for a custom keyword arg `authcreds` that - # a user would pass like `HTTP.get(...; authcreds=creds)`. - # We also accept trailing keyword args `kw...` and pass them along later. - return function(req; authcreds=nothing, kw...) - # only apply the auth layer if the user passed `authcreds` - if authcreds !== nothing - # we add a custom header with stringified auth creds - HTTP.setheader(req, "X-Auth-Creds" => string(authcreds)) - end - # pass the request along to the next layer by calling `auth_layer` arg `handler` - # also pass along the trailing keyword args `kw...` - return handler(req; kw...) - end -end - -# Create a new client with the auth layer added -HTTP.@client [auth_layer] - -end # module - -# Can now use custom client like: -Auth.get(url; authcreds=creds) # performs GET request with auth_layer layer included - -# Or can include layer globally in all HTTP.jl requests -HTTP.pushlayer!(Auth.auth_layer) - -# Now can use normal HTTP.jl methods and auth_layer will be included -HTTP.get(url; authcreds=creds) -``` - -For more ideas or examples on how client-side layers work, it can be useful to see how `HTTP.request` is built on layers internally, in the [`/src/clientlayers`](https://github.com/JuliaWeb/HTTP.jl/tree/master/src/clientlayers) source code directory. diff --git a/docs/src/index.md b/docs/src/index.md index 4cbc3046a..74493cc77 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,156 +1,85 @@ # HTTP.jl Documentation -## Overview +Welcome to the HTTP.jl docs—a fast, full‑featured HTTP client and server library for Julia. -HTTP.jl provides both client and server functionality for the [http](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [websocket](https://en.wikipedia.org/wiki/WebSocket) protocols. As a client, it provides the ability to make a wide range of -requests, including GET, POST, websocket upgrades, form data, multipart, chunking, and cookie handling. There is also advanced functionality to provide client-side middleware and generate your own customized HTTP client. -On the server side, it provides the ability to listen, accept, and route http requests, with middleware and -handler interfaces to provide flexibility in processing responses. +## Overview -## Quickstart +HTTP.jl provides a comprehensive set of tools for sending HTTP requests, building HTTP servers, and managing WebSocket connections. This documentation is organized into the following sections: +- **Installation & Quick Start:** Learn how to install HTTP.jl and see basic examples. +- **Manual Guides:** Detailed instructions on client usage, server setup, middleware, and websockets. +- **API Reference:** An auto‑generated reference listing all public functions and types. -### Making requests (client) +## Installation -[`HTTP.request`](@ref) sends an http request and returns a response. +To install the stable release from the General registry, run: ```julia -# make a GET request, both forms are equivalent -resp = HTTP.request("GET", "http://httpbin.org/ip") -resp = HTTP.get("http://httpbin.org/ip") -println(resp.status) -println(String(resp.body)) - -# make a POST request, sending data via `body` keyword argument -resp = HTTP.post("http://httpbin.org/body"; body="request body") - -# make a POST request, sending form-urlencoded body -resp = HTTP.post("http://httpbin.org/body"; body=Dict("nm" => "val")) - -# include query parameters in a request -# and turn on verbose logging of the request/response process -resp = HTTP.get("http://httpbin.org/anything"; query=["hello" => "world"], verbose=2) - -# simple websocket client -WebSockets.open("ws://websocket.org") do ws - # we can iterate the websocket - # where each iteration yields a received message - # iteration finishes when the websocket is closed - for msg in ws - # do stuff with msg - # send back message as String, Vector{UInt8}, or iterable of either - send(ws, resp) - end -end +using Pkg +Pkg.add("HTTP") ``` -### Handling requests (server) - -[`HTTP.serve`](@ref) allows specifying middleware + handlers for how incoming requests should be processed. +For the latest development version, run: ```julia -# authentication middleware to ensure property security -function auth(handler) - return function(req) - ident = parse_auth(req) - if ident === nothing - # failed to security authentication - return HTTP.Response(401, "unauthorized") - else - # store parsed identity in request context for handler usage - req.context[:auth] = ident - # pass request on to handler function for further processing - return handler(req) - end - end -end - -# handler function to return specific user's data -function handler(req) - ident = req.context[:auth] - return HTTP.Response(200, get_user_data(ident)) -end - -# start a server listening on port 8081 (default port) for localhost (default host) -# requests will first be handled by teh auth middleware before being passed to the `handler` -# request handler function -HTTP.serve(auth(handler)) - -# websocket server is very similar to client usage -WebSockets.listen("0.0.0.0", 8080) do ws - for msg in ws - # simple echo server - send(ws, msg) - end -end -``` - -## Further Documentation - -Check out the client, server, and websocket-specific documentation pages for more in-depth discussions -and examples for the many configurations available. - -```@contents -Pages = ["client.md", "server.md", "websockets.md", "reference.md"] +using Pkg +Pkg.develop(url="https://github.com/JuliaWeb/HTTP.jl.git") ``` -## Migrating Legacy Code to 1.0 - -The 1.0 release is finally here! It's been a lot of work over the course of about 9 months combing through every part of the codebase to try and modernize APIs, fix long-standing issues, and bring the level of functionality up to par with other language http implementations. Along the way, some breaking changes were made, but with the aim that the package will now be committed to current published APIs for a long time to come. With the amount of increased functionality and fixes, we hope it provides enough incentive to make the update; as always, if you run into issues upgrading or feel something didn't get polished or fixed quite right, don't hesitate to [open an issue](https://github.com/JuliaWeb/HTTP.jl/issues/new) so we can help. +## Quick Start -The sections below outline a mix of breaking changes that were made, in addition to some of the new features in 1.0 with the aim to help those updating legacy codebases. +### Client Usage -### Struct Changes +A simple example of making a GET request: - * The `HTTP.Request` and `HTTP.Response` `body` fields are not restricted to `Vector{UInt8}`; if a `response_stream` is passed to `HTTP.request`, it will be set as the `resp.body` (previously the body was an empty `UInt8[]`). This simplified many codepaths so these "other body object types" didn't have to be held in some other state, but could be stored in the `Request`/`Response` directly. It also opens up the possibility, (as shown in the [Cors Server](@ref) example), where middleware can serialize/deserialize to/from the `body` field directly. - * In related news, a `Request` body can now be passed as a `Dict` or `NamedTuple` to have the key-value pairs serialized in the `appliction/x-www-form-urlencoded` Content-Type matching many other libraries functionality - * Responses with the `Transfer-Encoding: gzip` header will now also be automatically decompressed, and this behavior is configurable via the `decompress::Bool` keyword argument for `HTTP.request` - * If a `response_stream` is provided for streaming a request's response body, `HTTP.request` will not call `close` before returning, leaving that up to the caller. - In addition, in the face of redirects or retried requests, note the `response_stream` will not be written to until the *final* response is received. - * If a streaming request `body` is provided, it should support the `mark`/`reset` methods in case the request needs to be retried. - * Users are encouraged to access the publicly documented fields of `Request`/`Response` instead of the previously documented "accessor" functions; these fields are now committed as the public API, so feel free to do `resp.body` instead of `HTTP.body(resp)`. The accessor methods are still defined for backwards compat. - * The `Request` object now stores the original `url` argument provided to `HTTP.request` as a parsed `URIs.URI` object, and accessed via the `req.url` field. This is commonly desired in handlers/middleware, so convenient to keep it around. - * The `Request` object also has a new `req.context` field of type `Dict{Symbol, Any}` for storing/sharing state between handler/middleware layers. For example, the `HTTP.Router` now parses and stores named path parameters with the `:params` key in the context for handlers to access. Another `HTTP.cookie_middleware` will parse and store any request `Cookie` header in the `:cookies` context key. - * `HTTP.request` now throws more consistent and predictable error types, including (and restricted to): `HTTP.ConnectError`, `HTTP.StatusError`, `HTTP.TimeoutError`, and `HTTP.RequestError`. See the [Request exceptions](@ref) section for more details on each exception type. - * Cookie persistence used to use a `Dict` per thread to store domain-specific cookie sessions. A new threadsafe `CookieJar` struct now globally manages cookie persistence by default. Users can still construct and pass their own `cookiejar` keyword argument to `HTTP.request` if desired. +```julia +using HTTP -### Keyword Argument Changes +response = HTTP.get("https://api.example.com/data") +println(String(response.body)) +``` - * The `pipeline_limit` keyword argument (and support for it) were removed in `HTTP.request`; the implementation was poor and it drastically complicated the request internal implementation. In addition, it's not commonly supported in other modern http implementations, which encourage use of HTTP/2 for better designed functionality. - * `reuse_limit` support was removed in both `HTTP.request` and `HTTP.listen`; another feature that complicated code more than it was actually useful and hence removed. - * `aws_authentication` and its related keyword arguments have been removed in favor of using the AWS.jl package - * A new `redirect_method` keyword argument exists and supports finer-grained control over which method to use in the case of a request redirect +### Server Usage -### Other Largish Changes +An example of a minimal server that responds with "Hello, world!": -#### "Handlers" framework overhaul +```julia +using HTTP -The server-side Handlers framework has been changed to a more modern and flexible framework, including the [`Handler`](@ref) and [`Middleware`](@ref) interfaces. It's similar in ways to the old interfaces, but in our opinion, simpler and more straightforward with the clear distinction/pattern between what a `Handler` does vs. a `Middlware`. +function handle_request(req) + HTTP.Response(200, "Hello, world!") +end -In that vein, `HTTP.Handlers.handle` has been removed. `HTTP.serve` expects a single request or stream `Handler` function, which should be of the form `f(::Request)::Response` for the request case, or `f(::Stream)::Nothing` for streams. +server = HTTP.serve!(handle_request, "0.0.0.0", 8080) +``` -There are also plans to either include some common useful middleware functions in HTTP.jl directly, or a sister package specifically for collecting useful middlewares people can reuse. +Open [http://localhost:8080](http://localhost:8080) in your browser to see the result. Call `close(server)` to stop the server. -### WebSockets overhaul +### WebSocket Example -The WebSockets code was some of the oldest and least maintained code in HTTP.jl. It was debated removing it entirely, but there aren't really other modern implementations that are well-maintained. So the WebSockets code was overhauled, modernized, and is now tested against the industry standard autobahn test suite (yay for 3rd party verification!). The API changed as well; while `WebSockets.open` and `WebSockets.listen` have stayed the same, the `WebSocket` object itself now doesn't subtype `IO` and has a restricted interface like: - * `ws.id` access a unique generated UUID id for this websocket connection - * `receive(ws)` receive a single non-control message on a websocket, returning a `String` or `Vector{UInt8}` depending on whether the message was sent as TEXT or BINARY - * `send(ws, msg)` send a message; supports TEXT and BINARY messages, and can provide an iterable for `msg` to send fragmented messages - * `close(ws)` close a websocket connection - * For convenience, a `WebSocket` object can be iterated, where each iteration yields a non-control message and iteration terminates when the connection is closed +A simple example of connecting to a WebSocket server: -### HTTP.Router reimplementation +```julia +using HTTP -While clever, the old `HTTP.Router` implementation relied on having routes registered "statically", which can be really inconvenient for any cases where the routes are generated programmatically or need to be set/updated dynamically. +WebSockets.open("ws://echo.websocket.org") do ws + WebSockets.send(ws, "Hello WebSocket!") + message = WebSockets.receive(ws) + println("Received: ", message) +end +``` -The new `HTTP.Router` implementation uses a text-matching based trie data structure on incoming request path segments to find the right matching handler to process the request. It also supports parsing and storing path variables, like `/api/{id}` or double wildcards for matching trailing path segments, like `/api/**`. +## Additional Resources -`HTTP.Router` now also supports complete unrestricted route registration via `HTTP.register!`. +For more detailed information, please refer to the following guides: -### Internal client-side layers overhaul +- **Manual Guides:** + - [Client Guide](manual/client.md) + - [Server Guide](manual/server.md) + - [WebSockets](manual/websockets.md) -While grandiose in vision, the old type-based "layers" framework relied heavily on type parameter abuse for generating a large "stack" of layers to handle different parts of each `HTTP.request`. The new framework actually matches very closely with the server-side `Handler` and `Middleware` interfaces, and can be found in more detail under the [Client-side Middleware (Layers)](@ref) section of the docs. The new implementation, while hopefully bringing greater consistency between client-side and server-side frameworks, is much simpler and forced a large cleanup of state-handling in the `HTTP.request` process for the better. +- **API Reference:** + See the [API Reference](api/reference.md) for complete details on all public APIs. -In addition to the changing of all the client-side layer definitions, `HTTP.stack` now behaves slightly different in returning the new "layer" chain for `HTTP.request`, while also accepting custom request/stream layers is provided. A new `HTTP.@client` macro is provided for convenience in the case that users want to write a custom client-side middleware/layer and wrap its usage in an HTTP.jl-like client. +## Contributing & License -There also existed a few internal methods previously for manipulating the global stack of client-side layers (insert, insert_default!, etc.). These have been removed and replaced with a more formal (and documented) API via `HTTP.pushlayer!` and `HTTP.poplayer!`. These can be used to globally manipulate the client-side stack of layers for any `HTTP.request` that is made. +Contributions are welcome! Please review the [Contributing Guidelines](CONTRIBUTING.md) for further details. HTTP.jl is distributed under the MIT License—see the [LICENSE](LICENSE) file for more information. \ No newline at end of file diff --git a/docs/src/manual/client.md b/docs/src/manual/client.md new file mode 100644 index 000000000..a29b0404f --- /dev/null +++ b/docs/src/manual/client.md @@ -0,0 +1,182 @@ +# HTTP Client Guide + +This guide provides an in-depth look at HTTP.jl’s client functionality. It covers the primary function for making requests, the available helper functions, how to configure and use a preconfigured client for advanced use cases, and a brief overview of what happens behind the scenes. + +## Overview + +HTTP.jl enables you to issue HTTP requests using the primary function `HTTP.request`. In addition, there are convenience functions like `HTTP.get`, `HTTP.post`, `HTTP.put`, `HTTP.delete` etc., which are simply aliases that pass the appropriate HTTP method automatically. You can also pass a wide variety of keyword arguments to control the behavior of both the request and the underlying client. + +## The HTTP.request Function + +The general form of the request function is: + +\`\`\`julia +HTTP.request(method, url[, headers, body]; keyword_arguments...) +\`\`\` + +Here, **method** is a string such as `"GET"`, `"POST"`, or `"PUT"`, and **url** can be either a string or a URI. +**headers** is an optional array of key-value pairs, and **body**, also optional, is the request payload. The function returns a `Response` object. + +### Helper Functions + +- **HTTP.get, HTTP.post, HTTP.put, HTTP.delete, etc.:** + These functions are shortcuts for calling `HTTP.request` with the corresponding HTTP method. For example, calling `HTTP.get(url; kwargs...)` is equivalent to calling `HTTP.request("GET", url; kwargs...)`. + +### Available Keyword Arguments + +The following keyword arguments (which correspond to the non-`scheme`/`host`/`port` fields of `ClientSettings`) allow you to finely control request and client behavior: + +- **query**: Any iterable that yields length-2 iterables can be provided as query params that will be joined with the `url` argument. e.g. a `Dict`, `Vector{Pair}`, etc. +- **headers**: Any iterable that yields length-2 iterables can be provided as headers for the HTTP request. Some default headers are set if not provided. +- **client**: A custom-configured `HTTP.Client` can be provided to precisely control settings for classes of requests. +- **verbose**: Controls the verbosity of logging during the request. +-- Request body options: + - **body**: Can be one of: `AbstractString`, `AbstractVector{UInt8}`, `IO`, `AbstractDict`, or `NamedTuple`. Strings and byte vectors will be sent "as-is. `IO` request bodies will be read completely and then sent. Dicts and namedtuples will be encoded in the `x-www-form-urlencoded` content-type format before being sent. + - **chunkedbody**: To send a request body in chunks, an iterable must be provided where each element is one of the valid types of request bodies mentioned above. + - **modifier**: A function of the form `f(request, body) -> newbody`, i.e. that takes the HTTP request object and proposed request body, and can optionally return a new request body. If the modifer only modifies the request object, it should return `nothing`, which will ensure the original request body is sent unmodified. +-- Response options: + - **response_body**: By default, response bodies are returned as `Vector{UInt8}`. Alternatively, a preallocated `AbstractVector{UInt8}` or any `IO` object can be provided for the response body to be written into. + - **decompress**: If `true`, the response body will be decompressed if it is compressed. By default, response bodies with the `Content-Encoding: gzip` header are decompressed. + - **status_exception**: Default `true`. If `true`, an exception will be thrown if the response status code is not in the 200-299 range. + - **readtimeout**: The maximum time in seconds to wait for a response from the server. Only valid for HTTP/1.1 connections. +-- Redirect options: + - **redirect**: Default `true`. If `true`, the client will follow redirects. + - **redirect_limit**: The maximum number of redirects to follow. Default is 3. + - **redirect_method**: The method to use for redirected requests. Pass `:same` to use the same method as the original request, or the specific method to use (like `:POST`). + - **forwardheaders**: Default `true`. If `true`, non-sensitive headers from the original request will be forwarded to the redirected request. +-- Authentication options: + - **username**: The username for basic authentication. + - **password**: The password for basic authentication. + - **bearer**: The bearer token for authentication. +-- Cookie options: + - **cookies**: Default `true`. If `true`, cookies will be stored and sent with subsequent requests that match appropriate domains. If `false`, no cookies will be stored or sent. Can also pass a `Dict` of cookies to send with the request. + - **cookiejar**: The `HTTP.CookieJar` object to use for storing and matching cookies for requests. +-- Client options: + -- Socket options: + - **connect_timeout_ms**: The maximum time in milliseconds to wait for a connection to be established. Default is 3000. + - **socket_domain**: The socket domain to use for the connection. Default is `:ipv4`. Can also be `:ipv6`. + - **keep_alive_interval_sec**: The time in seconds to wait before sending a keep-alive probe. Default is 0. + - **keep_alive_timeout_sec**: The time in seconds to wait for a keep-alive probe response. Default is 0. + - **keep_alive_max_failed_probes**: The maximum number of failed keep-alive probes before the connection is closed. Default is 0. + - **keepalive**: Default `false`. If `true`, the connection will utilize keep-alive probes to maintain the connection. + -- SSL options: + - **ssl_insecure**: Default `false`. If `true`, SSL certificate verification will be disabled. + - **ssl_cert**: The path to the SSL certificate file. + - **ssl_key**: The path to the SSL key file. + - **ssl_capath**: The path to the directory containing CA certificates. + - **ssl_cacert**: The path to the CA certificate file. + - **ssl_alpn_list**: A list of ALPN protocols to use for the connection. Default is `"h2;http/1.1"`. + -- Proxy options: + - **proxy_allow_env_var**: Default `true`. If `true`, the `HTTP_PROXY` environment variable will be used to set the proxy settings. + - **proxy_connection_type**: The type of connection to use for the proxy. Default is `:http`. Can also be `:https`. + - **proxy_host**: The host of the proxy server. + - **proxy_port**: The port of the proxy server. + - **proxy_ssl_cert**: The path to the SSL certificate file for the proxy. + - **proxy_ssl_key**: The path to the SSL key file for the proxy. + - **proxy_ssl_capath**: The path to the directory containing CA certificates for the proxy. + - **proxy_ssl_cacert**: The path to the CA certificate file for the proxy. + - **proxy_ssl_insecure**: Default `false`. If `true`, SSL certificate verification will be disabled for the proxy. + - **proxy_ssl_alpn_list**: A list of ALPN protocols to use for the proxy connection. Default is `"h2;http/1.1"`. + -- Retry options: + - **max_retries**: The maximum number of times to retry a request. Default is 10. + - **retry_partition**: Requests utilizing the same retry partition (an arbitrary string) will coordinate retries against each other to not overwhelm a temporarily unresponsive server. + - **backoff_scale_factor_ms**: The factor by which to scale the backoff time between retries. Default is 25. + - **max_backoff_secs**: The maximum time in seconds to wait between retries. Default is 20. + - **retry_timeout_ms**: The maximum time in milliseconds to wait for a retry to complete. Default is 60000. + - **initial_bucket_capacity**: The initial capacity of the retry bucket. Default is 500. + - **retry_non_idempotent**: Default `false`. If `true`, non-idempotent requests will be retried. + -- Connection pool options: + - **max_connections**: The maximum number of connections to keep open in the connection pool. Default is 512. + - **max_connection_idle_in_milliseconds**: The maximum time in milliseconds to keep a connection open in the pool. Default is 60000. + -- AWS runtime options: + - **allocator**: The allocator to use for AWS-allocated memory during the request. + - **bootstrap**: The AWS client bootstrap to use for the request. + - **event_loop_group**: The AWS event loop group to use for the request. + +Any combination of these keyword arguments can be passed to `HTTP.request` (or its helper functions) to customize your HTTP requests. + +## Examples + +### Basic GET Request + +\`\`\`julia +using HTTP + +response = HTTP.request("GET", "https://httpbin.org/get") +println("Status: ", response.status) +println("Body: ", String(response.body)) +\`\`\` + +### POST Request with JSON Payload + +\`\`\`julia +using HTTP, JSON + +payload = JSON.json(Dict("name" => "HTTP.jl", "version" => "2.0")) +headers = ["Content-Type" => "application/json"] + +response = HTTP.request("POST", "https://httpbin.org/post"; headers = headers, body = payload) +println("Response: ", String(response.body)) +\`\`\` + +### GET Request with Query Parameters and Custom Headers + +\`\`\`julia +using HTTP + +response = HTTP.request( + "GET", + "https://httpbin.org/get"; + query = Dict("search" => "Julia HTTP client"), + headers = ["Accept" => "application/json"] +) +println("Response JSON: ", String(response.body)) +\`\`\` + +## Preconfigured Clients (Advanced) + +For advanced use cases, you can create a custom client with specific parameters for a particular `scheme`, `host`, and `port`. This custom client is preconfigured with your desired connection settings (such as timeouts, retries, and SSL options) and can be passed to any HTTP request function using the `client` keyword. By default, HTTP.jl will create or reuse an existing client that matches the exact combination of `scheme`, `host`, `port`, and all other provided keyword arguments. + +For example: + +\`\`\`julia +using HTTP + +# Instantiate a client with custom settings +custom_client = HTTP.Client( + "https", + "api.example.com", + 443; + connect_timeout_ms = 5000, + max_retries = 3, + ssl_insecure = false + # Additional settings can be specified here... +) + +# Use the custom client in a request; subsequent requests with the same parameters will reuse this client +response = HTTP.get("https://api.example.com/data"; client = custom_client) +println(String(response.body)) +\`\`\` + +## Under the Hood (Advanced) + +When you call `HTTP.request`, the following advanced steps occur: + +1. **URI Parsing:** + The URL is parsed and combined with any query parameters provided. + +2. **Client Selection and Initialization:** + A client is either chosen from a pool or created based on the provided (or default) settings. Clients are matched using the `scheme`, `host`, `port`, and all other keyword arguments. + +3. **Connection Management:** + The selected client configures and acquires a connection, using client socket options, TLS configuration, and proxy settings as necessary. + +4. **Request Assembly:** + A complete request message is constructed with the specified method, path, headers, and body. For chunked transfers, HTTP.jl manages the sequential writing of each chunk. + +5. **Retry and Redirect Handling:** + Built-in logic handles following redirects and retrying requests based on transient errors and the provided configuration. + +6. **Response Processing:** + The response is parsed, and if errors occur (as dictated by your settings), an exception is raised. + diff --git a/docs/src/manual/server.md b/docs/src/manual/server.md new file mode 100644 index 000000000..9dfc75b1e --- /dev/null +++ b/docs/src/manual/server.md @@ -0,0 +1,211 @@ +# HTTP Server Guide + +This document provides a comprehensive overview of the server functionality in HTTP.jl. It covers creating and managing an `HTTP.Server`, using the `HTTP.serve!` function with its many keyword arguments, understanding how handler functions work (including middleware), and a detailed explanation of the `HTTP.Router` for routing requests. + +## Overview + +HTTP.jl's server interface lets you build and run HTTP servers with ease. You can manage the server lifecycle with functions such as `isopen`, `close`, and `wait`. In addition, you can define request handlers, wrap them with middleware to extend or transform behavior, and use the `HTTP.Router` to organize and route requests based on method and URL. + +## Server Lifecycle Management + +An `HTTP.Server` object encapsulates a running server instance. Key lifecycle functions include: + +- **`isopen(server)`** + Checks whether the server is currently running. + +- **`close(server)`** + Closes the server and releases any associated resources. + +- **`wait(server)`** + Will block until the server is closed. + +### Example + +```julia +using HTTP + +# Define a simple request handler +function handle_request(req) + return HTTP.Response(200, "Hello from HTTP.jl server!") +end + +# Start the server +server = HTTP.serve!(handle_request, "127.0.0.1", 8080) + +# Check if the server is running +if HTTP.isopen(server) + println("Server is running on http://127.0.0.1:8080") +end + +# Later, close the server +HTTP.close(server) +``` + +## Using HTTP.serve! + +The `HTTP.serve!` function is the primary entry point for starting an HTTP server. It accepts a handler function along with many keyword arguments that configure the server. + +### Syntax + +```julia +HTTP.serve!(handler_function, host::AbstractString="127.0.0.1", port::Integer=8080; + allocator = default_aws_allocator(), + bootstrap::Ptr{aws_server_bootstrap} = default_aws_server_bootstrap(), + endpoint = nothing, + listenany::Bool = false, + access_log::Union{Nothing, Function} = nothing, + # Socket options + socket_options = nothing, + socket_domain = :ipv4, + connect_timeout_ms::Integer = 3000, + keep_alive_interval_sec::Integer = 0, + keep_alive_timeout_sec::Integer = 0, + keep_alive_max_failed_probes::Integer = 0, + keepalive::Bool = false, + # TLS options + tls_options = nothing, + ssl_cert = nothing, + ssl_key = nothing, + ssl_capath = nothing, + ssl_cacert = nothing, + ssl_insecure::Bool = false, + ssl_alpn_list = "h2;http/1.1", + initial_window_size = typemax(UInt64) +) +``` + +### Keyword Arguments Description + +- **`listenany`**: Boolean indicating whether to listen on any available port. If `true`, the `port` argument will be used as a starting port, and the server will attempt to bind to the next available port if the specified port is already in use. +- **`access_log`**: A function to log access details, like as returned by the `logfmt"..."` string macro. The function takes an `HTTP.Stream` as input and returns the string to log. If `nothing`, no access logging is performed (default). +-- Socket options: + - **`socket_options`**: Custom AWS socket options struct. + - **`socket_domain`**: Domain type (e.g. `:ipv4` or `:ipv6`). + - **`connect_timeout_ms`**: Connection timeout in milliseconds. Default is 3000 ms. + - **`keep_alive_interval_sec`**: Interval for sending keep-alive messages in seconds. + - **`keep_alive_timeout_sec`**: Timeout for keep-alive responses in seconds. + - **`keep_alive_max_failed_probes`**: Maximum number of failed keep-alive probes allowed. + - **`keepalive`**: Boolean to enable socket keepalive. +-- TLS options: + - **`tls_options`**: Custom AWS TLS configuration options. + - **`ssl_cert`**: Path to the SSL certificate file. + - **`ssl_key`**: Path to the SSL key file. + - **`ssl_capath`**: Path to the directory containing CA certificates. + - **`ssl_cacert`**: Path to the CA certificate file. + - **`ssl_insecure`**: Boolean flag to disable SSL certificate verification. + - **`ssl_alpn_list`**: List of ALPN protocols (e.g., `"h2;http/1.1"`). +-- AWS runtime options - + - **`allocator`**: The AWS allocator to use for AWS-allocated memory while handling connections and requests. + - **`bootstrap`**: Pointer to an AWS server bootstrap. + - **`endpoint`**: Custom AWS endpoint for binding the server. + +### Example + +```julia +using HTTP + +function handle_request(req) + return HTTP.Response(200, "Hello, customized server!") +end + +server = HTTP.serve!(handle_request, "127.0.0.1", 8080; + connect_timeout_ms = 5000, + keep_alive_interval_sec = 30, + ssl_insecure = false +) + +println("Server started on http://127.0.0.1:8080") +``` + +## Handlers and Middleware + +### Handler Functions + +A **Handler** is a function that accepts an `HTTP.Request` (or a stream in advanced cases) and returns an `HTTP.Response`. This is the fundamental building block of how the server processes requests. + +Example of a simple handler: + +```julia +function simple_handler(req) + return HTTP.Response(200, "Simple response") +end +``` + +More involved handler: +```julia +using HTTP, JSON + +function advanced_handler(req) + # Prepare custom headers + headers = [ + "Content-Type" => "application/json", + "X-Custom-Header" => "AdvancedExample" + ] + + # Set a non-200 status code (for example, 404 Not Found) + status_code = 404 + + # Build a JSON response body with error details + body = JSON.json(Dict( + "error" => true, + "message" => "Resource not found", + "requested_path" => req.path + )) + + return HTTP.Response(status_code, headers, body) +end +``` + +### Middleware + +**Middleware** are functions that wrap a handler to add pre‑ or post‑processing steps. They are useful for logging, authentication, modifying requests/responses, and more. You can think of a "middleware" as a function that takes a +handler function as input and returns a new function that acts as a handler. The middleware just should most likely +ensure that it calls the original handler at some point. + +Example of a logging middleware: + +```julia +function logging_middleware(handler) + return function(req) + @info "Request received for " * req.path + return handler(req) + end +end + +# Wrap the simple handler with the middleware and pass the resulting modified handler to serve! +HTTP.serve!(logging_middleware(simple_handler), "127.0.0.1", 8080) +``` + +### HTTP.Router + +The **HTTP.Router** provides a structured way to route incoming requests based on their URL path and HTTP method. It supports: +- Registering routes using `HTTP.register!`. +- Extracting parameters from URL patterns. +- Retrieving routing metadata such as the original route string and path parameters. + +#### Creating and Using a Router + +```julia +using HTTP + +router = HTTP.Router() + +# Register a route for GET requests to "/api/data" +HTTP.register!(router, "GET", "/api/data") do req + return HTTP.Response(200, "Data for GET requests") +end + +# Register a route with a path parameter, e.g., "/api/item/{id}" +HTTP.register!(router, "GET", "/api/item/{id}") do req + id = HTTP.getparam(req, "id") + return HTTP.Response(200, "Item ID: " * id) +end + +# Use the router as your server handler +server = HTTP.serve!(router, "127.0.0.1", 8080) +``` + +The router also offers helper functions: +- **`HTTP.getroute(req)`**: Returns the original registered route. +- **`HTTP.getparams(req)`**: Returns a dictionary of URL parameters. +- **`HTTP.getparam(req, name, default)`**: Returns the value of a specific parameter, with an optional default. diff --git a/docs/src/manual/websockets.md b/docs/src/manual/websockets.md new file mode 100644 index 000000000..259650e0c --- /dev/null +++ b/docs/src/manual/websockets.md @@ -0,0 +1,106 @@ +# HTTP WebSockets Guide + +This guide provides a comprehensive overview of the WebSocket features in HTTP.jl. It explains how to establish a WebSocket connection, send and receive both text and binary messages, handle control frames (such as ping, pong, and close), and manage the lifecycle of WebSocket connections. + +## Overview + +HTTP.jl’s WebSockets API offers a convenient way to create real‑time communication channels over HTTP. The key functions include: +- **`WebSockets.open`** – Opens a WebSocket connection. +- **`WebSockets.send`** – Sends a message (text or binary) over an open connection. +- **`WebSockets.receive`** – Receives messages from the WebSocket. +- **`WebSockets.ping`** and **`WebSockets.pong`** – Send control frames to check connection health. +- **Iteration** – A WebSocket can be iterated over to continuously process incoming messages. + +## Connecting to a WebSocket Server + +To open a WebSocket connection, use the `WebSockets.open` function with a URL (starting with either `"ws://"` or `"wss://"`). A user-defined function is provided to interact with the open connection. + +Example: + +```julia +using HTTP + +WebSockets.open("wss://echo.websocket.org") do ws + println("Connected to the WebSocket server!") + # Further operations with the ws object... +end +``` + +## Sending and Receiving Messages + +Once connected, you can send messages using `WebSockets.send` and retrieve them using `WebSockets.receive`. The API handles both text and binary messages seamlessly. + +Example of sending a text message and receiving an echo: + +```julia +using HTTP + +WebSockets.open("wss://echo.websocket.org") do ws + WebSockets.send(ws, "Hello, WebSocket!") + message = WebSockets.receive(ws) + println("Received message: ", message) +end +``` + +For binary messages, pass a `Vector{UInt8}` to `send`; `receive` will return binary data accordingly. + +## Handling Control Frames + +WebSocket control frames are used for protocol-level tasks such as keeping the connection alive or closing it gracefully. HTTP.jl provides: +- **`WebSockets.ping(ws, [data])`** – Sends a PING frame. +- **`WebSockets.pong(ws, [data])`** – Sends a PONG frame. + +Example of sending a PING frame: + +```julia +using HTTP, WebSockets + +WebSockets.open("wss://echo.websocket.org") do ws + WebSockets.ping(ws) + println("Ping sent!") + # Optionally, you can handle pong responses as needed. +end +``` + +## Iterating Over WebSocket Messages + +A WebSocket connection can be used as an iterator, which continuously yields incoming messages until the connection is closed. + +Example: + +```julia +using HTTP, WebSockets + +WebSockets.open("wss://echo.websocket.org") do ws + for message in ws + println("Received: ", message) + # Exit the loop if a specific message is received + if message == "exit" + break + end + end +end +``` + +## Connection Lifecycle and Error Handling + +You can check whether a WebSocket is open using `WebSockets.isclosed(ws)` and close it with `close(ws)`. The API is designed to raise exceptions for connection issues or protocol errors, allowing you to handle errors using try‑catch blocks. + +Example: + +```julia +using HTTP, WebSockets + +WebSockets.open("wss://echo.websocket.org") do ws + try + WebSockets.send(ws, "Test message") + msg = WebSockets.receive(ws) + println("Received: ", msg) + catch e + @error "WebSocket error:" e + finally + close(ws) + println("WebSocket closed.") + end +end +``` diff --git a/docs/src/reference.md b/docs/src/reference.md deleted file mode 100644 index 9a9630bc4..000000000 --- a/docs/src/reference.md +++ /dev/null @@ -1,196 +0,0 @@ -# API Reference - -```@contents -Pages = ["reference.md"] -Depth = 3 -``` - -## Client Requests - -```@docs -HTTP.request -HTTP.get -HTTP.put -HTTP.post -HTTP.head -HTTP.patch -HTTP.delete -HTTP.open -HTTP.download -``` - -### Request/Response Objects - -```@docs -HTTP.Request -HTTP.Response -HTTP.Stream -HTTP.WebSockets.WebSocket -HTTP.Messages.header -HTTP.Messages.headers -HTTP.Messages.hasheader -HTTP.Messages.headercontains -HTTP.Messages.setheader -HTTP.Messages.appendheader -HTTP.Messages.removeheader -HTTP.Messages.decode -``` - -### Request body types - -```@docs -HTTP.Form -HTTP.Multipart -``` - -### Request exceptions - -Request functions may throw the following exceptions: - -```@docs -HTTP.ConnectError -HTTP.TimeoutError -HTTP.StatusError -HTTP.RequestError -``` - -### URIs - -HTTP.jl uses the [URIs.jl](https://github.com/JuliaWeb/URIs.jl) package for handling -URIs. Some functionality from URIs.jl, relevant to HTTP.jl, are listed below: - -```@docs -URI -URIs.escapeuri -URIs.unescapeuri -URIs.splitpath -Base.isvalid(::URIs.URI) -``` - -### Cookies - -```@docs -HTTP.Cookie -HTTP.Cookies.stringify -HTTP.Cookies.addcookie! -HTTP.Cookies.cookies -``` - -### WebSockets - -```@docs -HTTP.WebSockets.send(::HTTP.WebSockets.WebSocket, msg) -HTTP.WebSockets.receive -HTTP.WebSockets.close(::HTTP.WebSockets.WebSocket, body) -HTTP.WebSockets.ping -HTTP.WebSockets.pong -HTTP.WebSockets.iterate(::HTTP.WebSockets.WebSocket, st) -HTTP.WebSockets.isclosed -HTTP.WebSockets.isok -``` - -## Utilities - -```@docs -HTTP.sniff -HTTP.Strings.escapehtml -HTTP.Strings.tocameldash -HTTP.Strings.iso8859_1_to_utf8 -HTTP.Strings.ascii_lc_isequal -HTTP.statustext -``` - -## Server / Handlers - -### Core Server - -```@docs -HTTP.listen -HTTP.serve -WebSockets.listen -``` - -### Middleware / Handlers - -```@docs -HTTP.Handler -HTTP.Middleware -HTTP.streamhandler -HTTP.Router -HTTP.register! -HTTP.getparam -HTTP.getparams -HTTP.Handlers.cookie_middleware -HTTP.getcookies -HTTP.@logfmt_str -``` - -## Advanced Topics - -### Messages Interface - -```@docs -HTTP.Messages.iserror -HTTP.Messages.isredirect -HTTP.Messages.ischunked -HTTP.Messages.issafe -HTTP.Messages.isidempotent -HTTP.Messages.retryable -HTTP.Messages.defaultheader! -HTTP.Messages.readheaders -HTTP.HeadersRequest.setuseragent! -HTTP.Messages.readchunksize -HTTP.Messages.headerscomplete(::HTTP.Messages.Response) -HTTP.Messages.writestartline -HTTP.Messages.writeheaders -Base.write(::IO,::HTTP.Messages.Message) -HTTP.Streams.closebody -HTTP.Streams.isaborted -``` - -### Cookie Persistence - -```@docs -HTTP.Cookies.CookieJar -HTTP.Cookies.getcookies! -HTTP.Cookies.setcookies! -``` - -### Client-side Middleware (Layers) - -```@docs -HTTP.Layer -HTTP.@client -HTTP.pushlayer! -HTTP.pushfirstlayer! -HTTP.poplayer! -HTTP.popfirstlayer! -HTTP.MessageRequest.messagelayer -HTTP.RedirectRequest.redirectlayer -HTTP.HeadersRequest.headerslayer -HTTP.CookieRequest.cookielayer -HTTP.TimeoutRequest.timeoutlayer -HTTP.ExceptionRequest.exceptionlayer -HTTP.RetryRequest.retrylayer -HTTP.ConnectionRequest.connectionlayer -HTTP.StreamRequest.streamlayer -``` - -### Raw Request Connection - -```@docs -HTTP.openraw -HTTP.Connection -``` - -### Parser Interface - -```@docs -HTTP.Parsers.find_end_of_header -HTTP.Parsers.find_end_of_chunk_size -HTTP.Parsers.find_end_of_trailer -HTTP.Parsers.parse_status_line! -HTTP.Parsers.parse_request_line! -HTTP.Parsers.parse_header_field -HTTP.Parsers.parse_chunk_size -``` \ No newline at end of file diff --git a/docs/src/server.md b/docs/src/server.md deleted file mode 100644 index 17e62a76b..000000000 --- a/docs/src/server.md +++ /dev/null @@ -1,115 +0,0 @@ -# Server - -For server-side functionality, HTTP.jl provides a robust framework for core -HTTP and websocket serving, flexible handler and middleware interfaces, and -low-level access for unique workflows. The core server listening code is in -the [`/src/Servers.jl`](https://github.com/JuliaWeb/HTTP.jl/blob/master/src/Servers.jl) file, while the handler, middleware, router, and -higher level `HTTP.serve` function are defined in the [`/src/Handlers.jl`](https://github.com/JuliaWeb/HTTP.jl/blob/master/src/Handlers.jl) -file. - -## `HTTP.serve` - -`HTTP.serve`/`HTTP.serve!` are the primary entrypoints for HTTP server functionality, while `HTTP.listen`/`HTTP.listen!` are considered the lower-level core server loop methods that only operate directly with `HTTP.Stream`s. `HTTP.serve` is also built directly integrated with the `Handler` and `Middleware` interfaces and provides easy flexibility by doing so. The signature is: - -```julia -HTTP.serve(f, host, port; kw...) -HTTP.serve!(f, host, port; kw...) -> HTTP.Server -``` - -Where `f` is a `Handler` function, typically of the form `f(::Request) -> Response`, but can also operate directly on an `HTTP.Stream` of the form `f(::Stream) -> Nothing` while also passing `stream=true` to the keyword arguments. The `host` argument should be a `String`, or `IPAddr`, created like `ip"0.0.0.0"`. `port` should be a valid port number as an `Integer`. -`HTTP.serve` is the blocking server method, whereas `HTTP.serve!` is non-blocking and returns -the listening `HTTP.Server` object that can be `close(server)`ed manually. - -Supported keyword arguments include: - * `sslconfig=nothing`, Provide an `MbedTLS.SSLConfig` object to handle ssl - connections. Pass `sslconfig=MbedTLS.SSLConfig(false)` to disable ssl - verification (useful for testing). Construct a custom `SSLConfig` object - with `MbedTLS.SSLConfig(certfile, keyfile)`. - * `tcpisvalid = tcp->true`, function `f(::TCPSocket)::Bool` to check if accepted - connections are valid before processing requests. e.g. to do source IP filtering. - * `readtimeout::Int=0`, close the connection if no data is received for this - many seconds. Use readtimeout = 0 to disable. - * `reuseaddr::Bool=false`, allow multiple servers to listen on the same port. - Not supported on some OS platforms. Can check `HTTP.Servers.supportsreuseaddr()`. - * `server::Base.IOServer=nothing`, provide an `IOServer` object to listen on; - allows manually closing or configuring the server socket. - * `verbose::Union{Int,Bool}=false`, log connection information to `stdout`. Use `-1` - to also silence the server start and stop logs. - * `access_log::Function`, function for formatting access log messages. The - function should accept two arguments, `io::IO` to which the messages should - be written, and `http::HTTP.Stream` which can be used to query information - from. See also [`@logfmt_str`](@ref). - * `on_shutdown::Union{Function, Vector{<:Function}, Nothing}=nothing`, one or - more functions to be run if the server is closed (for example by an - `InterruptException`). Note, shutdown function(s) will not run if an - `IOServer` object is supplied to the `server` keyword argument and closed - by `close(server)`. - -## `HTTP.Handler` - -Abstract type for the handler interface that exists for documentation purposes. -A `Handler` is any function of the form `f(req::HTTP.Request) -> HTTP.Response`. -There is no requirement to subtype `Handler` and users should not rely on or dispatch -on `Handler`. A `Handler` function `f` can be passed to [`HTTP.serve`](@ref) -wherein a server will pass each incoming request to `f` to be handled and a response -to be returned. Handler functions are also the inputs to [`Middleware`](@ref) functions -which are functions of the form `f(::Handler) -> Handler`, i.e. they take a `Handler` -function as input, and return a "modified" or enhanced `Handler` function. - -For advanced cases, a `Handler` function can also be of the form `f(stream::HTTP.Stream) -> Nothing`. -In this case, the server would be run like `HTTP.serve(f, ...; stream=true)`. For this use-case, -the handler function reads the request and writes the response to the stream directly. Note that -any middleware used with a stream handler also needs to be of the form `f(stream_handler) -> stream_handler`, -i.e. it needs to accept a stream `Handler` function and return a stream `Handler` function. - -## `HTTP.Middleware` - -Abstract type for the middleware interface that exists for documentation purposes. -A `Middleware` is any function of the form `f(::Handler) -> Handler` (ref: [`Handler`](@ref)). -There is no requirement to subtype `Middleware` and users should not rely on or dispatch -on the `Middleware` type. While `HTTP.serve(f, ...)` requires a _handler_ function `f` to be -passed, middleware can be "stacked" to create a chain of functions that are called in sequence, -like `HTTP.serve(base_handler |> cookie_middleware |> auth_middlware, ...)`, where the -`base_handler` `Handler` function is passed to `cookie_middleware`, which takes the handler -and returns a "modified" handler (that parses and stores cookies). This "modified" handler is -then an input to the `auth_middlware`, which further enhances/modifies the handler. - -## `HTTP.Router` - -Object part of the `Handler` framework for routing requests based on path matching registered routes. - -```julia -r = HTTP.Router(_404, _405) -``` - -Define a router object that maps incoming requests by path to registered routes and -associated handlers. Paths can be registered using [`HTTP.register!`](@ref). The router -object itself is a "request handler" that can be called like: -``` -r = HTTP.Router() -resp = r(request) -``` - -Which will inspect the `request`, find the matching, registered handler from the url, -and pass the request on to be handled further. - -See [`HTTP.register!`](@ref) for additional information on registering handlers based on routes. - -If a request doesn't have a matching, registered handler, the `_404` handler is called which, -by default, returns a `HTTP.Response(404)`. If a route matches the path, but not the method/verb -(e.g. there's a registered route for "GET /api", but the request is "POST /api"), then the `_405` -handler is called, which by default returns `HTTP.Response(405)` (method not allowed). - -## `HTTP.listen` - -Lower-level core server functionality that only operates on `HTTP.Stream`. Provides a level of separation from `HTTP.serve` and the `Handler` framework. Supports all the same arguments and keyword arguments as [`HTTP.serve`](@ref), but the handler function `f` _must_ take a single `HTTP.Stream` as argument. `HTTP.listen!` is the non-blocking counterpart to `HTTP.listen` (like `HTTP.serve!` is to `HTTP.serve`). - -## Log formatting - -Nginx-style log formatting is supported via the [`HTTP.@logfmt_str`](@ref) macro and can be passed via the `access_log` keyword argument for [`HTTP.listen`](@ref) or [`HTTP.serve`](@ref). - -## Serving on the interactive thead pool - -Beginning in Julia 1.9, the main server loop is spawned on the [interactive threadpool](https://docs.julialang.org/en/v1.9/manual/multi-threading/#man-threadpools) by default. If users do a Threads.@spawn from a handler, those threaded tasks should run elsewhere and not in the interactive threadpool, keeping the web server responsive. - -Note that just having a reserved interactive thread doesn’t guarantee CPU cycles, so users need to properly configure their running Julia session appropriately (i.e. ensuring non-interactive threads available to run tasks, etc). \ No newline at end of file diff --git a/docs/src/websockets.md b/docs/src/websockets.md deleted file mode 100644 index 3ad5a9ada..000000000 --- a/docs/src/websockets.md +++ /dev/null @@ -1,68 +0,0 @@ -# WebSockets - -HTTP.jl provides a complete and well-tested websockets implementation via the `WebSockets` module. It is tested against the industry standard [autobahn testsuite](https://github.com/crossbario/autobahn-testsuite). The client and server usage are similar to their HTTP counterparts, but provide a `WebSocket` object that enables sending and receiving messages. - -## WebSocket object - -Both `WebSockets.open` (client) and `WebSockets.listen` (server) take a handler function that should accept a single `WebSocket` object argument. It has a small, simple API to provide full websocket functionality, including: - * `receive(ws)`: blocking call to receive a single, non-control message from the remote. If ping/pong messages are received, they are handled/responded to automatically and a non-control message is waited for. If a CLOSE message is received, an error will be thrown. Returns either a `String` or `Vector{UInt8}` depending on whether the message had a TEXT or BINARY frame type. Fragmented messages are received fully before returning and the bodies of each frame are concatenated for a full, final message body. - * `send(ws, msg)`: sends a message to the remote. If `msg` is `AbstractVector{UInt8}`, the message will have the BINARY type, if `AbstractString`, it will have TEXT type. `msg` can also be an iterable of either `AbstractVector{UInt8}` or `AbstractString` and a fragmented message will be sent, with one fragment for each iterated element. - * `close(ws)`: initiate the close sequence of the websocket - * `ping(ws[, data])`: send a PING message to the remote with optional `data`. PONG responses are received by calling `receive(ws)`. - * `pong(ws[, data])`: send a PONG message to the remote with optional `data`. - * `for msg in ws`: for convenience, the `WebSocket` object supports the iteration protocol, which results in a call to `receive(ws)` to produce each iterated element. Iteration terminates when a non-error CLOSE message is received. This is the most common way to handle the life of a `WebSocket`, and looks like: - -```julia -# client -WebSockets.open(url) do ws - for msg in ws - # do cool stuff with msg - end -end - -# server -WebSockets.listen(host, port) do ws - # iterate incoming websocket messages - for msg in ws - # send message back to client or do other logic here - send(ws, msg) - end - # iteration ends when the websocket connection is closed by client or error -end -``` - -## WebSocket Client - -To initiate a websocket client connection, the [`WebSockets.open`](@ref) function is provided, which operates similar to [`HTTP.open`](@ref), but the handler function should operate on a single `WebSocket` argument instead of an `HTTP.Stream`. - -### Example - -```julia -# simple websocket client -WebSockets.open("ws://websocket.org") do ws - # we can iterate the websocket - # where each iteration yields a received message - # iteration finishes when the websocket is closed - for msg in ws - # do stuff with msg - # send back message as String, Vector{UInt8}, or iterable of either - send(ws, resp) - end -end -``` - -## WebSocket Server - -To start a websocket server to listen for client connections, the [`WebSockets.listen`](@ref) function is provided, which mirrors the [`HTTP.listen`](@ref) function, but the provided handler should operate on a single `WebSocket` argument instead of an `HTTP.Stream`. - -### Example - -```julia -# websocket server is very similar to client usage -WebSockets.listen("0.0.0.0", 8080) do ws - for msg in ws - # simple echo server - send(ws, msg) - end -end -``` \ No newline at end of file diff --git a/src/Conditions.jl b/src/Conditions.jl deleted file mode 100644 index c3134bddf..000000000 --- a/src/Conditions.jl +++ /dev/null @@ -1,76 +0,0 @@ - -module Conditions - -export @require, @ensure, precondition_error, postcondition_error - -import ..DEBUG_LEVEL - -# Get the calling function. See https://github.com/JuliaLang/julia/issues/6733 -# (The macro form @__FUNCTION__ is hard to escape correctly, so just us a function.) -function _funcname_expr() - return :($(esc(Expr(:isdefined, Symbol("#self#")))) ? nameof($(esc(Symbol("#self#")))) : nothing) -end - -@noinline function precondition_error(msg, calling_funcname) - calling_funcname = calling_funcname === nothing ? "unknown" : calling_funcname - return ArgumentError("$calling_funcname() requires $msg") -end - - -""" - @require precondition [message] -Throw `ArgumentError` if `precondition` is false. -""" -macro require(condition, msg = "`$condition`") - :(if ! $(esc(condition)) throw(precondition_error($(esc(msg)), $(_funcname_expr()))) end) -end - -@noinline function postcondition_error(msg, calling_funcname, ls="", l="", rs="", r="") - calling_funcname = calling_funcname === nothing ? "unknown" : calling_funcname - msg = "$calling_funcname() failed to ensure $msg" - if ls != "" - msg = string(msg, "\n", ls, " = ", sprint(show, l), - "\n", rs, " = ", sprint(show, r)) - end - return AssertionError(msg) -end - - -# Copied from stdlib/Test/src/Test.jl:get_test_result() -iscondition(ex) = isa(ex, Expr) && - ex.head == :call && - length(ex.args) == 3 && - first(string(ex.args[1])) != '.' && - (!isa(ex.args[2], Expr) || ex.args[2].head != :...) && - (!isa(ex.args[3], Expr) || ex.args[3].head != :...) && - (ex.args[1] === :(==) || - Base.operator_precedence(ex.args[1]) == - Base.operator_precedence(:(==))) - - -""" - @ensure postcondition [message] -Throw `ArgumentError` if `postcondition` is false. -""" -macro ensure(condition, msg = "`$condition`") - - if DEBUG_LEVEL[] < 0 - return :() - end - - if iscondition(condition) - l,r = condition.args[2], condition.args[3] - ls, rs = string(l), string(r) - return quote - if ! $(esc(condition)) - # FIXME double-execution of condition l and r! - throw(postcondition_error($(esc(msg)), $(_funcname_expr()), - $ls, $(esc(l)), $rs, $(esc(r)))) - end - end - end - - :(if ! $(esc(condition)) throw(postcondition_error($(esc(msg)), $(_funcname_expr()))) end) -end - -end # module \ No newline at end of file diff --git a/src/Connections.jl b/src/Connections.jl deleted file mode 100644 index ef586b668..000000000 --- a/src/Connections.jl +++ /dev/null @@ -1,673 +0,0 @@ -""" -This module provides the [`newconnection`](@ref) function with support for: -- Opening TCP and SSL connections. -- Reusing connections for multiple Request/Response Messages - -This module defines a [`Connection`](@ref) -struct to manage the lifetime of a connection and its reuse. -Methods are provided for `eof`, `readavailable`, -`unsafe_write` and `close`. -This allows the `Connection` object to act as a proxy for the -`TCPSocket` or `SSLContext` that it wraps. - -[`POOLS`](@ref) are used to manage connection pooling. Connections -are identified by their host, port, whether they require -ssl verification, and whether they are a client or server connection. -If a subsequent request matches these properties of a previous connection -and limits are respected (reuse limit, idle timeout), and it wasn't otherwise -remotely closed, a connection will be reused. -""" -module Connections - -export Connection, newconnection, releaseconnection, getrawstream, inactiveseconds, shouldtimeout, default_connection_limit, set_default_connection_limit!, Pool - -using Sockets, NetworkOptions -using MbedTLS: SSLConfig, SSLContext, setup!, associate!, hostname!, handshake! -using MbedTLS, OpenSSL, ConcurrentUtilities -using ..IOExtras, ..Conditions, ..Exceptions - -const nolimit = typemax(Int) - -taskid(t=current_task()) = string(hash(t) & 0xffff, base=16, pad=4) - -const default_connection_limit = Ref{Int}() - -function __init__() - # default connection limit is 4x the number of threads - # this was chosen after some empircal benchmarking on aws/azure machines - # where, for non-data-intensive workloads, having at least 4x ensured - # there was no artificial restriction on overall throughput - default_connection_limit[] = max(16, Threads.nthreads() * 4) - nosslcontext[] = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) - TCP_POOL[] = CPool{Sockets.TCPSocket}(default_connection_limit[]) - MBEDTLS_POOL[] = CPool{MbedTLS.SSLContext}(default_connection_limit[]) - OPENSSL_POOL[] = CPool{OpenSSL.SSLStream}(default_connection_limit[]) - return -end - -function set_default_connection_limit!(n) - default_connection_limit[] = n - # reinitialize the global connection pools - TCP_POOL[] = CPool{Sockets.TCPSocket}(n) - MBEDTLS_POOL[] = CPool{MbedTLS.SSLContext}(n) - OPENSSL_POOL[] = CPool{OpenSSL.SSLStream}(n) - return -end - -""" - Connection - -A `Sockets.TCPSocket`, `MbedTLS.SSLContext` or `OpenSSL.SSLStream` connection to a HTTP `host` and `port`. - -Fields: -- `host::String` -- `port::String`, exactly as specified in the URI (i.e. may be empty). -- `idle_timeout`, No. of seconds to maintain connection after last request/response. -- `require_ssl_verification`, whether ssl verification is required for an ssl connection -- `keepalive`, whether the tcp socket should have keepalive enabled -- `peerip`, remote IP adress (used for debug/log messages). -- `peerport`, remote TCP port number (used for debug/log messages). -- `localport`, local TCP port number (used for debug messages). -- `io::T`, the `Sockets.TCPSocket`, `MbedTLS.SSLContext` or `OpenSSL.SSLStream`. -- `clientconnection::Bool`, whether the Connection was created from client code (as opposed to server code) -- `buffer::IOBuffer`, left over bytes read from the connection after - the end of a response header (or chunksize). These bytes are usually - part of the response body. -- `timestamp`, time data was last received. -- `readable`, whether the Connection object is readable -- `writable`, whether the Connection object is writable -""" -mutable struct Connection{IO_t <: IO} <: IO - host::String - port::String - idle_timeout::Int - require_ssl_verification::Bool - keepalive::Bool - peerip::IPAddr # for debugging/logging - peerport::UInt16 # for debugging/logging - localport::UInt16 # debug only - io::IO_t - clientconnection::Bool - buffer::IOBuffer - timestamp::Float64 - readable::Bool - writable::Bool - writebuffer::IOBuffer - state::Any # populated & used by Servers code -end - -has_tcpsocket(c::Connection) = applicable(tcpsocket, c.io) -IOExtras.tcpsocket(c::Connection) = tcpsocket(c.io) - -""" - connectionkey - -Used for "hashing" a Connection object on just the key properties necessary for determining -connection re-useability. That is, when a new request calls `newconnection`, we take the -request parameters of host and port, and if ssl verification is required, if keepalive is enabled, -and if an existing Connection was already created with the exact. -same parameters, we can re-use it (as long as it's not already being used, obviously). -""" -connectionkey(x::Connection) = (x.host, x.port, x.require_ssl_verification, x.keepalive, x.clientconnection) - -const ConnectionKeyType = Tuple{AbstractString, AbstractString, Bool, Bool, Bool} - -Connection(host::AbstractString, port::AbstractString, - idle_timeout::Int, - require_ssl_verification::Bool, keepalive::Bool, io::T, client=true) where {T}= - Connection{T}(host, port, idle_timeout, - require_ssl_verification, keepalive, - safe_getpeername(io)..., localport(io), - io, client, PipeBuffer(), time(), false, false, IOBuffer(), nothing) - -Connection(io; require_ssl_verification::Bool=true, keepalive::Bool=true) = - Connection("", "", 0, require_ssl_verification, keepalive, io, false) - -getrawstream(c::Connection) = c.io - -inactiveseconds(c::Connection)::Float64 = time() - c.timestamp - -shouldtimeout(c::Connection, readtimeout) = !isreadable(c) || inactiveseconds(c) > readtimeout - -Base.unsafe_write(c::Connection, p::Ptr{UInt8}, n::UInt) = - unsafe_write(c.io, p, n) - -Base.isopen(c::Connection) = isopen(c.io) - -""" - flush(c::Connection) - -Flush a TCP buffer by toggling the Nagle algorithm off and on again for a socket. -This forces the socket to send whatever data is within its buffer immediately, -rather than waiting 10's of milliseconds for the buffer to fill more. -""" -function Base.flush(c::Connection) - # Flushing the TCP buffer requires support for `Sockets.nagle()` - # which was only added in Julia v1.3 - @static if VERSION >= v"1.3" - sock = tcpsocket(c) - # I don't understand why uninitializd sockets can get here, but they can - if sock.status ∉ (Base.StatusInit, Base.StatusUninit) && isopen(sock) - Sockets.nagle(sock, false) - Sockets.nagle(sock, true) - end - end -end - -Base.isreadable(c::Connection) = c.readable -Base.iswritable(c::Connection) = c.writable - -function Base.eof(c::Connection) - @require isreadable(c) || !isopen(c) - if bytesavailable(c) > 0 - return false - end - return eof(c.io) -end - -Base.bytesavailable(c::Connection) = bytesavailable(c.buffer) + - bytesavailable(c.io) - -function Base.read(c::Connection, nb::Int) - nb = min(nb, bytesavailable(c)) - bytes = Base.StringVector(nb) - GC.@preserve bytes unsafe_read(c, pointer(bytes), nb) - return bytes -end - -function Base.read(c::Connection, ::Type{UInt8}) - if bytesavailable(c.buffer) == 0 - read_to_buffer(c) - end - return read(c.buffer, UInt8) -end - -function Base.unsafe_read(c::Connection, p::Ptr{UInt8}, n::UInt) - l = bytesavailable(c.buffer) - if l > 0 - nb = min(l, n) - unsafe_read(c.buffer, p, nb) - p += nb - n -= nb - c.timestamp = time() - end - if n > 0 - # try-catch underlying errors here - # as the Connection object, we don't really care - # if the underlying socket was closed/terminated - # or just plain reached EOF, so we catch any - # Base.IOErrors and just throw as EOFError - # that way we get more consistent errors thrown - # at the headers/body parsing level - try - unsafe_read(c.io, p, n) - c.timestamp = time() - catch e - e isa Base.IOError && throw(EOFError()) - rethrow(e) - end - end - return nothing -end - -function read_to_buffer(c::Connection, sizehint=4096) - buf = c.buffer - - # Reset the buffer if it is empty. - if bytesavailable(buf) == 0 - buf.size = 0 - buf.ptr = 1 - end - - # Wait for data. - if eof(c.io) - throw(EOFError()) - end - - # Read from stream into buffer. - n = min(sizehint, bytesavailable(c.io)) - buf = c.buffer - Base.ensureroom(buf, n) - GC.@preserve buf unsafe_read(c.io, pointer(buf.data, buf.size + 1), n) - buf.size += n -end - -""" -Read until `find_delimiter(bytes)` returns non-zero. -Return view of bytes up to the delimiter. -""" -function IOExtras.readuntil(c::Connection, f::F #=Vector{UInt8} -> Int=#, - sizehint=4096) where {F <: Function} - buf = c.buffer - if bytesavailable(buf) == 0 - read_to_buffer(c, sizehint) - end - while isempty(begin bytes = IOExtras.readuntil(buf, f) end) - read_to_buffer(c, sizehint) - end - return bytes -end - -""" - startwrite(::Connection) -""" -function IOExtras.startwrite(c::Connection) - @require !iswritable(c) - c.writable = true - @debug "👁 Start write:$c" - return -end - -""" - closewrite(::Connection) - -Signal that an entire Request Message has been written to the `Connection`. -""" -function IOExtras.closewrite(c::Connection) - @require iswritable(c) - c.writable = false - @debug "🗣 Write done: $c" - flush(c) - return -end - -""" - startread(::Connection) -""" -function IOExtras.startread(c::Connection) - @require !isreadable(c) - c.timestamp = time() - c.readable = true - @debug "👁 Start read: $c" - return -end - -""" -Wait for `c` to receive data or reach EOF. -Close `c` on EOF. -TODO: or if response data arrives when no request was sent (isreadable == false). -""" -function monitor_idle_connection(c::Connection) - try - if eof(c.io) ;@debug "💀 Closed: $c" - close(c.io) - end - catch ex - @try Base.IOError close(c.io) - ex isa Base.IOError || rethrow() - end - nothing -end - -""" - closeread(::Connection) - -Signal that an entire Response Message has been read from the `Connection`. -""" -function IOExtras.closeread(c::Connection) - @require isreadable(c) - c.readable = false - @debug "✉️ Read done: $c" - if c.clientconnection - t = Threads.@spawn monitor_idle_connection(c) - @isdefined(errormonitor) && errormonitor(t) - end - return -end - -Base.wait_close(c::Connection) = Base.wait_close(tcpsocket(c)) - -function Base.close(c::Connection) - if iswritable(c) - closewrite(c) - end - if isreadable(c) - closeread(c) - end - try - close(c.io) - if bytesavailable(c) > 0 - purge(c) - end - catch - # ignore errors closing underlying socket - end - return -end - -""" - purge(::Connection) - -Remove unread data from a `Connection`. -""" -function purge(c::Connection) - @require !isopen(c.io) - while !eof(c.io) - readavailable(c.io) - end - c.buffer.size = 0 - c.buffer.ptr = 1 - @ensure bytesavailable(c) == 0 -end - -const CPool{T} = ConcurrentUtilities.Pool{ConnectionKeyType, Connection{T}} - -""" - HTTP.Pool(max::Int=HTTP.default_connection_limit[]) - -Connection pool for managing the reuse of HTTP connections. -`max` controls the maximum number of concurrent connections allowed -and defaults to the `HTTP.default_connection_limit` value. - -A pool can be passed to any of the `HTTP.request` methods via the `pool` keyword argument. -""" -struct Pool - lock::ReentrantLock - tcp::CPool{Sockets.TCPSocket} - mbedtls::CPool{MbedTLS.SSLContext} - openssl::CPool{OpenSSL.SSLStream} - other::IdDict{Type, CPool} - max::Int -end - -function Pool(max::Union{Int, Nothing}=nothing) - max = something(max, default_connection_limit[]) - return Pool(ReentrantLock(), - CPool{Sockets.TCPSocket}(max), - CPool{MbedTLS.SSLContext}(max), - CPool{OpenSSL.SSLStream}(max), - IdDict{Type, CPool}(), - max, - ) -end - -# Default HTTP global connection pools -const TCP_POOL = Ref{CPool{Sockets.TCPSocket}}() -const MBEDTLS_POOL = Ref{CPool{MbedTLS.SSLContext}}() -const OPENSSL_POOL = Ref{CPool{OpenSSL.SSLStream}}() -const OTHER_POOL = Lockable(IdDict{Type, CPool}()) - -getpool(::Nothing, ::Type{Sockets.TCPSocket}) = TCP_POOL[] -getpool(::Nothing, ::Type{MbedTLS.SSLContext}) = MBEDTLS_POOL[] -getpool(::Nothing, ::Type{OpenSSL.SSLStream}) = OPENSSL_POOL[] -getpool(::Nothing, ::Type{T}) where {T} = Base.@lock OTHER_POOL get!(OTHER_POOL[], T) do - CPool{T}(default_connection_limit[]) -end - -function getpool(pool::Pool, ::Type{T})::CPool{T} where {T} - if T === Sockets.TCPSocket - return pool.tcp - elseif T === MbedTLS.SSLContext - return pool.mbedtls - elseif T === OpenSSL.SSLStream - return pool.openssl - else - return Base.@lock pool.lock get!(() -> CPool{T}(pool.max), pool.other, T) - end -end - -""" - closeall(pool::HTTP.Pool=nothing) - -Remove and close all connections in the `pool` to avoid any connection reuse. -If `pool` is not specified, the default global pools are closed. -""" -function closeall(pool::Union{Nothing, Pool}=nothing) - if pool === nothing - drain!(TCP_POOL[]) - drain!(MBEDTLS_POOL[]) - drain!(OPENSSL_POOL[]) - Base.@lock OTHER_POOL foreach(drain!, values(OTHER_POOL[])) - else - drain!(pool.tcp) - drain!(pool.mbedtls) - drain!(pool.openssl) - Base.@lock pool.lock foreach(drain!, values(pool.other)) - end - return -end - -function connection_isvalid(c, idle_timeout) - check = isopen(c) && inactiveseconds(c) <= idle_timeout - check || close(c) - return check -end - -@noinline connection_limit_warning(cl) = cl === nothing || - @warn "connection_limit no longer supported as a keyword argument; use `HTTP.set_default_connection_limit!($cl)` before any requests are made or construct a shared pool via `POOL = HTTP.Pool($cl)` and pass to each request like `pool=POOL` instead." - -const DEFAULT_CONNECT_TIMEOUT = Ref{Int}(30) - -""" - newconnection(type, host, port) -> Connection - -Find a reusable `Connection` in the `pool`, -or create a new `Connection` if required. -""" -function newconnection(::Type{T}, - host::AbstractString, - port::AbstractString; - pool::Union{Nothing, Pool}=nothing, - connection_limit=nothing, - forcenew::Bool=false, - idle_timeout=typemax(Int), - connect_timeout::Int=DEFAULT_CONNECT_TIMEOUT[], - require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), - keepalive::Bool=true, - kw...) where {T <: IO} - connection_limit_warning(connection_limit) - return acquire( - getpool(pool, T), - (host, port, require_ssl_verification, keepalive, true); - forcenew=forcenew, - isvalid=c->connection_isvalid(c, Int(idle_timeout))) do - Connection(host, port, - idle_timeout, require_ssl_verification, keepalive, - connect_timeout > 0 ? - try_with_timeout(_ -> - getconnection(T, host, port; - require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...), - connect_timeout) : - getconnection(T, host, port; - require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) - ) - end -end - -function releaseconnection(c::Connection{T}, reuse; pool::Union{Nothing, Pool}=nothing, kw...) where {T} - c.timestamp = time() - release(getpool(pool, T), connectionkey(c), reuse ? c : nothing) -end - -function keepalive!(tcp) - Base.iolock_begin() - try - Base.check_open(tcp) - msg = ccall(:uv_tcp_keepalive, Cint, (Ptr{Nothing}, Cint, Cuint), - tcp.handle, 1, 1) - Base.uv_error("failed to set keepalive on tcp socket", msg) - finally - Base.iolock_end() - end - return -end - -struct ConnectTimeout <: Exception - host - port -end - -function checkconnected(tcp) - if tcp.status == Base.StatusConnecting - close(tcp) - return false - end - return true -end - -function getconnection(::Type{TCPSocket}, - host::AbstractString, - port::AbstractString; - # set keepalive to true by default since it's cheap and helps keep long-running requests/responses - # alive in the face of heavy workloads where Julia's task scheduler might take a while to - # keep up with midflight requests - keepalive::Bool=true, - readtimeout::Int=0, - kw...)::TCPSocket - - p::UInt = isempty(port) ? UInt(80) : parse(UInt, port) - @debug "TCP connect: $host:$p..." - addrs = Sockets.getalladdrinfo(host) - err = ErrorException("failed to connect") - for addr in addrs - try - tcp = Sockets.connect(addr, p) - keepalive && keepalive!(tcp) - return tcp - catch e - err = e - end - end - throw(err) -end - -const nosslconfig = SSLConfig() -const nosslcontext = Ref{OpenSSL.SSLContext}() -const default_sslconfig = Ref{Union{Nothing, SSLConfig}}(nothing) -const noverify_sslconfig = Ref{Union{Nothing, SSLConfig}}(nothing) - -function global_sslconfig(require_ssl_verification::Bool)::SSLConfig - if default_sslconfig[] === nothing - default_sslconfig[] = SSLConfig(true) - noverify_sslconfig[] = SSLConfig(false) - end - if haskey(ENV, "HTTP_CA_BUNDLE") - MbedTLS.ca_chain!(default_sslconfig[], MbedTLS.crt_parse(read(ENV["HTTP_CA_BUNDLE"], String))) - elseif haskey(ENV, "CURL_CA_BUNDLE") - MbedTLS.ca_chain!(default_sslconfig[], MbedTLS.crt_parse(read(ENV["CURL_CA_BUNDLE"], String))) - end - return require_ssl_verification ? default_sslconfig[] : noverify_sslconfig[] -end - -function global_sslcontext()::OpenSSL.SSLContext - @static if isdefined(OpenSSL, :ca_chain!) - if haskey(ENV, "HTTP_CA_BUNDLE") - sslcontext = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) - OpenSSL.ca_chain!(sslcontext, ENV["HTTP_CA_BUNDLE"]) - return sslcontext - elseif haskey(ENV, "CURL_CA_BUNDLE") - sslcontext = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) - OpenSSL.ca_chain!(sslcontext, ENV["CURL_CA_BUNDLE"]) - return sslcontext - end - end - return nosslcontext[] -end - -function getconnection(::Type{SSLContext}, - host::AbstractString, - port::AbstractString; - kw...)::SSLContext - - port = isempty(port) ? "443" : port - @debug "SSL connect: $host:$port..." - tcp = getconnection(TCPSocket, host, port; kw...) - return sslconnection(SSLContext, tcp, host; kw...) -end - -function getconnection(::Type{SSLStream}, - host::AbstractString, - port::AbstractString; - kw...)::SSLStream - - port = isempty(port) ? "443" : port - @debug "SSL connect: $host:$port..." - tcp = getconnection(TCPSocket, host, port; kw...) - return sslconnection(SSLStream, tcp, host; kw...) -end - -function sslconnection(::Type{SSLStream}, tcp::TCPSocket, host::AbstractString; - require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), - sslconfig::OpenSSL.SSLContext=nosslcontext[], - kw...)::SSLStream - if sslconfig === nosslcontext[] - sslconfig = global_sslcontext() - end - # Create SSL stream. - ssl_stream = SSLStream(sslconfig, tcp) - OpenSSL.hostname!(ssl_stream, host) - OpenSSL.connect(ssl_stream; require_ssl_verification) - return ssl_stream -end - -function sslconnection(::Type{SSLContext}, tcp::TCPSocket, host::AbstractString; - require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), - sslconfig::SSLConfig=nosslconfig, - kw...)::SSLContext - if sslconfig === nosslconfig - sslconfig = global_sslconfig(require_ssl_verification) - end - io = SSLContext() - setup!(io, sslconfig) - associate!(io, tcp) - hostname!(io, host) - handshake!(io) - return io -end - -function sslupgrade(::Type{IOType}, c::Connection{T}, - host::AbstractString; - pool::Union{Nothing, Pool}=nothing, - require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), - keepalive::Bool=true, - readtimeout::Int=0, - kw...)::Connection{IOType} where {T, IOType} - # initiate the upgrade to SSL - # if the upgrade fails, an error will be thrown and the original c will be closed - # in ConnectionRequest - tls = if readtimeout > 0 - try_with_timeout(readtimeout) do _ - sslconnection(IOType, c.io, host; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) - end - else - sslconnection(IOType, c.io, host; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) - end - # success, now we turn it into a new Connection - conn = Connection(host, "", 0, require_ssl_verification, keepalive, tls) - # release the "old" one, but don't return the connection since we're hijacking the socket - release(getpool(pool, T), connectionkey(c)) - # and return the new one - return acquire(() -> conn, getpool(pool, IOType), connectionkey(conn); forcenew=true) -end - -function Base.show(io::IO, c::Connection) - nwaiting = has_tcpsocket(c) ? bytesavailable(tcpsocket(c)) : 0 - print( - io, - tcpstatus(c), " ", - "$(lpad(round(Int, time() - c.timestamp), 3))s ", - c.host, ":", - c.port != "" ? c.port : Int(c.peerport), ":", Int(c.localport), - bytesavailable(c.buffer) > 0 ? - " $(bytesavailable(c.buffer))-byte excess" : "", - nwaiting > 0 ? " $nwaiting bytes waiting" : "", - has_tcpsocket(c) ? " $(Base._fd(tcpsocket(c)))" : "") -end - -function tcpstatus(c::Connection) - if !has_tcpsocket(c) - return "" - end - s = Base.uv_status_string(tcpsocket(c)) - if s == "connecting" return "🔜🔗" - elseif s == "open" return "🔗 " - elseif s == "active" return "🔁 " - elseif s == "paused" return "⏸ " - elseif s == "closing" return "🔜💀" - elseif s == "closed" return "💀 " - else - return s - end -end - -end # module Connections diff --git a/src/Exceptions.jl b/src/Exceptions.jl deleted file mode 100644 index 579a1ae95..000000000 --- a/src/Exceptions.jl +++ /dev/null @@ -1,112 +0,0 @@ -module Exceptions - -export @try, HTTPError, ConnectError, TimeoutError, StatusError, RequestError, current_exceptions_to_string -using ExceptionUnwrapping -import ..HTTP # for doc references - -@eval begin -""" - @try Permitted Error Types expr - -Convenience macro for wrapping an expression in a try/catch block -where thrown exceptions are ignored. -""" -macro $(:try)(exes...) - errs = Any[exes...] - ex = pop!(errs) - isempty(errs) && error("no permitted errors") - quote - try $(esc(ex)) - catch e - e isa InterruptException && rethrow(e) - |($([:(e isa $(esc(err))) for err in errs]...)) || rethrow(e) - end - end -end -end # @eval - -abstract type HTTPError <: Exception end - -""" - HTTP.ConnectError - -Raised when an error occurs while trying to establish a request connection to -the remote server. To see the underlying error, see the `error` field. -""" -struct ConnectError <: HTTPError - url::String # the URL of the request - error::Any # underlying error -end - -ExceptionUnwrapping.unwrap_exception(e::ConnectError) = e.error - -function Base.showerror(io::IO, e::ConnectError) - print(io, "HTTP.ConnectError for url = `$(e.url)`: ") - Base.showerror(io, e.error) -end - -""" - HTTP.TimeoutError - -Raised when a request times out according to `readtimeout` keyword argument provided. -""" -struct TimeoutError <: HTTPError - readtimeout::Int -end - -Base.showerror(io::IO, e::TimeoutError) = - print(io, "TimeoutError: Connection closed after $(e.readtimeout) seconds") - -""" - HTTP.StatusError - -Raised when an `HTTP.Response` has a `4xx`, `5xx` or unrecognised status code. - -Fields: - - `status::Int16`, the response status code. - - `method::String`, the request method. - - `target::String`, the request target. - - `response`, the [`HTTP.Response`](@ref) -""" -struct StatusError <: HTTPError - status::Int16 - method::String - target::String - response::Any -end - -""" - HTTP.RequestError - -Raised when an error occurs while physically sending a request to the remote server -or reading the response back. To see the underlying error, see the `error` field. -""" -struct RequestError <: HTTPError - request::Any - error::Any -end - -ExceptionUnwrapping.unwrap_exception(e::RequestError) = e.error - -function Base.showerror(io::IO, e::RequestError) - println(io, "HTTP.RequestError:") - println(io, "HTTP.Request:") - Base.show(io, e.request) - println(io, "Underlying error:") - Base.showerror(io, e.error) -end - -function current_exceptions_to_string() - buf = IOBuffer() - println(buf) - println(buf, "\n===========================\nHTTP Error message:\n") - exc = @static if VERSION >= v"1.8.0-" - Base.current_exceptions() - else - Base.catch_stack() - end - Base.display_error(buf, exc) - return String(take!(buf)) -end - -end # module Exceptions diff --git a/src/HTTP.jl b/src/HTTP.jl index cbbf1be70..ae1e59cfd 100644 --- a/src/HTTP.jl +++ b/src/HTTP.jl @@ -1,634 +1,110 @@ module HTTP -export startwrite, startread, closewrite, closeread, - @logfmt_str, common_logfmt, combined_logfmt, WebSockets +using CodecZlib, URIs, Mmap, Base64, Dates, Sockets +using LibAwsCommon, LibAwsIO, LibAwsHTTPFork -const DEBUG_LEVEL = Ref(0) +export @logfmt_str, common_logfmt, combined_logfmt +export WebSockets -Base.@deprecate escape escapeuri - -using Base64, Sockets, Dates, URIs, MbedTLS, OpenSSL - -function access_threaded(f, v::Vector) - tid = Threads.threadid() - 0 < tid <= length(v) || _length_assert() - if @inbounds isassigned(v, tid) - @inbounds x = v[tid] - else - x = f() - @inbounds v[tid] = x - end - return x -end -@noinline _length_assert() = @assert false "0 < tid <= v" - -function open end - -const SOCKET_TYPE_TLS = Ref{Any}(OpenSSL.SSLStream) - -include("Conditions.jl") ;using .Conditions +include("utils.jl") include("access_log.jl") - -include("Pairs.jl") ;using .Pairs -include("IOExtras.jl") ;using .IOExtras -include("Strings.jl") ;using .Strings -include("Exceptions.jl") ;using .Exceptions -include("sniff.jl") ;using .Sniff -include("multipart.jl") ;using .Forms -include("Parsers.jl") ;import .Parsers: Headers, Header, - ParseError -include("Connections.jl") ;using .Connections -# backwards compat -const ConnectionPool = Connections -include("StatusCodes.jl") ;using .StatusCodes -include("Messages.jl") ;using .Messages -include("cookies.jl") ;using .Cookies -include("Streams.jl") ;using .Streams - -getrequest(r::Request) = r -getrequest(s::Stream) = s.message.request - -# Wraps client-side "layer" to track the amount of time spent in it as a request is processed. -function observelayer(f) - function observation(req_or_stream; kw...) - req = getrequest(req_or_stream) - nm = nameof(f) - cntnm = Symbol(nm, "_count") - durnm = Symbol(nm, "_duration_ms") - start_time = time() - req.context[cntnm] = Base.get(req.context, cntnm, 0) + 1 - try - return f(req_or_stream; kw...) - finally - req.context[durnm] = Base.get(req.context, durnm, 0) + (time() - start_time) * 1000 - # @info "observed layer = $f, count = $(req.context[cntnm]), duration = $(req.context[durnm])" - end - end -end - -include("clientlayers/MessageRequest.jl"); using .MessageRequest -include("clientlayers/RedirectRequest.jl"); using .RedirectRequest -include("clientlayers/HeadersRequest.jl"); using .HeadersRequest -include("clientlayers/CookieRequest.jl"); using .CookieRequest -include("clientlayers/TimeoutRequest.jl"); using .TimeoutRequest -include("clientlayers/ExceptionRequest.jl"); using .ExceptionRequest -include("clientlayers/RetryRequest.jl"); using .RetryRequest -include("clientlayers/ConnectionRequest.jl"); using .ConnectionRequest -include("clientlayers/StreamRequest.jl"); using .StreamRequest - -include("download.jl") -include("Servers.jl") ;using .Servers; using .Servers: listen -include("Handlers.jl") ;using .Handlers; using .Handlers: serve -include("parsemultipart.jl") ;using .MultiPartParsing: parse_multipart_form -include("WebSockets.jl") ;using .WebSockets - -const nobody = UInt8[] - -""" - - HTTP.request(method, url [, headers [, body]]; ]) -> HTTP.Response - -Send a HTTP Request Message and receive a HTTP Response Message. - -e.g. -```julia -r = HTTP.request("GET", "http://httpbin.org/ip") -r = HTTP.get("http://httpbin.org/ip") # equivalent shortcut -println(r.status) -println(String(r.body)) -``` - -`headers` can be any collection where -`[string(k) => string(v) for (k,v) in headers]` yields `Vector{Pair}`. -e.g. a `Dict()`, a `Vector{Tuple}`, a `Vector{Pair}` or an iterator. -By convention, if a header _value_ is an empty string, it will not be written -when sending a request (following the curl convention). By default, a copy of -provided headers is made (since required headers are typically set during the request); -to avoid this copy and have HTTP.jl mutate the provided headers array, pass `copyheaders=false` -as an additional keyword argument to the request. - -The `body` argument can be a variety of objects: - - - an `AbstractDict` or `NamedTuple` to be serialized as the "application/x-www-form-urlencoded" content type - - any `AbstractString` or `AbstractVector{UInt8}` which will be sent "as is" for the request body - - a readable `IO` stream or any `IO`-like type `T` for which - `eof(T)` and `readavailable(T)` are defined. This stream will be read and sent until `eof` is `true`. - This object should support the `mark`/`reset` methods if request retries are desired (if not, no retries will be attempted). - - Any collection or iterable of the above (`AbstractDict`, `AbstractString`, `AbstractVector{UInt8}`, or `IO`) - which will result in a "chunked" request body, where each iterated element will be sent as a separate chunk - - a [`HTTP.Form`](@ref), which will be serialized as the "multipart/form-data" content-type - -The `HTTP.Response` struct contains: - - - `status::Int16` e.g. `200` - - `headers::Vector{Pair{String,String}}` - e.g. ["Server" => "Apache", "Content-Type" => "text/html"] - - `body::Vector{UInt8}` or `::IO`, the Response Body bytes or the `io` argument - provided via the `response_stream` keyword argument - -Functions `HTTP.get`, `HTTP.put`, `HTTP.post` and `HTTP.head` are defined as -shorthand for `HTTP.request("GET", ...)`, etc. - -Supported optional keyword arguments: - - - `query = nothing`, a `Pair` or `Dict` of key => values to be included in the url - - `response_stream = nothing`, a writeable `IO` stream or any `IO`-like - type `T` for which `write(T, AbstractVector{UInt8})` is defined. The response body - will be written to this stream instead of returned as a `Vector{UInt8}`. - - `verbose = 0`, set to `1` or `2` for increasingly verbose logging of the - request and response process - - `connect_timeout = 30`, close the connection after this many seconds if it - is still attempting to connect. Use `connect_timeout = 0` to disable. - - `pool = nothing`, an `HTTP.Pool` object to use for managing the reuse of connections between requests. - By default, a global pool is used, which is shared across all requests. To create a pool for a specific set of requests, - use `pool = HTTP.Pool(max::Int)`, where `max` controls the maximum number of concurrent connections allowed to be used for requests at a given time. - - `readtimeout = 0`, abort a request after this many seconds. Will trigger retries if applicable. Use `readtimeout = 0` to disable. - - `status_exception = true`, throw `HTTP.StatusError` for response status >= 300. - - Basic authentication is detected automatically from the provided url's `userinfo` (in the form `scheme://user:password@host`) - and adds the `Authorization: Basic` header; this can be disabled by passing `basicauth=false` - - `canonicalize_headers = false`, rewrite request and response headers in - Canonical-Camel-Dash-Format. - - `proxy = proxyurl`, pass request through a proxy given as a url; alternatively, the `http_proxy`, `HTTP_PROXY`, `https_proxy`, `HTTPS_PROXY`, and `no_proxy` - environment variables are also detected/used; if set, they will be used automatically when making requests. - - `detect_content_type = false`: if `true` and the request body is not a form or `IO`, it will be - inspected and the "Content-Type" header will be set to the detected content type. - - `decompress = nothing`, by default, decompress the response body if the response has a - "Content-Encoding" header set to "gzip". If `decompress=true`, decompress the response body - regardless of `Content-Encoding` header. If `decompress=false`, do not decompress the response body. - - `logerrors = false`, if `true`, `HTTP.StatusError`, `HTTP.TimeoutError`, `HTTP.IOError`, and `HTTP.ConnectError` will be - logged via `@error` as they happen, regardless of whether the request is then retried or not. Useful for debugging or - monitoring requests where there's worry of certain errors happening but ignored because of retries. - - `logtag = nothing`, if provided, will be used as the tag for error logging. Useful for debugging or monitoring requests. - - `observelayers = false`, if `true`, enables the `HTTP.observelayer` to wrap each client-side "layer" to track the amount of - time spent in each layer as a request is processed. This can be useful for debugging performance issues. Note that when retries - or redirects happen, the time spent in each layer is cumulative, as noted by the `[layer]_count`. The metrics are stored - in the `Request.context` dictionary, and can be accessed like `HTTP.get(...).request.context` - -Retry arguments: - - `retry = true`, retry idempotent requests in case of error. - - `retries = 4`, number of times to retry. - - `retry_non_idempotent = false`, retry non-idempotent requests too. e.g. POST. - - `retry_delays = ExponentialBackOff(n = retries)`, provide a custom `ExponentialBackOff` object to control the delay between retries. - - `retry_check = (s, ex, req, resp, resp_body) -> Bool`, provide a custom function to control whether a retry should be attempted. - The function should accept 5 arguments: the delay state, exception, request, response (an `HTTP.Response` object *if* a request was - successfully made, otherwise `nothing`), and `resp_body` response body (which may be `nothing` if there is no response yet, otherwise - a `Vector{UInt8}`), and return `true` if a retry should be attempted. - -Redirect arguments: - - `redirect = true`, follow 3xx redirect responses; i.e. additional requests will be made to the redirected location - - `redirect_limit = 3`, maximum number of times a redirect will be followed - - `redirect_method = nothing`, the method to use for the redirected request; by default, - GET will be used, only responses with 307/308 will use the same original request method. - Pass `redirect_method=:same` to pass the same method as the orginal request though note that some servers - may not respond/accept the same method. It's also valid to pass the exact method to use - as a string, like `redirect_method="PUT"`. - - `forwardheaders = true`, forward original headers on redirect. - -SSL arguments: - - `require_ssl_verification = NetworkOptions.verify_host(host)`, pass `MBEDTLS_SSL_VERIFY_REQUIRED` to - the mbed TLS library. - ["... peer must present a valid certificate, handshake is aborted if - verification failed."](https://tls.mbed.org/api/ssl_8h.html#a5695285c9dbfefec295012b566290f37) - - `sslconfig = SSLConfig(require_ssl_verification)` - - `socket_type_tls = MbedTLS.SSLContext`, the type of socket to use for TLS connections. Defaults to `MbedTLS.SSLContext`. - Also supported is passing `socket_type_tls = OpenSSL.SSLStream`. To change the global default, set `HTTP.SOCKET_TYPE_TLS[] = OpenSSL.SSLStream`. - -Cookie arguments: - - `cookies::Union{Bool, Dict{<:AbstractString, <:AbstractString}} = true`, enable cookies, or alternatively, - pass a `Dict{AbstractString, AbstractString}` of name-value pairs to manually pass cookies in the request "Cookie" header - - `cookiejar::HTTP.CookieJar=HTTP.COOKIEJAR`: threadsafe cookie jar struct for keeping track of cookies per host; - a global cookie jar is used by default. - -## Request Body Examples - -String body: -```julia -HTTP.request("POST", "http://httpbin.org/post", [], "post body data") -``` - -Stream body from file: -```julia -io = open("post_data.txt", "r") -HTTP.request("POST", "http://httpbin.org/post", [], io) -``` - -Generator body: -```julia -chunks = ("chunk\$i" for i in 1:1000) -HTTP.request("POST", "http://httpbin.org/post", [], chunks) -``` - -Collection body: -```julia -chunks = [preamble_chunk, data_chunk, checksum(data_chunk)] -HTTP.request("POST", "http://httpbin.org/post", [], chunks) -``` - -`open() do io` body: -```julia -HTTP.open("POST", "http://httpbin.org/post") do io - write(io, preamble_chunk) - write(io, data_chunk) - write(io, checksum(data_chunk)) -end -``` - -## Response Body Examples - -String body: -```julia -r = HTTP.request("GET", "http://httpbin.org/get") -println(String(r.body)) -``` - -Stream body to file: -```julia -io = open("get_data.txt", "w") -r = HTTP.request("GET", "http://httpbin.org/get", response_stream=io) -close(io) -println(read("get_data.txt")) -``` - -Stream body through buffer: -```julia -r = HTTP.get("http://httpbin.org/get", response_stream=IOBuffer()) -println(String(take!(r.body))) -``` - -Stream body through `open() do io`: -```julia -r = HTTP.open("GET", "http://httpbin.org/stream/10") do io - while !eof(io) - println(String(readavailable(io))) - end -end - -HTTP.open("GET", "https://tinyurl.com/bach-cello-suite-1-ogg") do http - n = 0 - r = startread(http) - l = parse(Int, HTTP.header(r, "Content-Length")) - open(`vlc -q --play-and-exit --intf dummy -`, "w") do vlc - while !eof(http) - bytes = readavailable(http) - write(vlc, bytes) - n += length(bytes) - println("streamed \$n-bytes \$((100*n)÷l)%\\u1b[1A") - end - end -end -``` - -Interfacing with RESTful JSON APIs: -```julia -using JSON -params = Dict("user"=>"RAO...tjN", "token"=>"NzU...Wnp", "message"=>"Hello!") -url = "http://api.domain.com/1/messages.json" -r = HTTP.post(url, body=JSON.json(params)) -println(JSON.parse(String(r.body))) -``` - -Stream bodies from and to files: -```julia -in = open("foo.png", "r") -out = open("foo.jpg", "w") -HTTP.request("POST", "http://convert.com/png2jpg", [], in, response_stream=out) -``` - -Stream bodies through: `open() do io`: -```julia -HTTP.open("POST", "http://music.com/play") do io - write(io, JSON.json([ - "auth" => "12345XXXX", - "song_id" => 7, - ])) - r = startread(io) - @show r.status - while !eof(io) - bytes = readavailable(io) - play_audio(bytes) - end -end -``` -""" -function request(method, url, h=nothing, b=nobody; - headers=h, body=b, query=nothing, observelayers::Bool=false, kw...)::Response - return request(HTTP.stack(observelayers), method, url, headers, body, query; kw...) -end - -# layers are applied from left to right, i.e. the first layer is the outermost that is called first, which then calls into the second layer, etc. -const STREAM_LAYERS = [timeoutlayer, exceptionlayer] -const REQUEST_LAYERS = [redirectlayer, headerslayer, cookielayer, retrylayer] - -""" - Layer - -Abstract type to represent a client-side middleware that exists for documentation purposes. -A layer is any function of the form `f(::Handler) -> Handler`, where [`Handler`](@ref) is -a function of the form `f(::Request) -> Response`. Note that the `Handler` definition is from -the server-side documentation, and is "hard-coded" on the client side. It may also be apparent -that a `Layer` is the same as the [`Middleware`](@ref) interface from server-side, which is true, but -we define `Layer` to clarify the client-side distinction and its unique usage. Custom layers can be -deployed in one of two ways: - * [`HTTP.@client`](@ref): Create a custom "client" with shorthand verb definitions, but which - include custom layers; only these new verb methods will use the custom layers. - * [`HTTP.pushlayer!`](@ref)/[`HTTP.poplayer!`](@ref): Allows globally adding and removing - layers from the default HTTP.jl layer stack; *all* http requests will then use the custom layers - -### Quick Examples -```julia -module Auth - -using HTTP - -function auth_layer(handler) - # returns a `Handler` function; check for a custom keyword arg `authcreds` that - # a user would pass like `HTTP.get(...; authcreds=creds)`. - # We also accept trailing keyword args `kw...` and pass them along later. - return function(req; authcreds=nothing, kw...) - # only apply the auth layer if the user passed `authcreds` - if authcreds !== nothing - # we add a custom header with stringified auth creds - HTTP.setheader(req, "X-Auth-Creds" => string(authcreds)) - end - # pass the request along to the next layer by calling `auth_layer` arg `handler` - # also pass along the trailing keyword args `kw...` - return handler(req; kw...) - end -end - -# Create a new client with the auth layer added -HTTP.@client [auth_layer] - -end # module - -# Can now use custom client like: -Auth.get(url; authcreds=creds) # performs GET request with auth_layer layer included - -# Or can include layer globally in all HTTP.jl requests -HTTP.pushlayer!(Auth.auth_layer) - -# Now can use normal HTTP.jl methods and auth_layer will be included -HTTP.get(url; authcreds=creds) -``` -""" -abstract type Layer end - -""" - HTTP.pushlayer!(layer; request=true) - -Push a layer onto the stack of layers that will be applied to all requests. -The "layer" is expected to be a function that takes and returns a `Handler` function. -See [`Layer`](@ref) for more details. -If `request=false`, the layer is expected to take and return a "stream" handler function. -The custom `layer` will be put on the top of the stack, so it will be the first layer -executed. To add a layer at the bottom of the stack, see [`HTTP.pushfirstlayer!`](@ref). -""" -pushlayer!(layer; request::Bool=true) = push!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) - -""" - HTTP.pushfirstlayer!(layer; request=true) - -Push a layer to the start of the stack of layers that will be applied to all requests. -The "layer" is expected to be a function that takes and returns a `Handler` function. -See [`Layer`](@ref) for more details. -If `request=false`, the layer is expected to take and return a "stream" handler function. -The custom `layer` will be put on the bottom of the stack, so it will be the last layer -executed. To add a layer at the top of the stack, see [`HTTP.pushlayer!`](@ref). -""" -pushfirstlayer!(layer; request::Bool=true) = pushfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) - -""" - HTTP.poplayer!(; request=true) - -Inverse of [`HTTP.pushlayer!`](@ref), removes the top layer of the global HTTP.jl layer stack. -Can be used to "cleanup" after a custom layer has been added. -If `request=false`, will remove the top "stream" layer as opposed to top "request" layer. -""" -poplayer!(; request::Bool=true) = pop!(request ? REQUEST_LAYERS : STREAM_LAYERS) - -""" - HTTP.popfirstlayer!(; request=true) - -Inverse of [`HTTP.pushfirstlayer!`](@ref), removes the bottom layer of the global HTTP.jl layer stack. -Can be used to "cleanup" after a custom layer has been added. -If `request=false`, will remove the bottom "stream" layer as opposed to bottom "request" layer. -""" -popfirstlayer!(; request::Bool=true) = popfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS) - -function stack( - observelayers::Bool=false, - # custom layers - requestlayers=(), - streamlayers=()) - - obs = observelayers ? observelayer : identity - # stream layers - if streamlayers isa NamedTuple - inner_stream_layers = haskey(streamlayers, :last) ? streamlayers.last : () - outer_stream_layers = haskey(streamlayers, :first) ? streamlayers.first : () - else - inner_stream_layers = streamlayers - outer_stream_layers = () - end - layers = foldr((x, y) -> obs(x(y)), inner_stream_layers, init=obs(streamlayer)) - layers2 = foldr((x, y) -> obs(x(y)), STREAM_LAYERS, init=layers) - if !isempty(outer_stream_layers) - layers2 = foldr((x, y) -> obs(x(y)), outer_stream_layers, init=layers2) - end - # request layers - # messagelayer must be the 1st/outermost layer to convert initial args to Request - if requestlayers isa NamedTuple - inner_request_layers = haskey(requestlayers, :last) ? requestlayers.last : () - outer_request_layers = haskey(requestlayers, :first) ? requestlayers.first : () +include("sniff.jl"); using .Sniff +include("forms.jl"); using .Forms +include("requestresponse.jl") +include("cookies.jl"); using .Cookies +include("client/redirects.jl") +include("client/client.jl") +include("client/retry.jl") +include("client/connection.jl") +include("client/request.jl") +include("client/stream.jl") +include("client/makerequest.jl") +include("websockets.jl"); using .WebSockets +include("server.jl") +include("handlers.jl"); using .Handlers +include("statuses.jl") + +struct StatusError <: Exception + request_method::String + request_uri::aws_uri + response::Response +end + +function Base.showerror(io::IO, e::StatusError) + println(io, "HTTP.StatusError:") + println(io, " Request method: $(e.request_method)") + println(io, " Request URI: $(makeuri(e.request_uri))") + println(io, " response:") + print_response(io, e.response) + return +end + +# backwards compatibility +function Base.getproperty(e::StatusError, s::Symbol) + if s == :status + return e.response.status + elseif s == :method + return e.request.method + elseif s == :target + return e.request.target else - inner_request_layers = requestlayers - outer_request_layers = () + return getfield(e, s) end - layers3 = foldr((x, y) -> obs(x(y)), inner_request_layers; init=obs(connectionlayer(layers2))) - layers4 = foldr((x, y) -> obs(x(y)), REQUEST_LAYERS; init=layers3) - if !isempty(outer_request_layers) - layers4 = foldr((x, y) -> obs(x(y)), outer_request_layers, init=layers4) - end - return messagelayer(layers4) end -function request(stack::Base.Callable, method, url, h=nothing, b=nobody, q=nothing; - headers=h, body=b, query=q, kw...)::Response - return stack(string(method), request_uri(url, query), headers, body; kw...) -end - -macro remove_linenums!(expr) - return esc(Base.remove_linenums!(expr)) -end - -""" - HTTP.@client requestlayers - HTTP.@client requestlayers streamlayers - HTTP.@client (first=requestlayers, last=requestlayers) (first=streamlayers, last=streamlayers) - -Convenience macro for creating a custom HTTP.jl client that will include custom layers when -performing requests. It's common to want to define a custom [`Layer`](@ref) to enhance a -specific category of requests, such as custom authentcation for a web API. Instead of affecting -the global HTTP.jl request stack via [`HTTP.pushlayer!`](@ref), a custom wrapper client can be -defined with convenient shorthand methods. See [`Layer`](@ref) for an example of defining a custom -layer and creating a new client that includes the layer. - -Custom layer arguments can be provided as a collection of request or stream-based layers; alternatively, -a NamedTuple with keys `first` and `last` can be provided with values being a collection of layers. -The NamedTuple form provides finer control over the order in which the layers will be included in the default -http layer stack: `first` request layers are executed before all other layers, `last` request layers -are executed right before all stream layers, and similarly for stream layers. - -An empty collection can always be passed for request or stream layers when not needed. - -One use case for custom clients is to control the value of standard `HTTP.request` keyword arguments. -This can be achieved by passing a `(first=[defaultkeywordlayer],)` where `defaultkeywordlayer` is defined -like: - -```julia -defaultkeywordlayer(handler) = (req; kw...) -> handler(req; retry=false, redirect=false, kw...) -``` - -This client-side layer is basically a no-op as it doesn't modify the request at all, except that it -hard-codes the value of the `retry` and `redirect` keyword arguments. When we pass this layer as -`(first=[defaultkeywordlayer],)` this ensures this layer will be executed before all other layers, -effectively over-writing the default and any user-provided keyword arguments for `retry` or `redirect`. -""" -macro client(requestlayers, streamlayers=[]) - return @remove_linenums! esc(quote - get(a...; kw...) = ($__source__; request("GET", a...; kw...)) - put(a...; kw...) = ($__source__; request("PUT", a...; kw...)) - post(a...; kw...) = ($__source__; request("POST", a...; kw...)) - patch(a...; kw...) = ($__source__; request("PATCH", a...; kw...)) - head(a...; kw...) = ($__source__; request("HEAD", a...; kw...)) - delete(a...; kw...) = ($__source__; request("DELETE", a...; kw...)) - open(f, a...; kw...) = ($__source__; request(a...; iofunction=f, kw...)) - function request(method, url, h=HTTP.Header[], b=HTTP.nobody; headers=h, body=b, query=nothing, observelayers::Bool=false, kw...)::HTTP.Response - $__source__ - HTTP.request(HTTP.stack(observelayers, $requestlayers, $streamlayers), method, url, headers, body, query; kw...) - end - end) -end - -""" - HTTP.get(url [, headers]; ) -> HTTP.Response - -Shorthand for `HTTP.request("GET", ...)`. See [`HTTP.request`](@ref). -""" -get(a...; kw...) = request("GET", a...; kw...) - -""" - HTTP.put(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("PUT", ...)`. See [`HTTP.request`](@ref). -""" -put(a...; kw...) = request("PUT", a...; kw...) - -""" - HTTP.post(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("POST", ...)`. See [`HTTP.request`](@ref). -""" -post(a...; kw...) = request("POST", a...; kw...) - -""" - HTTP.patch(url, headers, body; ) -> HTTP.Response - -Shorthand for `HTTP.request("PATCH", ...)`. See [`HTTP.request`](@ref). -""" -patch(a...; kw...) = request("PATCH", a...; kw...) - -""" - HTTP.head(url; ) -> HTTP.Response - -Shorthand for `HTTP.request("HEAD", ...)`. See [`HTTP.request`](@ref). -""" -head(u; kw...) = request("HEAD", u; kw...) - -""" - HTTP.delete(url [, headers]; ) -> HTTP.Response - -Shorthand for `HTTP.request("DELETE", ...)`. See [`HTTP.request`](@ref). -""" -delete(a...; kw...) = request("DELETE", a...; kw...) - -request_uri(url, query) = URI(URI(url); query=query) -request_uri(url, ::Nothing) = URI(url) - -""" - HTTP.open(method, url, [,headers]) do io - write(io, body) - [startread(io) -> HTTP.Response] - while !eof(io) - readavailable(io) -> AbstractVector{UInt8} +#NOTE: this is global process logging in the aws-crt libraries; not appropriate for request-level +# logging, but more for debugging the library itself +mutable struct AwsLogger + ptr::Ptr{aws_logger} + file_ref::Libc.FILE + options::aws_logger_standard_options + function AwsLogger(level::Integer, allocator::Ptr{aws_allocator}) + fr = Libc.FILE(Libc.RawFD(1), "w") + opts = aws_logger_standard_options(aws_log_level(0), C_NULL, Ptr{Libc.FILE}(fr.ptr)) + x = new(Ptr{aws_logger}(aws_mem_acquire(allocator, 64)), fr, opts) + aws_logger_init_standard(x.ptr, allocator, FieldRef(x, :options)) != 0 && aws_throw_error() + aws_logger_set(x.ptr) + return finalizer(x) do x + aws_logger_clean_up(x.ptr) + aws_mem_release(allocator, x.ptr) end - end -> HTTP.Response - -The `HTTP.open` API allows the request body to be written to (and/or the -response body to be read from) an `IO` stream. - -e.g. Streaming an audio file to the `vlc` player: -```julia -HTTP.open(:GET, "https://tinyurl.com/bach-cello-suite-1-ogg") do http - open(`vlc -q --play-and-exit --intf dummy -`, "w") do vlc - write(vlc, http) end end -``` -""" -open(f::Function, method::Union{String,Symbol}, url, headers=Header[]; kw...)::Response = - request(string(method), url, headers, nothing; iofunction=f, kw...) - -""" - HTTP.openraw(method, url, [, headers])::Tuple{Connection, Response} - -Open a raw socket that is unmanaged by HTTP.jl. Useful for doing HTTP upgrades -to other protocols. Any bytes of the body read from the socket when reading -headers, is returned as excess bytes in the last tuple argument. - -Example of a WebSocket upgrade: -```julia -headers = Dict( - "Upgrade" => "websocket", - "Connection" => "Upgrade", - "Sec-WebSocket-Key" => "dGhlIHNhbXBsZSBub25jZQ==", - "Sec-WebSocket-Version" => "13") -socket, response, excess = HTTP.openraw("GET", "ws://echo.websocket.org", headers) - -# Write a WebSocket frame -frame = UInt8[0x81, 0x85, 0x37, 0xfa, 0x21, 0x3d, 0x7f, 0x9f, 0x4d, 0x51, 0x58] -write(socket, frame) -``` -""" -function openraw(method::Union{String,Symbol}, url, headers=Header[]; kw...)::Tuple{IO, Response} - socketready = Channel{Tuple{IO, Response}}(0) - Threads.@spawn HTTP.open(method, url, headers; kw...) do http - HTTP.startread(http) - socket = http.stream - put!(socketready, (socket, http.message)) - while(isopen(socket)) - Base.wait_close(socket) - end - end - take!(socketready) -end - -""" - parse(Request, str) - parse(Response, str) - -Parse a string into a `Request` or `Response` object. -""" -function Base.parse(::Type{T}, str::AbstractString)::T where T <: Message - buffer = Base.BufferStream() - write(buffer, str) - close(buffer) - m = T() - http = Stream(m, Connection(buffer)) - m.body = read(http) - closeread(http) - return m +const LOGGER = Ref{AwsLogger}() + +function set_log_level!(level::Integer, allocator::Ptr{aws_allocator}=default_aws_allocator()) + @assert 0 <= level <= 7 "log level must be between 0 and 7" + LOGGER[] = AwsLogger(level, allocator) + @assert aws_logger_set_log_level(LOGGER[].ptr, aws_log_level(level)) == 0 + return +end + +function __init__() + allocator = default_aws_allocator() + LibAwsHTTPFork.init(allocator) + # intialize c functions + on_acquired[] = @cfunction(c_on_acquired, Cvoid, (Ptr{Cvoid}, Cint, Ptr{aws_retry_token}, Ptr{Cvoid})) + # on_shutdown[] = @cfunction(c_on_shutdown, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid})) + on_setup[] = @cfunction(c_on_setup, Cvoid, (Ptr{aws_http_connection}, Cint, Ptr{Cvoid})) + on_stream_write_on_complete[] = @cfunction(c_on_stream_write_on_complete, Cvoid, (Ptr{aws_http_stream}, Cint, Ptr{Cvoid})) + on_response_headers[] = @cfunction(c_on_response_headers, Cint, (Ptr{Cvoid}, Cint, Ptr{aws_http_header}, Csize_t, Ptr{Cvoid})) + on_response_header_block_done[] = @cfunction(c_on_response_header_block_done, Cint, (Ptr{Cvoid}, Cint, Ptr{Cvoid})) + on_response_body[] = @cfunction(c_on_response_body, Cint, (Ptr{Cvoid}, Ptr{aws_byte_cursor}, Ptr{Cvoid})) + on_metrics[] = @cfunction(c_on_metrics, Cvoid, (Ptr{Cvoid}, Ptr{aws_http_stream_metrics}, Ptr{Cvoid})) + on_complete[] = @cfunction(c_on_complete, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid})) + on_destroy[] = @cfunction(c_on_destroy, Cvoid, (Ptr{Cvoid},)) + retry_ready[] = @cfunction(c_retry_ready, Cvoid, (Ptr{aws_retry_token}, Cint, Ptr{Cvoid})) + on_incoming_connection[] = @cfunction(c_on_incoming_connection, Cvoid, (Ptr{Cvoid}, Ptr{aws_http_connection}, Cint, Ptr{Cvoid})) + on_connection_shutdown[] = @cfunction(c_on_connection_shutdown, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid})) + on_incoming_request[] = @cfunction(c_on_incoming_request, Ptr{aws_http_stream}, (Ptr{aws_http_connection}, Ptr{Cvoid})) + on_request_headers[] = @cfunction(c_on_request_headers, Cint, (Ptr{aws_http_stream}, Ptr{aws_http_header_block}, Ptr{aws_http_header}, Csize_t, Ptr{Cvoid})) + on_request_header_block_done[] = @cfunction(c_on_request_header_block_done, Cint, (Ptr{aws_http_stream}, Ptr{aws_http_header_block}, Ptr{Cvoid})) + on_request_body[] = @cfunction(c_on_request_body, Cint, (Ptr{aws_http_stream}, Ptr{aws_byte_cursor}, Ptr{Cvoid})) + on_request_done[] = @cfunction(c_on_request_done, Cint, (Ptr{aws_http_stream}, Ptr{Cvoid})) + on_server_stream_complete[] = @cfunction(c_on_server_stream_complete, Cint, (Ptr{aws_http_connection}, Cint, Ptr{Cvoid})) + on_destroy_complete[] = @cfunction(c_on_destroy_complete, Cvoid, (Ptr{Cvoid},)) + return end # only run if precompiling @@ -643,4 +119,4 @@ if VERSION >= v"1.9.0-0" && ccall(:jl_generating_output, Cint, ()) == 1 do_precompile && include("precompile.jl") end -end # module +end diff --git a/src/IOExtras.jl b/src/IOExtras.jl deleted file mode 100644 index 97ea35949..000000000 --- a/src/IOExtras.jl +++ /dev/null @@ -1,127 +0,0 @@ -""" - IOExtras - -This module defines extensions to the `Base.IO` interface to support: - - `startwrite`, `closewrite`, `startread` and `closeread` for streams - with transactional semantics. -""" -module IOExtras - -using Sockets -using MbedTLS: SSLContext, MbedException -using OpenSSL: SSLStream - -export bytes, isbytes, nbytes, nobytes, - startwrite, closewrite, startread, closeread, readuntil, - tcpsocket, localport, safe_getpeername - -""" - bytes(x) - -If `x` is "castable" to an `AbstractVector{UInt8}`, then an -`AbstractVector{UInt8}` is returned; otherwise `x` is returned. -""" -function bytes end -bytes(s::AbstractVector{UInt8}) = s -bytes(s::AbstractString) = codeunits(s) -bytes(x) = x - -"""whether `x` is "castable" to an `AbstractVector{UInt8}`; i.e. you can call `bytes(x)` if `isbytes(x)` === true""" -isbytes(x) = x isa AbstractVector{UInt8} || x isa AbstractString - -""" - nbytes(x) -> Int - -Length in bytes of `x` if `x` is `isbytes(x)`. -""" -function nbytes end -nbytes(x) = nothing -nbytes(x::AbstractVector{UInt8}) = length(x) -nbytes(x::AbstractString) = sizeof(x) -nbytes(x::Vector{T}) where T <: AbstractString = sum(sizeof, x) -nbytes(x::Vector{T}) where T <: AbstractVector{UInt8} = sum(length, x) -nbytes(x::IOBuffer) = bytesavailable(x) -nbytes(x::Vector{IOBuffer}) = sum(bytesavailable, x) - -_doc = """ - startwrite(::IO) - closewrite(::IO) - startread(::IO) - closeread(::IO) - -Signal start/end of write or read operations. -""" -@static if isdefined(Base, :startwrite) - "$_doc" - Base.startwrite(io) = nothing -else - "$_doc" - startwrite(io) = nothing -end - -@static if isdefined(Base, :closewrite) - "$_doc" - Base.closewrite(io) = nothing -else - "$_doc" - closewrite(io) = nothing -end - -@static if isdefined(Base, :startread) - "$_doc" - Base.startread(io) = nothing -else - "$_doc" - startread(io) = nothing -end - -@static if isdefined(Base, :closeread) - "$_doc" - Base.closeread(io) = nothing -else - "$_doc" - closeread(io) = nothing -end - -tcpsocket(io::SSLContext)::TCPSocket = io.bio -tcpsocket(io::SSLStream)::TCPSocket = io.io -tcpsocket(io::TCPSocket)::TCPSocket = io - -localport(io) = try !isopen(tcpsocket(io)) ? 0 : - Sockets.getsockname(tcpsocket(io))[2] - catch - 0 - end - -function safe_getpeername(io) - try - if isopen(tcpsocket(io)) - return Sockets.getpeername(tcpsocket(io)) - end - catch - end - return IPv4(0), UInt16(0) -end - - -const nobytes = view(UInt8[], 1:0) - -readuntil(args...) = Base.readuntil(args...) - -""" -Read from an `IO` stream until `find_delimiter(bytes)` returns non-zero. -Return view of bytes up to the delimiter. -""" -function readuntil(buf::IOBuffer, - find_delimiter::F #= Vector{UInt8} -> Int =# - ) where {F <: Function} - l = find_delimiter(view(buf.data, buf.ptr:buf.size)) - if l == 0 - return nobytes - end - bytes = buf.data[buf.ptr:buf.ptr + l - 1] - buf.ptr += l - return bytes -end - -end diff --git a/src/Messages.jl b/src/Messages.jl deleted file mode 100644 index 88ad23409..000000000 --- a/src/Messages.jl +++ /dev/null @@ -1,640 +0,0 @@ -""" -The `Messages` module defines structs that represent [`HTTP.Request`](@ref) -and [`HTTP.Response`](@ref) Messages. - -The `Response` struct has a `request` field that points to the corresponding -`Request`; and the `Request` struct has a `response` field. -The `Request` struct also has a `parent` field that points to a `Response` -in the case of HTTP redirects that occur and are followed. - -The Messages module defines `IO` `read` and `write` methods for Messages -but it does not deal with URIs, creating connections, or executing requests. - -The `read` methods throw `EOFError` exceptions if input data is incomplete. -and call parser functions that may throw `HTTP.ParsingError` exceptions. -The `read` and `write` methods may also result in low level `IO` exceptions. - -### Sending Messages - -Messages are formatted and written to an `IO` stream by -[`Base.write(::IO,::HTTP.Messages.Message)`](@ref) and/or -[`HTTP.Messages.writeheaders`](@ref). - -### Receiving Messages - -Messages are parsed from `IO` stream data by -[`HTTP.Messages.readheaders`](@ref). -This function calls [`HTTP.Parsers.parse_header_field`](@ref) and passes each -header-field to [`HTTP.Messages.appendheader`](@ref). - -### Headers - -Headers are represented by `Vector{Pair{String,String}}`. As compared to -`Dict{String,String}` this allows [repeated header fields and preservation of -order](https://tools.ietf.org/html/rfc7230#section-3.2.2). - -Header values can be accessed by name using -[`HTTP.header`](@ref) and -[`HTTP.setheader`](@ref) (case-insensitive). - -The [`HTTP.appendheader`](@ref) function handles combining -multi-line values, repeated header fields and special handling of -multiple `Set-Cookie` headers. - -### Bodies - -The `HTTP.Message` structs represent the message body by default as `Vector{UInt8}`. -If `IO` or iterator objects are passed as the body, they will be stored as is -in the `Request`/`Response` `body` field. -""" -module Messages - -export Message, Request, Response, - reset!, status, method, headers, uri, body, resource, - iserror, isredirect, retryablebody, retryable, retrylimitreached, ischunked, issafe, isidempotent, - header, hasheader, headercontains, setheader, defaultheader!, appendheader, - removeheader, mkheaders, readheaders, headerscomplete, - readchunksize, - writeheaders, writestartline, - bodylength, unknown_length, - payload, decode, sprintcompact - -using URIs, CodecZlib -using ..Pairs, ..IOExtras, ..Parsers, ..Strings, ..Forms, ..Conditions -using ..Connections, ..StatusCodes - -const nobody = UInt8[] -const unknown_length = typemax(Int) - -sprintcompact(x) = sprint(show, x; context=:compact => true) - -abstract type Message end - -# HTTP Response -""" - HTTP.Response(status, headers::HTTP.Headers, body; request=nothing) - HTTP.Response(status, body) - HTTP.Response(body) - -Represents an HTTP response message with fields: - -- `version::HTTPVersion` - [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) - -- `status::Int16` - [RFC7230 3.1.2](https://tools.ietf.org/html/rfc7230#section-3.1.2) - [RFC7231 6](https://tools.ietf.org/html/rfc7231#section-6) - -- `headers::Vector{Pair{String,String}}` - [RFC7230 3.2](https://tools.ietf.org/html/rfc7230#section-3.2) - -- `body::Vector{UInt8}` or `body::IO` - [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) - -- `request`, the `Request` that yielded this `Response`. - -""" -mutable struct Response <: Message - version::HTTPVersion - status::Int16 - headers::Headers - body::Any # Usually Vector{UInt8} or IO - request::Union{Message, Nothing} # Union{Request, Nothing} -end - -function Response(status::Integer, headers, body; version=HTTPVersion(1, 1), request=nothing) - b = isbytes(body) ? bytes(body) : something(body, nobody) - @assert (request isa Request || request === nothing) - return Response(version, status, mkheaders(headers), b, request) -end - -# legacy constructor -Response(status::Integer, headers=[]; body=nobody, request=nothing) = - Response(status, headers, body; request) - -Response() = Request().response -Response(s::Int, body::AbstractVector{UInt8}) = Response(s; body=body) -Response(s::Int, body::AbstractString) = Response(s; body=bytes(body)) -Response(body) = Response(200; body=body) - -Base.convert(::Type{Response}, s::AbstractString) = Response(s) - -function reset!(r::Response) - r.version = HTTPVersion(1, 1) - r.status = 0 - if !isempty(r.headers) - empty!(r.headers) - end - delete!(r.request.context, :response_body) - return -end - -status(r::Response) = getfield(r, :status) -headers(r::Response) = getfield(r, :headers) -body(r::Response) = getfield(r, :body) - -# HTTP Request -const Context = Dict{Symbol, Any} - -""" - HTTP.Request( - method, target, headers=[], body=nobody; - version=v"1.1", url::URI=URI(), responsebody=nothing, parent=nothing, context=HTTP.Context() - ) - -Represents a HTTP Request Message with fields: - -- `method::String` - [RFC7230 3.1.1](https://tools.ietf.org/html/rfc7230#section-3.1.1) - -- `target::String` - [RFC7230 5.3](https://tools.ietf.org/html/rfc7230#section-5.3) - -- `version::HTTPVersion` - [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) - -- `headers::HTTP.Headers` - [RFC7230 3.2](https://tools.ietf.org/html/rfc7230#section-3.2) - -- `body::Union{Vector{UInt8}, IO}` - [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) - -- `response`, the `Response` to this `Request` - -- `url::URI`, the full URI of the request - -- `parent`, the `Response` (if any) that led to this request - (e.g. in the case of a redirect). - [RFC7230 6.4](https://tools.ietf.org/html/rfc7231#section-6.4) - -- `context`, a `Dict{Symbol, Any}` store used by middleware to share state - -""" -mutable struct Request <: Message - method::String - target::String - version::HTTPVersion - headers::Headers - body::Any # Usually Vector{UInt8} or some kind of IO - response::Response - url::URI - parent::Union{Response, Nothing} - context::Context -end - -Request() = Request("", "") - -function Request( - method::String, target, headers=[], body=nobody; - version=HTTPVersion(1, 1), url::URI=URI(), responsebody=nothing, parent=nothing, context=Context() -) - b = isbytes(body) ? bytes(body) : body - r = Request(method, target == "" ? "/" : target, version, - mkheaders(headers), b, Response(0; body=responsebody), - url, parent, context) - r.response.request = r - return r -end - -""" -"request-target" per https://tools.ietf.org/html/rfc7230#section-5.3 -""" -resource(uri::URI) = string( isempty(uri.path) ? "/" : uri.path, - !isempty(uri.query) ? "?" : "", uri.query, - !isempty(uri.fragment) ? "#" : "", uri.fragment) - -mkheaders(h::Headers) = h -function mkheaders(h, headers=Vector{Header}(undef, length(h)))::Headers - # validation - for (i, head) in enumerate(h) - head isa String && throw(ArgumentError("header must be passed as key => value pair: `$head`")) - length(head) != 2 && throw(ArgumentError("invalid header key-value pair: $head")) - headers[i] = SubString(string(head[1])) => SubString(string(head[2])) - end - return headers -end - -method(r::Request) = getfield(r, :method) -target(r::Request) = getfield(r, :target) -url(r::Request) = getfield(r, :url) -headers(r::Request) = getfield(r, :headers) -body(r::Request) = getfield(r, :body) - -# HTTP Message state and type queries -""" - issafe(::Request) - -https://tools.ietf.org/html/rfc7231#section-4.2.1 -""" -issafe(r::Request) = issafe(r.method) -issafe(method) = method in ["GET", "HEAD", "OPTIONS", "TRACE"] - -""" - iserror(::Response) - -Does this `Response` have an error status? -""" -iserror(r::Response) = iserror(r.status) -iserror(status::Integer) = status != 0 && status != 100 && status != 101 && - (status < 200 || status >= 300) && !isredirect(status) - -""" - isredirect(::Response) - -Does this `Response` have a redirect status? -""" -isredirect(r::Response) = isredirect(r.status) -isredirect(r::Request) = allow_redirects(r) && !redirectlimitreached(r) -isredirect(status::Integer) = status in (301, 302, 303, 307, 308) - -# whether the redirect limit has been reached for a given request -# set in the RedirectRequest layer once the limit is reached -redirectlimitreached(r::Request) = get(r.context, :redirectlimitreached, false) -allow_redirects(r::Request) = get(r.context, :allow_redirects, false) - -""" - isidempotent(::Request) - -https://tools.ietf.org/html/rfc7231#section-4.2.2 -""" -isidempotent(r::Request) = isidempotent(r.method) -isidempotent(method) = issafe(method) || method in ["PUT", "DELETE"] -retry_non_idempotent(r::Request) = get(r.context, :retry_non_idempotent, false) -allow_retries(r::Request) = get(r.context, :allow_retries, false) -nothing_written(r::Request) = get(r.context, :nothingwritten, false) - -# whether the retry limit has been reached for a given request -# set in the RetryRequest layer once the limit is reached -retrylimitreached(r::Request) = get(r.context, :retrylimitreached, false) - -""" - retryable(::Request) - -Whether a `Request` is eligible to be retried. -""" -function retryable end - -supportsmark(x) = false -supportsmark(x::T) where {T <: IO} = length(Base.methods(mark, Tuple{T}, parentmodule(T))) > 0 || hasfield(T, :mark) - -# request body is retryable if it was provided as "bytes", an AbstractDict or NamedTuple, -# or a chunked array of "bytes"; OR if it supports mark() and is marked -retryablebody(r::Request) = (isbytes(r.body) || r.body isa Union{AbstractDict, NamedTuple} || - (r.body isa Vector && all(isbytes, r.body)) || (supportsmark(r.body) && ismarked(r.body))) - -# request is retryable if the body is retryable, the user is allowing retries at all, -# we haven't reached the retry limit, and either nothing has been written yet or -# the request is idempotent or the user has explicitly allowed non-idempotent retries -retryable(r::Request) = retryablebody(r) && - allow_retries(r) && !retrylimitreached(r) && - (nothing_written(r) || isidempotent(r) || retry_non_idempotent(r)) -retryable(r::Response) = retryable(r.status) -retryable(status) = status in (403, 408, 409, 429, 500, 502, 503, 504, 599) - -""" - ischunked(::Message) - -Does the `Message` have a "Transfer-Encoding: chunked" header? -""" -ischunked(m) = any(h->(field_name_isequal(h[1], "transfer-encoding") && - endswith(lowercase(h[2]), "chunked")), - m.headers) - -""" - headerscomplete(::Message) - -Have the headers been read into this `Message`? -""" -headerscomplete(r::Response) = r.status != 0 && r.status != 100 -headerscomplete(r::Request) = r.method != "" - -""" -"The presence of a message body in a response depends on both the - request method to which it is responding and the response status code. - Responses to the HEAD request method never include a message body []. - 2xx (Successful) responses to a CONNECT request method (Section 4.3.6 of - [RFC7231]) switch to tunnel mode instead of having a message body. - All 1xx (Informational), 204 (No Content), and 304 (Not Modified) - responses do not include a message body. All other responses do - include a message body, although the body might be of zero length." -[RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) -""" -bodylength(r::Response)::Int = - r.request.method == "HEAD" ? 0 : - ischunked(r) ? unknown_length : - r.status in [204, 304] ? 0 : - (l = header(r, "Content-Length")) != "" ? parse(Int, l) : - unknown_length - -""" -"The presence of a message body in a request is signaled by a - Content-Length or Transfer-Encoding header field. Request message - framing is independent of method semantics, even if the method does - not define any use for a message body." -[RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) -""" -bodylength(r::Request)::Int = - ischunked(r) ? unknown_length : - parse(Int, header(r, "Content-Length", "0")) - -# HTTP header-fields -Base.getindex(m::Message, k) = header(m, k) - -""" - Are `field-name`s `a` and `b` equal? - -[HTTP `field-name`s](https://tools.ietf.org/html/rfc7230#section-3.2) -are ASCII-only and case-insensitive. -""" -field_name_isequal(a, b) = ascii_lc_isequal(a, b) - -""" - HTTP.header(::Message, key [, default=""]) -> String - -Get header value for `key` (case-insensitive). -""" -header(m::Message, k, d="") = header(m.headers, k, d) -header(h::Headers, k::AbstractString, d="") = - getbyfirst(h, k, k => d, field_name_isequal)[2] - -""" - HTTP.headers(m::Message, key) -> Vector{String} - -Get all headers with key `k` or empty if none -""" -headers(h::Headers, k::AbstractString) = - map(x -> x[2], filter(x -> field_name_isequal(x[1], k), h)) -headers(m::Message, k::AbstractString) = - headers(headers(m), k) - -""" - HTTP.hasheader(::Message, key) -> Bool - -Does header value for `key` exist (case-insensitive)? -""" -hasheader(m, k::AbstractString) = header(m, k) != "" - -""" - HTTP.hasheader(::Message, key, value) -> Bool - -Does header for `key` match `value` (both case-insensitive)? -""" -hasheader(m, k::AbstractString, v::AbstractString) = - field_name_isequal(header(m, k), lowercase(v)) - -""" - HTTP.headercontains(::Message, key, value) -> Bool - -Does the header for `key` (interpreted as comma-separated list) contain `value` (both case-insensitive)? -""" -headercontains(m, k::AbstractString, v::AbstractString) = - any(field_name_isequal.(strip.(split(header(m, k), ",")), v)) - -""" - HTTP.setheader(::Message, key => value) - -Set header `value` for `key` (case-insensitive). -""" -setheader(m::Message, v) = setheader(m.headers, v) -setheader(h::Headers, v::Header) = setbyfirst(h, v, field_name_isequal) -setheader(h::Headers, v::Pair) = - setbyfirst(h, Header(SubString(v.first), SubString(v.second)), - field_name_isequal) - -""" - defaultheader!(::Message, key => value) - -Set header `value` in message for `key` if it is not already set. -""" -function defaultheader!(m, v::Pair) - # return nothing as default to allow users passing "" as empty header - # and not being overwritten by default headers - if header(m, first(v), nothing) === nothing - setheader(m, v) - end - return -end - -""" - HTTP.appendheader(::Message, key => value) - -Append a header value to `message.headers`. - -If `key` is the same as the previous header, the `value` is [appended to the -value of the previous header with a comma -delimiter](https://stackoverflow.com/a/24502264) - -`Set-Cookie` headers are not comma-combined because [cookies often contain -internal commas](https://tools.ietf.org/html/rfc6265#section-3). -""" -function appendheader(m::Message, header::Header) - c = m.headers - k,v = header - if k != "Set-Cookie" && length(c) > 0 && k == c[end][1] - c[end] = c[end][1] => string(c[end][2], ", ", v) - else - push!(m.headers, header) - end - return -end - -""" - HTTP.removeheader(::Message, key) - -Remove header for `key` (case-insensitive). -""" -removeheader(m::Message, header::String) = rmkv(m.headers, header, field_name_isequal) - -# HTTP payload body -function payload(m::Message)::Vector{UInt8} - enc = lowercase(first(split(header(m, "Transfer-Encoding"), ", "))) - return enc in ["", "identity", "chunked"] ? m.body : decode(m, enc) -end - -payload(m::Message, ::Type{String}) = - hasheader(m, "Content-Type", "ISO-8859-1") ? iso8859_1_to_utf8(payload(m)) : - String(payload(m)) - -""" - HTTP.decode(r::Union{Request, Response}) -> Vector{UInt8} - -For a gzip encoded request/response body, decompress it and return -the decompressed body. -""" -function decode(m::Message, encoding::String="gzip")::Vector{UInt8} - if encoding == "gzip" - return transcode(GzipDecompressor, m.body) - end - return m.body -end - -# Writing HTTP Messages to IO streams -Base.write(io::IO, v::HTTPVersion) = write(io, "HTTP/", string(v.major), ".", string(v.minor)) - -""" - writestartline(::IO, ::Message) - -e.g. `"GET /path HTTP/1.1\\r\\n"` or `"HTTP/1.1 200 OK\\r\\n"` -""" -function writestartline(io::IO, r::Request) - return write(io, r.method, " ", r.target, " ", r.version, "\r\n") -end - -function writestartline(io::IO, r::Response) - return write(io, r.version, " ", string(r.status), " ", StatusCodes.statustext(r.status), "\r\n") -end - -""" - writeheaders(::IO, ::Message) - -Write `Message` start line and -a line for each "name: value" pair and a trailing blank line. -""" -writeheaders(io::IO, m::Message) = writeheaders(io, m, IOBuffer()) -writeheaders(io::Connection, m::Message) = writeheaders(io, m, io.writebuffer) - -function writeheaders(io::IO, m::Message, buf::IOBuffer) - writestartline(buf, m) - for (name, value) in m.headers - # match curl convention of not writing empty headers - !isempty(value) && write(buf, name, ": ", value, "\r\n") - end - write(buf, "\r\n") - nwritten = write(io, take!(buf)) - return nwritten -end - -""" - write(::IO, ::Message) - -Write start line, headers and body of HTTP Message. -""" -function Base.write(io::IO, m::Message) - nwritten = writeheaders(io, m) - nwritten += write(io, m.body) - return nwritten -end - -function Base.String(m::Message) - io = IOBuffer() - write(io, m) - return String(take!(io)) -end - -# Reading HTTP Messages from IO streams - -""" - readheaders(::IO, ::Message) - -Read headers (and startline) from an `IO` stream into a `Message` struct. -Throw `EOFError` if input is incomplete. -""" -function readheaders(io::IO, message::Message) - bytes = String(IOExtras.readuntil(io, find_end_of_header)) - bytes = parse_start_line!(bytes, message) - parse_header_fields!(bytes, message) - return -end - -parse_start_line!(bytes, r::Response) = parse_status_line!(bytes, r) - -parse_start_line!(bytes, r::Request) = parse_request_line!(bytes, r) - -function parse_header_fields!(bytes::SubString{String}, m::Message) - - h, bytes = parse_header_field(bytes) - while !(h === Parsers.emptyheader) - appendheader(m, h) - h, bytes = parse_header_field(bytes) - end - return -end - -""" -Read chunk-size from an `IO` stream. -After the final zero size chunk, read trailers into a `Message` struct. -""" -function readchunksize(io::IO, message::Message)::Int - n = parse_chunk_size(IOExtras.readuntil(io, find_end_of_chunk_size)) - if n == 0 - bytes = IOExtras.readuntil(io, find_end_of_trailer) - if bytes[2] != UInt8('\n') - parse_header_fields!(SubString(String(bytes)), message) - end - end - return n -end - -# Debug message printing - -""" - set_show_max(x) - -Set the maximum number of body bytes to be displayed by `show(::IO, ::Message)` -""" -set_show_max(x) = BODY_SHOW_MAX[] = x -const BODY_SHOW_MAX = Ref(1000) - -""" - bodysummary(bytes) - -The first chunk of the Message Body (for display purposes). -""" -bodysummary(body) = isbytes(body) ? view(bytes(body), 1:min(nbytes(body), BODY_SHOW_MAX[])) : "[Message Body was streamed]" -bodysummary(body::Union{AbstractDict, NamedTuple}) = URIs.escapeuri(body) -function bodysummary(body::Form) - if length(body.data) == 1 && isa(body.data[1], IOBuffer) - return body.data[1].data[1:body.data[1].ptr-1] - end - return "[Message Body was streamed]" -end - -function compactstartline(m::Message) - b = IOBuffer() - writestartline(b, m) - return strip(String(take!(b))) -end - -# temporary replacement for isvalid(String, s), until the -# latter supports subarrays (JuliaLang/julia#36047): -isvalidstr(s) = ccall(:u8_isvalid, Int32, (Ptr{UInt8}, Int), s, sizeof(s)) ≠ 0 - -function Base.show(io::IO, m::Message) - if get(io, :compact, false) - print(io, compactstartline(m)) - if m isa Response - print(io, " <= (", compactstartline(m.request::Request), ")") - end - return - end - println(io, typeof(m), ":") - println(io, "\"\"\"") - - # Mask the following (potentially) sensitive headers with "******": - # - Authorization - # - Proxy-Authorization - # - Cookie - # - Set-Cookie - header_str = sprint(writeheaders, m) - header_str = replace(header_str, r"(*CRLF)^((?>(?>Proxy-)?Authorization|(?>Set-)?Cookie): ).+$"mi => s"\1******") - write(io, header_str) - - summary = bodysummary(m.body) - validsummary = isvalidstr(summary) - validsummary && write(io, summary) - if !validsummary || something(nbytes(m.body), 0) > length(summary) - println(io, "\n⋮\n$(nbytes(m.body))-byte body") - end - print(io, "\"\"\"") - return -end - -function statustext(status) - Base.depwarn("`Messages.statustext` is deprecated, use `StatusCodes.statustext` instead.", :statustext) - return StatusCodes.statustext(status) -end - -URIs.queryparams(r::Request) = URIs.queryparams(URI(r.target)) -URIs.queryparams(r::Response) = isnothing(r.request) ? nothing : URIs.queryparams(r.request) - -end # module Messages diff --git a/src/Pairs.jl b/src/Pairs.jl deleted file mode 100644 index 105dabee9..000000000 --- a/src/Pairs.jl +++ /dev/null @@ -1,78 +0,0 @@ -module Pairs - -export defaultbyfirst, setbyfirst, getbyfirst, setkv, getkv, rmkv - -""" - setbyfirst(collection, item) -> item - -Set `item` in a `collection`. -If `first() of an exisiting item matches `first(item)` it is replaced. -Otherwise the new `item` is inserted at the end of the `collection`. -""" -function setbyfirst(c, item, eq = ==) - k = first(item) - i = findfirst(x->eq(first(x), k), c) - if i === nothing - push!(c, item) - else - c[i] = item - end - return item -end - -""" - getbyfirst(collection, key [, default]) -> item - -Get `item` from collection where `first(item)` matches `key`. -""" -function getbyfirst(c, k, default=nothing, eq = ==) - i = findfirst(x->eq(first(x), k), c) - return i === nothing ? default : c[i] -end - -""" - defaultbyfirst(collection, item) - -If `first(item)` does not match match `first()` of any existing items, -insert the new `item` at the end of the `collection`. -""" -function defaultbyfirst(c, item, eq = ==) - k = first(item) - if findfirst(x->eq(first(x), k), c) === nothing - push!(c, item) - end - return -end - -""" - setkv(collection, key, value) - -Set `value` for `key` in collection of key/value `Pairs`. -""" -setkv(c, k, v) = setbyfirst(c, k => v) - -""" - getkv(collection, key [, default]) -> value - -Get `value` for `key` in collection of key/value `Pairs`, -where `first(item) == key` and `value = item[2]` -""" -function getkv(c, k, default=nothing) - i = findfirst(x->first(x) == k, c) - return i === nothing ? default : c[i][2] -end - -""" - rmkv(collection, key) - -Remove `key` from `collection` of key/value `Pairs`. -""" -function rmkv(c, k, eq = ==) - i = findfirst(x->eq(first(x), k), c) - if !(i === nothing) - deleteat!(c, i) - end - return -end - -end # module Pairs diff --git a/src/Parsers.jl b/src/Parsers.jl deleted file mode 100644 index 2bbf1095f..000000000 --- a/src/Parsers.jl +++ /dev/null @@ -1,367 +0,0 @@ -""" -The parser separates a raw HTTP Message into its component parts. - -If the input data is invalid the Parser throws a `HTTP.ParseError`. - -The `parse_*` functions processes a single element of a HTTP Message at a time -and return a `SubString` containing the unused portion of the input. - -The Parser does not interpret the Message Headers. It is beyond the scope of the -Parser to deal with repeated header fields, multi-line values, cookies or case -normalization. - -The Parser has no knowledge of the high-level `Request` and `Response` structs -defined in `Messages.jl`. However, the `Request` and `Response` structs must -have field names compatible with those expected by the `parse_status_line!` and -`parse_request_line!` functions. -""" -module Parsers - -import ..access_threaded -using ..Strings - -export Header, Headers, - find_end_of_header, find_end_of_chunk_size, find_end_of_trailer, - parse_status_line!, parse_request_line!, parse_header_field, - parse_chunk_size, - ParseError - -include("parseutils.jl") - -const emptyss = SubString("",1,0) -const emptyheader = emptyss => emptyss -const Header = Pair{SubString{String},SubString{String}} -const Headers = Vector{Header} - -""" - ParseError <: Exception - -Parser input was invalid. - -Fields: - - `code`, error code - - `bytes`, the offending input. -""" -struct ParseError <: Exception - code::Symbol - bytes::SubString{String} -end - -ParseError(code::Symbol, bytes="") = - ParseError(code, first(split(String(bytes), '\n'))) - -# Regular expressions for parsing HTTP start-line and header-fields - -init!(r::RegexAndMatchData) = (Base.compile(r.re); initialize!(r); r) - -""" -https://tools.ietf.org/html/rfc7230#section-3.1.1 -request-line = method SP request-target SP HTTP-version CRLF -""" -const request_line_regex = RegexAndMatchData[] -function request_line_regex_f() - r = RegexAndMatchData(r"""^ - (?: \r? \n) ? # ignore leading blank line - ([!#$%&'*+\-.^_`|~[:alnum:]]+) [ ]+ # 1. method = token (RFC7230 3.2.6) - ([^.][^ \r\n]*) [ ]+ # 2. target - HTTP/(\d\.\d) # 3. version - \r? \n # CRLF - """x) - init!(r) -end - - -""" -https://tools.ietf.org/html/rfc7230#section-3.1.2 -status-line = HTTP-version SP status-code SP reason-phrase CRLF - -See: -[#190](https://github.com/JuliaWeb/HTTP.jl/issues/190#issuecomment-363314009) -""" -const status_line_regex = RegexAndMatchData[] -function status_line_regex_f() - r = RegexAndMatchData(r"""^ - [ ]? # Issue #190 - HTTP/(\d\.\d) [ ]+ # 1. version - (\d\d\d) .* # 2. status - \r? \n # CRLF - """x) - init!(r) -end - -""" -https://tools.ietf.org/html/rfc7230#section-3.2 -header-field = field-name ":" OWS field-value OWS -""" -const header_field_regex = RegexAndMatchData[] -function header_field_regex_f() - r = RegexAndMatchData(r"""^ - ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) - [ \t]* # OWS - ([^\r\n]*?) # 2. field-value - [ \t]* # OWS - \r? \n # CRLF - (?= [^ \t]) # no WS on next line - """x) - init!(r) -end - - -""" -https://tools.ietf.org/html/rfc7230#section-3.2.4 -obs-fold = CRLF 1*( SP / HTAB ) -""" -const obs_fold_header_field_regex = RegexAndMatchData[] -function obs_fold_header_field_regex_f() - r = RegexAndMatchData(r"""^ - ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) - [ \t]* # OWS - ([^\r\n]* # 2. field-value - (?: \r? \n [ \t] [^\r\n]*)*) # obs-fold - [ \t]* # OWS - \r? \n # CRLF - """x) - init!(r) -end - -const empty_header_field_regex = RegexAndMatchData[] -function empty_header_field_regex_f() - r = RegexAndMatchData(r"^ \r? \n"x) - init!(r) -end - - -# HTTP start-line and header-field parsing - -""" -Arbitrary limit to protect against denial of service attacks. -""" -const header_size_limit = Int(0x10000) - -""" - find_end_of_header(bytes) -> length or 0 - -Find length of header delimited by `\\r\\n\\r\\n` or `\\n\\n`. -""" -function find_end_of_header(bytes::AbstractVector{UInt8}; allow_obs_fold=true) - - buf = 0xFFFFFFFF - l = min(length(bytes), header_size_limit) - i = 1 - while i <= l - @inbounds x = bytes[i] - if x == 0x0D || x == 0x0A - buf = (buf << 8) | UInt32(x) - # "Although the line terminator for the start-line and header - # fields is the sequence CRLF, a recipient MAY recognize a single - # LF as a line terminator" - # [RFC7230 3.5](https://tools.ietf.org/html/rfc7230#section-3.5) - buf16 = buf & 0xFFFF - if buf == 0x0D0A0D0A || buf16 == 0x0A0A - return i - end - # "A server that receives an obs-fold ... MUST either reject the - # message by sending a 400 (Bad Request) ... or replace each - # received obs-fold with one or more SP octets..." - # [RFC7230 3.2.4](https://tools.ietf.org/html/rfc7230#section-3.2.4) - if !allow_obs_fold && (buf16 == 0x0A20 || buf16 == 0x0A09) - throw(ParseError(:HEADER_CONTAINS_OBS_FOLD, bytes)) - end - else - buf = 0xFFFFFFFF - end - i += 1 - end - if i > header_size_limit - throw(ParseError(:HEADER_SIZE_EXCEEDS_LIMIT)) - end - - return 0 -end - -""" -Parse HTTP request-line `bytes` and set the -`method`, `target` and `version` fields of `request`. -Return a `SubString` containing the header-field lines. -""" -function parse_request_line!(bytes::AbstractString, request)::SubString{String} - re = access_threaded(request_line_regex_f, request_line_regex) - if !exec(re, bytes) - throw(ParseError(:INVALID_REQUEST_LINE, bytes)) - end - request.method = group(1, re, bytes) - request.target = group(2, re, bytes) - request.version = HTTPVersion(group(3, re, bytes)) - return nextbytes(re, bytes) -end - -""" -Parse HTTP response-line `bytes` and set the -`status` and `version` fields of `response`. -Return a `SubString` containing the header-field lines. -""" -function parse_status_line!(bytes::AbstractString, response)::SubString{String} - re = access_threaded(status_line_regex_f, status_line_regex) - if !exec(re, bytes) - throw(ParseError(:INVALID_STATUS_LINE, bytes)) - end - response.version = HTTPVersion(group(1, re, bytes)) - response.status = parse(Int, group(2, re, bytes)) - return nextbytes(re, bytes) -end - -""" -Parse HTTP header-field. -Return `Pair(field-name => field-value)` and -a `SubString` containing the remaining header-field lines. -""" -function parse_header_field(bytes::SubString{String})::Tuple{Header,SubString{String}} - # https://github.com/JuliaWeb/HTTP.jl/issues/796 - # there may be certain scenarios where non-ascii characters are - # included (illegally) in the headers; curl warns on these - # "malformed headers" and ignores them. we attempt to re-encode - # these from latin-1 => utf-8 and then try to parse. - if !isvalid(bytes) - @warn "malformed HTTP header detected; attempting to re-encode from Latin-1 to UTF8" - bytes = SubString(iso8859_1_to_utf8(codeunits(bytes))) - end - - # First look for: field-name ":" field-value - re = access_threaded(header_field_regex_f, header_field_regex) - if exec(re, bytes) - return (group(1, re, bytes) => group(2, re, bytes)), - nextbytes(re, bytes) - end - - # Then check for empty termination line: - re = access_threaded(empty_header_field_regex_f, empty_header_field_regex) - if exec(re, bytes) - return emptyheader, nextbytes(re, bytes) - end - - # Finally look for obsolete line folding format: - re = access_threaded(obs_fold_header_field_regex_f, obs_fold_header_field_regex) - if exec(re, bytes) - unfold = SubString(replace(group(2, re, bytes), r"\r?\n"=>"")) - return (group(1, re, bytes) => unfold), nextbytes(re, bytes) - end - -@label error - throw(ParseError(:INVALID_HEADER_FIELD, bytes)) -end - -# HTTP Chunked Transfer Coding - -""" -Arbitrary limit to protect against denial of service attacks. -""" -const chunk_size_line_max = 64 - -const chunk_size_line_min = ncodeunits("0\r\n") - -@inline function skip_crlf(bytes, i=1) - if @inbounds bytes[i] == UInt('\r') - i += 1 - end - if @inbounds bytes[i] == UInt('\n') - i += 1 - end - return i -end - - -""" -Find `\\n` after chunk size in `bytes`. -""" -function find_end_of_chunk_size(bytes::AbstractVector{UInt8}) - l = length(bytes) - if l < chunk_size_line_min - return 0 - end - if l > chunk_size_line_max - l = chunk_size_line_max - end - i = skip_crlf(bytes) - while i <= l - if @inbounds bytes[i] == UInt('\n') - return i - end - i += 1 - end - return 0 -end - -""" - find_end_of_trailer(bytes) -> length or 0 - -Find length of trailer delimited by `\\r\\n\\r\\n` (or starting with `\\r\\n`). -[RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) -""" -find_end_of_trailer(bytes::AbstractVector{UInt8}) = - length(bytes) < 2 ? 0 : - bytes[2] == UInt8('\n') ? 2 : - find_end_of_header(bytes) - -""" -Arbitrary limit to protect against denial of service attacks. -""" -const chunk_size_limit = typemax(Int32) - -""" -Parse HTTP chunk-size. -Return number of bytes of chunk-data. - - chunk-size = 1*HEXDIG -[RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) -""" -function parse_chunk_size(bytes::AbstractVector{UInt8})::Int - - chunk_size = Int64(0) - i = skip_crlf(bytes) - - while true - x = Int64(unhex[@inbounds bytes[i]]) - if x == -1 - break - end - chunk_size = chunk_size * Int64(16) + x - if chunk_size > chunk_size_limit - throw(ParseError(:CHUNK_SIZE_EXCEEDS_LIMIT, bytes)) - end - i += 1 - end - if i > 1 - return Int(chunk_size) - end - - throw(ParseError(:INVALID_CHUNK_SIZE, bytes)) -end - -const unhex = Int8[ - -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 - ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 - ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 -] - -function __init__() - # FIXME Consider turing off `PCRE.UTF` in `Regex.compile_options` - # https://github.com/JuliaLang/julia/pull/26731#issuecomment-380676770 - nt = @static if isdefined(Base.Threads, :maxthreadid) - Threads.maxthreadid() - else - Threads.nthreads() - end - resize!(empty!(status_line_regex), nt) - resize!(empty!(request_line_regex), nt) - resize!(empty!(header_field_regex), nt) - resize!(empty!(obs_fold_header_field_regex), nt) - resize!(empty!(empty_header_field_regex), nt) - return -end - -end # module Parsers diff --git a/src/Servers.jl b/src/Servers.jl deleted file mode 100644 index 868431548..000000000 --- a/src/Servers.jl +++ /dev/null @@ -1,535 +0,0 @@ -""" -The `HTTP.Servers` module provides core HTTP server functionality. - -The main entry point is `HTTP.listen(f, host, port; kw...)` which takes -a `f(::HTTP.Stream)::Nothing` function argument, a `host`, a `port` and -optional keyword arguments. For full details, see `?HTTP.listen`. - -For server functionality operating on full requests, see the `?HTTP.serve` function. -""" -module Servers - -export listen, listen!, Server, forceclose, port - -using Sockets, Logging, LoggingExtras, MbedTLS, Dates -using MbedTLS: SSLContext, SSLConfig -using ..IOExtras, ..Streams, ..Messages, ..Parsers, ..Connections, ..Exceptions -import ..access_threaded, ..SOCKET_TYPE_TLS, ..@logfmt_str - -TRUE(x) = true -getinet(host::String, port::Integer) = Sockets.InetAddr(parse(IPAddr, host), port) -getinet(host::IPAddr, port::Integer) = Sockets.InetAddr(host, port) - -struct Listener{S <: Union{SSLConfig, Nothing}, I <: Base.IOServer} - addr::Sockets.InetAddr - hostname::String - hostport::String - ssl::S - server::I -end - -function Listener(server::Base.IOServer; sslconfig::Union{MbedTLS.SSLConfig, Nothing}=nothing, kw...) - host, port = getsockname(server) - addr = getinet(host, port) - return Listener(addr, string(host), string(port), sslconfig, server) -end - -supportsreuseaddr() = ccall(:jl_has_so_reuseport, Int32, ()) == 1 - -function Listener(addr::Sockets.InetAddr, host::String, port::String; - sslconfig::Union{MbedTLS.SSLConfig, Nothing}=nothing, - reuseaddr::Bool=false, - backlog::Integer=Sockets.BACKLOG_DEFAULT, - server::Union{Nothing, Base.IOServer}=nothing, # for backwards compat - listenany::Bool=false, - kw...) - if server !== nothing - return Listener(server; sslconfig=sslconfig) - end - if listenany - p, server = Sockets.listenany(addr.host, addr.port) - addr = getinet(addr.host, p) - port = string(p) - elseif reuseaddr - if !supportsreuseaddr() - @warn "reuseaddr=true not supported on this platform: $(Sys.KERNEL)" - @goto fallback - end - server = Sockets.TCPServer(delay = false) - rc = ccall(:jl_tcp_reuseport, Int32, (Ptr{Cvoid},), server.handle) - if rc < 0 - close(server) - @warn "reuseaddr=true failed; falling back to regular listen: $(Sys.KERNEL)" - @goto fallback - end - Sockets.bind(server, addr.host, addr.port; reuseaddr=true) - Sockets.listen(server; backlog=backlog) - else -@label fallback - server = Sockets.listen(addr; backlog=backlog) - end - return Listener(addr, host, port, sslconfig, server) -end - -Listener(host::Union{IPAddr, String}, port::Integer; kw...) = Listener(getinet(host, port), string(host), string(port); kw...) -Listener(port::Integer; kw...) = Listener(Sockets.localhost, port; kw...) -Listener(; kw...) = Listener(Sockets.localhost, 8081; kw...) - -Base.isopen(l::Listener) = isopen(l.server) -Base.close(l::Listener) = close(l.server) - -accept(s::Listener{Nothing}) = Sockets.accept(s.server)::TCPSocket -accept(s::Listener{SSLConfig}) = getsslcontext(Sockets.accept(s.server), s.ssl) - -function getsslcontext(tcp, sslconfig) - try - ssl = MbedTLS.SSLContext() - MbedTLS.setup!(ssl, sslconfig) - MbedTLS.associate!(ssl, tcp) - MbedTLS.handshake!(ssl) - return ssl - catch e - @try Base.IOError close(tcp) - e isa Base.IOError && return nothing - e isa MbedTLS.MbedException && return nothing - rethrow(e) - end -end - -""" - HTTP.Server - -Returned from `HTTP.listen!`/`HTTP.serve!` once a server is up listening -and ready to accept connections. Internally keeps track of active connections. -Also holds reference to any `on_shutdown` functions to be called when the server -is closed. Also holds a reference to the listening loop `Task` that can be -waited on via `wait(server)`, which provides similar functionality to `HTTP.listen`/ -`HTTP.serve`. Can initiate a graceful shutdown where active connections are allowed -to finish being handled by calling `close(server)`. For a more forceful and immediate -shutdown, use `HTTP.forceclose(server)`. -""" -struct Server{L <: Listener} - # listener socket + details - listener::L - # optional function or vector of functions - # to call when closing server - on_shutdown::Any - # list of currently acctive connections - connections::Set{Connection} - # server listenandserve loop task - task::Task - # Protects the connections Set which is mutated in the listenloop - # while potentially being accessed by the close method at the same time - connections_lock::ReentrantLock -end - -port(s::Server) = Int(s.listener.addr.port) -Base.isopen(s::Server) = isopen(s.listener) -Base.wait(s::Server) = wait(s.task) - -function forceclose(s::Server) - shutdown(s.on_shutdown) - close(s.listener) - Base.@lock s.connections_lock begin - for c in s.connections - close(c) - end - end - return wait(s.task) -end - -""" - ConnectionState - -When a connection is first made, it immediately goes into IDLE state. -Once startread(stream) returns, it's marked as ACTIVE. -Once closewrite(stream) returns, it's put back in IDLE, -unless it's been given the CLOSING state, then -it will close(c) itself and mark itself as CLOSED. -""" -@enum ConnectionState IDLE ACTIVE CLOSING CLOSED -closedorclosing(st) = st == CLOSING || st == CLOSED - -function requestclose!(c::Connection) - if c.state == IDLE - c.state = CLOSED - close(c) - else - c.state = CLOSING - end - return -end - -function closeconnection(c::Connection) - c.state = CLOSED - close(c) - return -end - -# graceful shutdown that waits for active connectiosn to finish being handled -function Base.close(s::Server) - shutdown(s.on_shutdown) - close(s.listener) - # first pass to mark or request connections to close - Base.@lock s.connections_lock begin - for c in s.connections - requestclose!(c) - end - end - # second pass to wait for connections to close - # we wait for connections to empty because as - # connections close themselves, they are removed - # from our connections Set - while true - Base.@lock s.connections_lock begin - isempty(s.connections) && break - end - sleep(0.5 + rand() * 0.1) - end - return wait(s.task) -end - -""" - shutdown(fns::Vector{<:Function}) - shutdown(fn::Function) - shutdown(::Nothing) - -Runs function(s) in `on_shutdown` field of `Server` when -`Server` is closed. -""" -shutdown(fns::Vector{<:Function}) = foreach(shutdown, fns) -shutdown(::Nothing) = nothing -function shutdown(fn::Function) - try - fn() - catch - @error begin - msg = current_exceptions_to_string() - "shutdown function $fn failed. $msg" - end - end -end - -""" - HTTP.listen(handler, host=Sockets.localhost, port=8081; kw...) - HTTP.listen(handler, port::Integer=8081; kw...) - HTTP.listen(handler, server::Base.IOServer; kw...) - HTTP.listen!(args...; kw...) -> HTTP.Server - -Listen for HTTP connections and execute the `handler` function for each request. -Listening details can be passed as `host`/`port` pair, a single `port` (`host` will -default to `localhost`), or an already listening `server` object, as returned from -`Sockets.listen`. To open up a server to external requests, the `host` argument is -typically `"0.0.0.0"`. - -The `HTTP.listen!` form is non-blocking and returns an `HTTP.Server` object which can be -`wait(server)`ed on manually, or `close(server)`ed to gracefully shut down the server. -Calling `HTTP.forceclose(server)` will immediately force close the server and all active -connections. `HTTP.listen` will block on the server listening loop until interrupted or -and an irrecoverable error occurs. - -The `handler` function should be of the form `f(::HTTP.Stream)::Nothing`, and should -at the minimum set a status via `setstatus()` and call `startwrite()` either -explicitly or implicitly by writing out a response via `write()`. Failure to -do this will result in an HTTP 500 error being transmitted to the client. - -Optional keyword arguments: - - `sslconfig=nothing`, Provide an `MbedTLS.SSLConfig` object to handle ssl - connections. Pass `sslconfig=MbedTLS.SSLConfig(false)` to disable ssl - verification (useful for testing). Construct a custom `SSLConfig` object - with `MbedTLS.SSLConfig(certfile, keyfile)`. - - `tcpisvalid = tcp->true`, function `f(::TCPSocket)::Bool` to check if accepted - connections are valid before processing requests. e.g. to do source IP filtering. - - `readtimeout::Int=0`, close the connection if no data is received for this - many seconds. Use readtimeout = 0 to disable. - - `reuseaddr::Bool=false`, allow multiple servers to listen on the same port. - Not supported on some OS platforms. Can check `HTTP.Servers.supportsreuseaddr()`. - - `server::Base.IOServer=nothing`, provide an `IOServer` object to listen on; - allows manually closing or configuring the server socket. - - `verbose::Bool=false`, log connection information to `stdout`. - - `access_log::Function`, function for formatting access log messages. The - function should accept two arguments, `io::IO` to which the messages should - be written, and `http::HTTP.Stream` which can be used to query information - from. See also [`@logfmt_str`](@ref). - - `on_shutdown::Union{Function, Vector{<:Function}, Nothing}=nothing`, one or - more functions to be run if the server is closed (for example by an - `InterruptException`). Note, shutdown function(s) will not run if an - `IOServer` object is supplied to the `server` keyword argument and closed - by `close(server)`. - -e.g. -```julia -# start a blocking server -HTTP.listen("127.0.0.1", 8081) do http - HTTP.setheader(http, "Content-Type" => "text/html") - write(http, "target uri: \$(http.message.target)
") - write(http, "request body:
")
-    write(http, read(http))
-    write(http, "
") - return -end - -# non-blocking server -server = HTTP.listen!("127.0.0.1", 8081) do http - @show http.message - @show HTTP.header(http, "Content-Type") - while !eof(http) - println("body data: ", String(readavailable(http))) - end - HTTP.setstatus(http, 404) - HTTP.setheader(http, "Foo-Header" => "bar") - startwrite(http) - write(http, "response body") - write(http, "more response body") -end -# can gracefully close server manually -close(server) -``` - -To run the following HTTP chat example, open two Julia REPL windows and paste -the example code into both of them. Then in one window run `chat_server()` and -in the other run `chat_client()`, then type `hello` and press return. -Whatever you type on the client will be displayed on the server and vis-versa. - -``` -using HTTP - -function chat(io::HTTP.Stream) - Threads.@spawn while !eof(io) - write(stdout, readavailable(io), "\\n") - end - while isopen(io) - write(io, readline(stdin)) - end -end - -chat_server() = HTTP.listen("127.0.0.1", 8087) do io - write(io, "HTTP.jl Chat Server. Welcome!") - chat(io) -end - -chat_client() = HTTP.open("POST", "http://127.0.0.1:8087") do io - chat(io) -end -``` -""" -function listen end - -""" - HTTP.listen!(args...; kw...) -> HTTP.Server - -Non-blocking version of [`HTTP.listen`](@ref); see that function for details. -""" -function listen! end - -listen(f, args...; kw...) = listen(f, Listener(args...; kw...); kw...) -listen!(f, args...; kw...) = listen!(f, Listener(args...; kw...); kw...) - -function listen(f, listener::Listener; kw...) - server = listen!(f, listener; kw...) - # block on server task - try - wait(server) - finally - # try to gracefully close - close(server) - end - return server -end - -# compat for `Threads.@spawn :interactive expr` -@static if hasmethod(getfield(Threads, Symbol("@spawn")), Tuple{LineNumberNode, Module, Symbol, Expr}) - macro _spawn_interactive(ex) - esc(:(Threads.@spawn :interactive $ex)) - end -else - macro _spawn_interactive(ex) - esc(:(@async $ex)) - end -end - -function listen!(f, listener::Listener; - on_shutdown=nothing, - tcpisvalid=TRUE, - max_connections::Integer=typemax(Int), - readtimeout::Integer=0, - access_log::Union{Function,Nothing}=nothing, - verbose=false, kw...) - conns = Set{Connection}() - conns_lock = ReentrantLock() - ready_to_accept = Threads.Event() - if verbose > 0 - tsk = @_spawn_interactive LoggingExtras.withlevel(Logging.Debug) do - listenloop(f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, conns_lock, verbose) - end - else - tsk = @_spawn_interactive listenloop(f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, conns_lock, verbose) - end - # wait until the listenloop enters the loop - wait(ready_to_accept) - return Server(listener, on_shutdown, conns, tsk, conns_lock) -end - -"""" -Main server loop. -Accepts new tcp connections and spawns async tasks to handle them." -""" -function listenloop( - f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, - conns_lock, verbose -) - sem = Base.Semaphore(max_connections) - verbose >= 0 && @info "Listening on: $(listener.hostname):$(listener.hostport), thread id: $(Threads.threadid())" - notify(ready_to_accept) - while isopen(listener) - try - Base.acquire(sem) - io = accept(listener) - if io === nothing - verbose >= 0 && @warn "unable to accept new connection" - continue - elseif !tcpisvalid(io) - verbose >= 0 && @warn "!tcpisvalid: $io" - close(io) - continue - end - conn = Connection(io) - conn.state = IDLE - Base.@lock conns_lock push!(conns, conn) - conn.host, conn.port = listener.hostname, listener.hostport - @async try - handle_connection(f, conn, listener, readtimeout, access_log, verbose) - finally - # handle_connection is in charge of closing the underlying io - Base.@lock conns_lock delete!(conns, conn) - Base.release(sem) - end - catch e - if e isa Base.IOError && e.code == Base.UV_ECONNABORTED - verbose >= 0 && @info "Server on $(listener.hostname):$(listener.hostport) closing" - else - verbose >= 1 && @error begin - msg = current_exceptions_to_string() - "Server on $(listener.hostname):$(listener.hostport) errored. $msg" - end - # quick little sleep in case there's a temporary - # local error accepting and this might help avoid quickly re-erroring - sleep(0.05 + rand() * 0.05) - end - end - end - return -end - -""" -Start a `check_readtimeout` task to close the `Connection` if it is inactive. -Passes the `Connection` object to handle a single request/response transaction -for each HTTP Request received. -After `reuse_limit + 1` transactions, signal `final_transaction` to the -transaction handler, which will close the connection. -""" -function handle_connection(f, c::Connection, listener, readtimeout, access_log, verbose) - wait_for_timeout = Ref{Bool}(true) - if readtimeout > 0 - @async check_readtimeout(c, readtimeout, wait_for_timeout, verbose) - end - try - # if the connection socket or listener close, we stop taking requests - while isopen(c) && !closedorclosing(c.state) && isopen(listener) - # create new Request to be populated by parsing code - request = Request() - # wrap Request in Stream w/ Connection for request reading/response writing - http = Stream(request, c) - # attempt to read request line and headers - try - startread(http) - @debug "startread called" - c.state = ACTIVE # once we've started reading, set ACTIVE state - catch e - # for ParserErrors, try to inform client of the problem - if e isa ParseError - write(c, Response(e.code == :HEADER_SIZE_EXCEEDS_LIMIT ? 431 : 400, string(e.code))) - end - @debug begin - msg = current_exceptions_to_string() - "handle_connection startread error. $msg" - end - break - end - - if hasheader(request, "Connection", "close") - c.state = CLOSING # set CLOSING so no more requests are read - setheader(request.response, "Connection" => "close") - end - request.response.status = 200 - - try - # invokelatest becuase the perf is negligible, but this makes live-editing handlers more Revise friendly - @debug "invoking handler" - Base.invokelatest(f, http) - # If `startwrite()` was never called, throw an error so we send a 500 and log this - if isopen(http) && !iswritable(http) - error("Server never wrote a response.\n\n$request") - end - @debug "closeread" - closeread(http) - @debug "closewrite" - closewrite(http) - c.state = IDLE - catch e - # The remote can close the stream whenever it wants to, but there's nothing - # anyone can do about it on this side. No reason to log an error in that case. - level = e isa Base.IOError && !isopen(c) ? Logging.Debug : Logging.Error - @logmsgv 1 level begin - msg = current_exceptions_to_string() - "handle_connection handler error. $msg" - end request - - if isopen(http) && !iswritable(http) - request.response.status = 500 - startwrite(http) - closewrite(http) - end - c.state = CLOSING - finally - if access_log !== nothing - @try(Any, @info sprint(access_log, http) _group=:access) - end - end - end - catch - # we should be catching everything inside the while loop, but just in case - verbose >= 0 && @error begin - msg = current_exceptions_to_string() - "error while handling connection. $msg" - end - finally - if readtimeout > 0 - wait_for_timeout[] = false - end - # when we're done w/ the connection, ensure it's closed and state is properly set - closeconnection(c) - end - return -end - -""" -If `c` is inactive for a more than `readtimeout` then close the `c`." -""" -function check_readtimeout(c, readtimeout, wait_for_timeout, verbose) - while wait_for_timeout[] - if inactiveseconds(c) > readtimeout - verbose >= 0 && @warn "Connection Timeout: $c" - try - writeheaders(c, Response(408, ["Connection" => "close"])) - finally - closeconnection(c) - end - break - end - sleep(readtimeout + rand() * readtimeout) - end - return -end - -end # module diff --git a/src/Streams.jl b/src/Streams.jl deleted file mode 100644 index 26ce305b3..000000000 --- a/src/Streams.jl +++ /dev/null @@ -1,416 +0,0 @@ -module Streams - -export Stream, closebody, isaborted, setstatus, readall! - -using Sockets -using ..IOExtras, ..Messages, ..Connections, ..Conditions, ..Exceptions -import ..HTTP # for doc references - -mutable struct Stream{M <: Message, S <: IO} <: IO - message::M - stream::S - writechunked::Bool - readchunked::Bool - warn_not_to_read_one_byte_at_a_time::Bool - ntoread::Int - nwritten::Int -end - -""" - Stream(::Request, ::IO) - -Creates a `HTTP.Stream` that wraps an existing `IO` stream. - - - `startwrite(::Stream)` sends the `Request` headers to the `IO` stream. - - `write(::Stream, body)` sends the `body` (or a chunk of the body). - - `closewrite(::Stream)` sends the final `0` chunk (if needed) and calls - `closewrite` on the `IO` stream. - - - `startread(::Stream)` calls `startread` on the `IO` stream then - reads and parses the `Response` headers. - - `eof(::Stream)` and `readavailable(::Stream)` parse the body from the `IO` - stream. - - `closeread(::Stream)` reads the trailers and calls `closeread` on the `IO` - stream. When the `IO` stream is a [`HTTP.Connections.Connection`](@ref), - calling `closeread` releases the connection back to the connection pool - for reuse. If a complete response has not been received, `closeread` throws - `EOFError`. -""" -Stream(r::M, io::S) where {M, S} = Stream{M, S}(r, io, false, false, true, 0, -1) - -Messages.header(http::Stream, a...) = header(http.message, a...) -setstatus(http::Stream, status) = (http.message.response.status = status) -Messages.setheader(http::Stream, a...) = setheader(http.message.response, a...) -Connections.getrawstream(http::Stream) = getrawstream(http.stream) - -Sockets.getsockname(http::Stream) = Sockets.getsockname(IOExtras.tcpsocket(getrawstream(http))) -function Sockets.getpeername(http::Stream) - # TODO: MbedTLS only forwards getsockname(::SSLContext) - # so we use IOExtras.tcpsocket to reach into the MbedTLS internals - # for now to keep compatibility with older MbedTLS versions. - # return Sockets.getpeername(getrawstream(http)) - return Sockets.getpeername(IOExtras.tcpsocket(getrawstream(http))) -end - -IOExtras.isopen(http::Stream) = isopen(http.stream) - -# Writing HTTP Messages - -messagetowrite(http::Stream{<:Response}) = http.message.request::Request -messagetowrite(http::Stream{<:Request}) = http.message.response - -IOExtras.iswritable(http::Stream) = iswritable(http.stream) - -function IOExtras.startwrite(http::Stream) - if !iswritable(http.stream) - startwrite(http.stream) - end - m = messagetowrite(http) - if !hasheader(m, "Content-Length") && - !hasheader(m, "Transfer-Encoding") && - !hasheader(m, "Upgrade") && - (m isa Request || (m.request.version >= v"1.1" && bodylength(m) > 0)) - - http.writechunked = true - setheader(m, "Transfer-Encoding" => "chunked") - else - http.writechunked = ischunked(m) - end - n = writeheaders(http.stream, m) - # nwritten starts at -1 so that we can tell if we've written anything yet - http.nwritten = 0 # should not include headers - return n -end - -function Base.unsafe_write(http::Stream, p::Ptr{UInt8}, n::UInt) - if n == 0 - return 0 - end - if !iswritable(http) && isopen(http.stream) - startwrite(http) - end - nw = if !http.writechunked - unsafe_write(http.stream, p, n) - else - write(http.stream, string(n, base=16), "\r\n") + - unsafe_write(http.stream, p, n) + - write(http.stream, "\r\n") - end - http.nwritten += nw - return nw -end - -""" - closebody(::Stream) - -Write the final `0` chunk if needed. -""" -function closebody(http::Stream) - if http.writechunked - http.writechunked = false - @try Base.IOError write(http.stream, "0\r\n\r\n") - end -end - -function IOExtras.closewrite(http::Stream{<:Response}) - if !iswritable(http) - return - end - closebody(http) - closewrite(http.stream) -end - -function IOExtras.closewrite(http::Stream{<:Request}) - - if iswritable(http) - closebody(http) - closewrite(http.stream) - end - - if hasheader(http.message, "Connection", "close") || - hasheader(http.message, "Connection", "upgrade") || - http.message.version < v"1.1" && - !hasheader(http.message, "Connection", "keep-alive") - - @debug "✋ \"Connection: close\": $(http.stream)" - close(http.stream) - end -end - -# Reading HTTP Messages - -IOExtras.isreadable(http::Stream) = isreadable(http.stream) - -Base.bytesavailable(http::Stream) = min(ntoread(http), - bytesavailable(http.stream)) - -function IOExtras.startread(http::Stream) - - if !isreadable(http.stream) - startread(http.stream) - end - - readheaders(http.stream, http.message) - handle_continue(http) - - http.readchunked = ischunked(http.message) - http.ntoread = bodylength(http.message) - - return http.message -end - -""" -100 Continue -https://tools.ietf.org/html/rfc7230#section-5.6 -https://tools.ietf.org/html/rfc7231#section-6.2.1 -""" -function handle_continue(http::Stream{<:Response}) - if http.message.status == 100 - @debug "✅ Continue: $(http.stream)" - readheaders(http.stream, http.message) - end -end - -function handle_continue(http::Stream{<:Request}) - if hasheader(http.message, "Expect", "100-continue") - if !iswritable(http.stream) - startwrite(http.stream) - end - @debug "✅ Continue: $(http.stream)" - writeheaders(http.stream, Response(100)) - end -end - -function Base.eof(http::Stream) - if !headerscomplete(http.message) - startread(http) - end - if http.ntoread == 0 - return true - end - return eof(http.stream) -end - -@inline function ntoread(http::Stream) - - if !headerscomplete(http.message) - startread(http) - end - - # Find length of next chunk - if http.ntoread == unknown_length && http.readchunked - http.ntoread = readchunksize(http.stream, http.message) - end - - return http.ntoread -end - -@inline function update_ntoread(http::Stream, n) - - if http.ntoread != unknown_length - http.ntoread -= n - end - - if http.readchunked - if http.ntoread == 0 - http.ntoread = unknown_length - end - end - - @ensure http.ntoread >= 0 -end - -function Base.readavailable(http::Stream, n::Int=typemax(Int)) - - ntr = ntoread(http) - if ntr == 0 - return UInt8[] - end - - bytes = read(http.stream, min(n, ntr)) - update_ntoread(http, length(bytes)) - return bytes -end - -Base.read(http::Stream, n::Integer) = readavailable(http, Int(n)) - -function Base.read(http::Stream, ::Type{UInt8}) - - if http.warn_not_to_read_one_byte_at_a_time - @warn "Reading one byte at a time from HTTP.Stream is inefficient.\n" * - "Use: io = BufferedInputStream(http::HTTP.Stream) instead.\n" * - "See: https://github.com/BioJulia/BufferedStreams.jl" - http.warn_not_to_read_one_byte_at_a_time = false - end - - if ntoread(http) == 0 - throw(EOFError()) - end - update_ntoread(http, 1) - - return read(http.stream, UInt8) -end - -function http_unsafe_read(http::Stream, p::Ptr{UInt8}, n::UInt)::Int - ntr = UInt(ntoread(http)) - ntr == 0 && return 0 - # If there is spare space in `p` - # read two extra bytes - # (`\r\n` at end ofchunk). - unsafe_read(http.stream, p, min(n, ntr + (http.readchunked ? 2 : 0))) - n = min(n, ntr) - update_ntoread(http, n) - return n -end - -function Base.readbytes!(http::Stream, buf::AbstractVector{UInt8}, n=length(buf)) - n > length(buf) && resize!(buf, n) - return GC.@preserve buf http_unsafe_read(http, pointer(buf), UInt(n)) -end - -function Base.unsafe_read(http::Stream, p::Ptr{UInt8}, n::UInt) - nread = 0 - while nread < n - if eof(http) - throw(EOFError()) - end - nread += http_unsafe_read(http, p + nread, n - nread) - end - nothing -end - -_alloc_request(buf::IOBuffer, recommended_size::UInt) = Base.alloc_request(buf, recommended_size) - -@static if VERSION < v"1.11" - function _alloc_request(buffer::Base.GenericIOBuffer, recommended_size::UInt) - Base.ensureroom(buffer, Int(recommended_size)) - ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data), buffer.maxsize) - ptr + 1 - return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) - end -else - function _alloc_request(buffer::Base.GenericIOBuffer, recommended_size::UInt) - Base.ensureroom(buffer, Int(recommended_size)) - ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data)-buffer.offset, buffer.maxsize) + buffer.offset - ptr + 1 - return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) - end -end - -function Base.readbytes!(http::Stream, buf::Base.GenericIOBuffer, n=bytesavailable(http)) - p, nbmax = _alloc_request(buf, UInt(n)) - nbmax < n && throw(ArgumentError("Unable to grow response stream IOBuffer $nbmax large enough for response body size: $n")) - GC.@preserve buf unsafe_read(http, p, UInt(n)) - # TODO: use `Base.notify_filled(buf, Int(n))` here, but only once it is identical to this: - if buf.append - buf.size += Int(n) - else - buf.ptr += Int(n) - buf.size = max(buf.size, buf.ptr - 1) - end - return n -end - -function Base.read(http::Stream, buf::Base.GenericIOBuffer=PipeBuffer()) - readall!(http, buf) - return take!(buf) -end - -function readall!(http::Stream, buf::Base.GenericIOBuffer=PipeBuffer()) - n = 0 - if ntoread(http) == unknown_length - while !eof(http) - n += readbytes!(http, buf) - end - else - # even if we know the length, we still need to read until eof - # because Transfer-Encoding: chunked comes in piece-by-piece - while !eof(http) - n += readbytes!(http, buf, ntoread(http)) - end - end - return n -end - -function IOExtras.readuntil(http::Stream, f::Function) - UInt(ntoread(http)) == 0 && return Connections.nobytes - try - bytes = IOExtras.readuntil(http.stream, f) - update_ntoread(http, length(bytes)) - return bytes - catch ex - ex isa EOFError || rethrow() - # if we error, it means we didn't find what we were looking for - # TODO: this seems very sketchy - return UInt8[] - end -end - -""" - isaborted(::Stream{<:Response}) - -Has the server signaled that it does not wish to receive the message body? - -"If [the response] indicates the server does not wish to receive the - message body and is closing the connection, the client SHOULD - immediately cease transmitting the body and close the connection." -[RFC7230, 6.5](https://tools.ietf.org/html/rfc7230#section-6.5) -""" -function isaborted(http::Stream{<:Response}) - - if iswritable(http.stream) && - iserror(http.message) && - hasheader(http.message, "Connection", "close") - @debug "✋ Abort on $(sprint(writestartline, http.message)): " * - "$(http.stream)" - @debug "✋ $(http.message)" - return true - end - return false -end - -Messages.isredirect(http::Stream{<:Response}) = isredirect(http.message) && isredirect(http.message.request) -Messages.retryable(http::Stream{<:Response}) = retryable(http.message) && retryable(http.message.request) - -incomplete(http::Stream) = - http.ntoread > 0 && (http.readchunked || http.ntoread != unknown_length) - -function IOExtras.closeread(http::Stream{<:Response}) - - if hasheader(http.message, "Connection", "close") - # Close conncetion if server sent "Connection: close"... - @debug "✋ \"Connection: close\": $(http.stream)" - close(http.stream) - # Error if Message is not complete... - incomplete(http) && throw(EOFError()) - else - - # Discard body bytes that were not read... - @try Base.IOError EOFError while !eof(http) - readavailable(http) - end - - if incomplete(http) - # Error if Message is not complete... - close(http.stream) - throw(EOFError()) - elseif isreadable(http.stream) - closeread(http.stream) - end - end - - return http.message -end - -function IOExtras.closeread(http::Stream{<:Request}) - if incomplete(http) - # Error if Message is not complete... - close(http.stream) - throw(EOFError()) - end - if isreadable(http) - closeread(http.stream) - end -end - -end #module Streams diff --git a/src/Strings.jl b/src/Strings.jl deleted file mode 100644 index 56f05fd15..000000000 --- a/src/Strings.jl +++ /dev/null @@ -1,172 +0,0 @@ -module Strings - -export HTTPVersion, escapehtml, tocameldash, iso8859_1_to_utf8, ascii_lc_isequal - -using ..IOExtras - -# A `Base.VersionNumber` is a SemVer spec, whereas a HTTP versions is just 2 digits, -# This allows us to use a smaller type and more importantly write a simple parse method -# that avoid allocations. -""" - HTTPVersion(major, minor) - -The HTTP version number consists of two digits separated by a -"." (period or decimal point). The first digit (`major` version) -indicates the HTTP messaging syntax, whereas the second digit (`minor` -version) indicates the highest minor version within that major -version to which the sender is conformant and able to understand for -future communication. - -See [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) -""" -struct HTTPVersion - major::UInt8 - minor::UInt8 -end - -HTTPVersion(major::Integer) = HTTPVersion(major, 0x00) -HTTPVersion(v::AbstractString) = parse(HTTPVersion, v) -HTTPVersion(v::VersionNumber) = convert(HTTPVersion, v) -# Lossy conversion. We ignore patch/prerelease/build parts even if non-zero/non-empty, -# because we don't want to add overhead for a case that should never be relevant. -Base.convert(::Type{HTTPVersion}, v::VersionNumber) = HTTPVersion(v.major, v.minor) -Base.VersionNumber(v::HTTPVersion) = VersionNumber(v.major, v.minor) - -Base.show(io::IO, v::HTTPVersion) = print(io, "HTTPVersion(\"", string(v.major), ".", string(v.minor), "\")") - -Base.:(==)(va::VersionNumber, vb::HTTPVersion) = va == VersionNumber(vb) -Base.:(==)(va::HTTPVersion, vb::VersionNumber) = VersionNumber(va) == vb - -Base.isless(va::VersionNumber, vb::HTTPVersion) = isless(va, VersionNumber(vb)) -Base.isless(va::HTTPVersion, vb::VersionNumber) = isless(VersionNumber(va), vb) -function Base.isless(va::HTTPVersion, vb::HTTPVersion) - va.major < vb.major && return true - va.major > vb.major && return false - va.minor < vb.minor && return true - return false -end - -function Base.parse(::Type{HTTPVersion}, v::AbstractString) - ver = tryparse(HTTPVersion, v) - ver === nothing && throw(ArgumentError("invalid HTTP version string: $(repr(v))")) - return ver -end - -# We only support single-digits for major and minor versions -# - we can parse 0.9 but not 0.10 -# - we can parse 9.0 but not 10.0 -function Base.tryparse(::Type{HTTPVersion}, v::AbstractString) - isempty(v) && return nothing - len = ncodeunits(v) - - i = firstindex(v) - d1 = v[i] - if isdigit(d1) - major = parse(UInt8, d1) - else - return nothing - end - - i = nextind(v, i) - i > len && return HTTPVersion(major) - dot = v[i] - dot == '.' || return nothing - - i = nextind(v, i) - i > len && return HTTPVersion(major) - d2 = v[i] - if isdigit(d2) - minor = parse(UInt8, d2) - else - return nothing - end - return HTTPVersion(major, minor) -end - -""" - escapehtml(i::String) - -Returns a string with special HTML characters escaped: &, <, >, ", ' -""" -function escapehtml(i::AbstractString) - # Refer to http://stackoverflow.com/a/7382028/3822752 for spec. links - o = replace(i, "&" =>"&") - o = replace(o, "\""=>""") - o = replace(o, "'" =>"'") - o = replace(o, "<" =>"<") - o = replace(o, ">" =>">") - return o -end - -""" - tocameldash(s::String) - -Ensure the first character and characters that follow a '-' are uppercase. -""" -function tocameldash(s::String) - toUpper = UInt8('A') - UInt8('a') - v = Vector{UInt8}(bytes(s)) - upper = true - for i = 1:length(v) - @inbounds b = v[i] - if upper - islower(b) && (v[i] = b + toUpper) - else - isupper(b) && (v[i] = lower(b)) - end - upper = b == UInt8('-') - end - return String(v) -end - -tocameldash(s::AbstractString) = tocameldash(String(s)) - -@inline islower(b::UInt8) = UInt8('a') <= b <= UInt8('z') -@inline isupper(b::UInt8) = UInt8('A') <= b <= UInt8('Z') -@inline lower(c::UInt8) = c | 0x20 - -""" - iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) - -Convert from ISO8859_1 to UTF8. -""" -function iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) - io = IOBuffer() - for b in bytes - if b < 0x80 - write(io, b) - else - write(io, 0xc0 | (b >> 6)) - write(io, 0x80 | (b & 0x3f)) - end - end - return String(take!(io)) -end - -""" -Convert ASCII (RFC20) character `c` to lower case. -""" -ascii_lc(c::UInt8) = c in UInt8('A'):UInt8('Z') ? c + 0x20 : c - -""" -Case insensitive ASCII character comparison. -""" -ascii_lc_isequal(a::UInt8, b::UInt8) = ascii_lc(a) == ascii_lc(b) - -""" - HTTP.ascii_lc_isequal(a::String, b::String) - -Case insensitive ASCII string comparison. -""" -function ascii_lc_isequal(a, b) - acu = codeunits(a) - bcu = codeunits(b) - len = length(acu) - len != length(bcu) && return false - for i = 1:len - @inbounds !ascii_lc_isequal(acu[i], bcu[i]) && return false - end - return true -end - -end # module Strings diff --git a/src/WebSockets.jl b/src/WebSockets.jl deleted file mode 100644 index 1afb5f761..000000000 --- a/src/WebSockets.jl +++ /dev/null @@ -1,729 +0,0 @@ -module WebSockets - -using Base64, UUIDs, Sockets, Random -using MbedTLS: digest, MD_SHA1, SSLContext -using ..IOExtras, ..Streams, ..Connections, ..Messages, ..Conditions, ..Servers -using ..Exceptions: current_exceptions_to_string -import ..open -import ..HTTP # for doc references - -export WebSocket, send, receive, ping, pong - -# 1st 2 bytes of a frame -primitive type FrameFlags 16 end -uint16(x::FrameFlags) = Base.bitcast(UInt16, x) -FrameFlags(x::UInt16) = Base.bitcast(FrameFlags, x) - -const WS_FINAL = 0b1000000000000000 -const WS_RSV1 = 0b0100000000000000 -const WS_RSV2 = 0b0010000000000000 -const WS_RSV3 = 0b0001000000000000 -const WS_OPCODE = 0b0000111100000000 -const WS_MASK = 0b0000000010000000 -const WS_LEN = 0b0000000001111111 - -@enum OpCode::UInt8 CONTINUATION=0x00 TEXT=0x01 BINARY=0x02 CLOSE=0x08 PING=0x09 PONG=0x0A - -iscontrol(opcode::OpCode) = opcode > BINARY - -Base.propertynames(x::FrameFlags) = (:final, :rsv1, :rsv2, :rsv3, :opcode, :mask, :len) -function Base.getproperty(x::FrameFlags, nm::Symbol) - ux = uint16(x) - if nm == :final - return ux & WS_FINAL > 0 - elseif nm == :rsv1 - return ux & WS_RSV1 > 0 - elseif nm == :rsv2 - return ux & WS_RSV2 > 0 - elseif nm == :rsv3 - return ux & WS_RSV3 > 0 - elseif nm == :opcode - return OpCode(((ux & WS_OPCODE) >> 8) % UInt8) - elseif nm == :masked - return ux & WS_MASK > 0 - elseif nm == :len - return ux & WS_LEN - end -end - -FrameFlags(final::Bool, opcode::OpCode, masked::Bool, len::Integer; rsv1::Bool=false, rsv2::Bool=false, rsv3::Bool=false) = - FrameFlags( - (final ? WS_FINAL : UInt16(0)) | - (rsv1 ? WS_RSV1 : UInt16(0)) | (rsv2 ? WS_RSV2 : UInt16(0)) | (rsv3 ? WS_RSV3 : UInt16(0)) | - (UInt16(opcode) << 8) | - (masked ? WS_MASK : UInt16(0)) | - (len % UInt16) - ) - -Base.show(io::IO, x::FrameFlags) = - print(io, "FrameFlags(", "final=", x.final, ", ", "opcode=", x.opcode, ", ", "masked=", x.masked, ", ", "len=", x.len, ")") - -primitive type Mask 32 end -Base.UInt32(x::Mask) = Base.bitcast(UInt32, x) -Mask(x::UInt32) = Base.bitcast(Mask, x) -Base.getindex(x::Mask, i::Int) = (UInt32(x) >> (8 * ((i - 1) % 4))) % UInt8 -mask() = Mask(rand(Random.RandomDevice(), UInt32)) -const EMPTY_MASK = Mask(UInt32(0)) - -# representation of a single websocket frame -struct Frame - flags::FrameFlags - extendedlen::Union{Nothing, UInt16, UInt64} - mask::Mask - # when sending, Vector{UInt8} if client, any AbstractVector{UInt8} if server - # when receiving: - # CONTINUATION: String or Vector{UInt8} based on first fragment frame opcode TEXT/BINARY - # TEXT: String - # BINARY/PING/PONG: Vector{UInt8} - # CLOSE: CloseFrameBody - payload::Any -end - -# given a payload total length, split into 7-bit length + 16-bit or 64-bit extended length -wslength(l) = l < 0x7E ? (UInt8(l), nothing) : - l <= 0xFFFF ? (0x7E, UInt16(l)) : - (0x7F, UInt64(l)) - -# give a mutable byte payload + mask, perform client websocket masking -function mask!(bytes::Vector{UInt8}, mask) - for i in 1:length(bytes) - @inbounds bytes[i] = bytes[i] ⊻ mask[i] - end - return -end - -# send method Frame constructor -function Frame(final::Bool, opcode::OpCode, client::Bool, payload::AbstractVector{UInt8}; rsv1::Bool=false, rsv2::Bool=false, rsv3::Bool=false) - len, extlen = wslength(length(payload)) - if client - msk = mask() - mask!(payload, msk) - else - msk = EMPTY_MASK - end - return Frame(FrameFlags(final, opcode, client, len; rsv1, rsv2, rsv3), extlen, msk, payload) -end - -Base.show(io::IO, x::Frame) = - print(io, "Frame(", "flags=", x.flags, ", ", "extendedlen=", x.extendedlen, ", ", "mask=", x.mask, ", ", "payload=", x.payload, ")") - -# reading a single frame - -# If _The WebSocket Connection is Closed_ and no Close control frame was received by the -# endpoint (such as could occur if the underlying transport connection -# is lost), _The WebSocket Connection Close Code_ is considered to be 1006. -@noinline iocheck(io) = isopen(io) || throw(WebSocketError(CloseFrameBody(1006, "WebSocket connection is closed"))) - -""" - WebSockets.readframe(ws) -> WebSockets.Frame - WebSockets.readframe(io, Frame, buffer, first_fragment_opcode) -> WebSockets.Frame - -Read a single websocket frame from a `WebSocket` or `IO` stream. -Frame may be a control frame with `PING`, `PONG`, or `CLOSE` opcode. -Frame may also be part of fragmented message, with opcdoe `CONTINUATION`; -`first_fragment_opcode` should be passed from the 1st frame of a fragmented message -to ensure each subsequent frame payload is converted correctly (String or Vector{UInt8}). -""" -function readframe(io::IO, ::Type{Frame}, buffer::Vector{UInt8}=UInt8[], first_fragment_opcode::OpCode=CONTINUATION) - iocheck(io) - flags = FrameFlags(ntoh(read(io, UInt16))) - if flags.len == 0x7E - extlen = ntoh(read(io, UInt16)) - len = UInt64(extlen) - elseif flags.len == 0x7F - extlen = ntoh(read(io, UInt64)) - len = extlen - else - extlen = nothing - len = UInt64(flags.len) - end - mask = flags.masked ? Mask(read(io, UInt32)) : EMPTY_MASK - # even if len is 0, we need to resize! so previously filled buffers aren't erroneously reused - resize!(buffer, len) - if len > 0 - # NOTE: we could support a pure streaming case by allowing the caller to pass - # an IO instead of buffer and writing directly from io -> out_io. - # The tricky case would be server-side streaming, where we need to unmask - # the incoming client payload; we could just buffer the payload + unmask - # and then write out to the out_io. - read!(io, buffer) - end - if flags.masked - mask!(buffer, mask) - end - if flags.opcode == CONTINUATION && first_fragment_opcode == CONTINUATION - throw(WebSocketError(CloseFrameBody(1002, "Continuation frame cannot be the first frame in a message"))) - elseif first_fragment_opcode != CONTINUATION && flags.opcode in (TEXT, BINARY) - throw(WebSocketError(CloseFrameBody(1002, "Received unfragmented frame while still processing fragmented frame"))) - end - op = flags.opcode == CONTINUATION ? first_fragment_opcode : flags.opcode - if op == TEXT - # TODO: possible avoid the double copy from read!(io, buffer) + unsafe_string? - payload = unsafe_string(pointer(buffer), len) - elseif op == CLOSE - if len == 1 - throw(WebSocketError(CloseFrameBody(1002, "Close frame cannot have body of length 1"))) - end - control_len_check(len) - if len >= 2 - st = Int(UInt16(buffer[1]) << 8 | buffer[2]) - validclosecheck(st) - status = st - else - status = 1005 - end - payload = CloseFrameBody(status, len > 2 ? unsafe_string(pointer(buffer) + 2, len - 2) : "") - utf8check(payload.message) - else # BINARY - payload = copy(buffer) - end - return Frame(flags, extlen, mask, payload) -end - -# writing a single frame -function writeframe(io::IO, x::Frame) - buff = IOBuffer() - n = write(buff, hton(uint16(x.flags))) - if x.extendedlen !== nothing - n += write(buff, hton(x.extendedlen)) - end - if x.mask != EMPTY_MASK - n += write(buff, UInt32(x.mask)) - end - pl = x.payload - # manually unroll a few known type cases to help the compiler - if pl isa Vector{UInt8} - n += write(buff, pl) - elseif pl isa Base.CodeUnits{UInt8,String} - n += write(buff, pl) - else - n += write(buff, pl) - end - write(io.io, take!(buff)) - return n -end - -"Status codes according to RFC 6455 7.4.1" -const STATUS_CODE_DESCRIPTION = Dict{Int, String}( - 1000=>"Normal", 1001=>"Going Away", - 1002=>"Protocol Error", 1003=>"Unsupported Data", - 1004=>"Reserved", 1005=>"No Status Recvd- reserved", - 1006=>"Abnormal Closure- reserved", 1007=>"Invalid frame payload data", - 1008=>"Policy Violation", 1009=>"Message too big", - 1010=>"Missing Extension", 1011=>"Internal Error", - 1012=>"Service Restart", 1013=>"Try Again Later", - 1014=>"Bad Gateway", 1015=>"TLS Handshake") - -@noinline validclosecheck(x) = (1000 <= x < 5000 && !(x in (1004, 1005, 1006, 1016, 1100, 2000, 2999))) || throw(WebSocketError(CloseFrameBody(1002, "Invalid close status code"))) - -""" - WebSockets.CloseFrameBody(status, message) - -Represents the payload of a CLOSE control websocket frame. -For error close `status`, it can be wrapped in a `WebSocketError` -and thrown. -""" -struct CloseFrameBody - status::Int - message::String -end - -struct WebSocketError <: Exception - message::Union{String, CloseFrameBody} -end - -""" - WebSockets.isok(x::WebSocketError) -> Bool - -Returns true if the `WebSocketError` has a non-error status code. -When calling `receive(websocket)`, if a CLOSE frame is received, -the CLOSE frame body is parsed and thrown inside the `WebSocketError`, -but if the CLOSE frame has a non-error status code, it's safe to -ignore the error and return from the `WebSockets.open` or `WebSockets.listen` -calls without throwing. -""" -isok(x) = x isa WebSocketError && x.message isa CloseFrameBody && (x.message.status == 1000 || x.message.status == 1001 || x.message.status == 1005) - -""" - WebSocket(io::HTTP.Connection, req, resp; client=true) - -Representation of a websocket connection. -Use `WebSockets.open` to open a websocket connection, passing a -handler function `f(ws)` to send and receive messages. -Use `WebSockets.listen` to listen for incoming websocket connections, -passing a handler function `f(ws)` to send and receive messages. - -Call `send(ws, msg)` to send a message; if `msg` is an `AbstractString`, -a TEXT websocket message will be sent; if `msg` is an `AbstractVector{UInt8}`, -a BINARY websocket message will be sent. Otherwise, `msg` should be an iterable -of either `AbstractString` or `AbstractVector{UInt8}`, and a fragmented message -will be sent, one frame for each iterated element. - -Control frames can be sent by calling `ping(ws[, data])`, `pong(ws[, data])`, -or `close(ws[, body::WebSockets.CloseFrameBody])`. Calling `close` will initiate -the close sequence and close the underlying connection. - -To receive messages, call `receive(ws)`, which will block until a non-control, -full message is received. PING messages will automatically be responded to when -received. CLOSE messages will also be acknowledged and then a `WebSocketError` -will be thrown with the `WebSockets.CloseFrameBody` payload, which may include -a non-error CLOSE frame status code. `WebSockets.isok(err)` can be called to -check if the CLOSE was normal or unexpected. Fragmented messages will be -received until the final frame is received and the full concatenated payload -can be returned. `receive(ws)` returns a `Vector{UInt8}` for BINARY messages, -and a `String` for TEXT messages. - -For convenience, `WebSocket`s support the iteration protocol, where each iteration -will `receive` a non-control message, with iteration terminating when the connection -is closed. E.g.: -```julia -WebSockets.open(url) do ws - for msg in ws - # do cool stuff with msg - end -end -``` -""" -mutable struct WebSocket - id::UUID - io::Connection - request::Request - response::Response - maxframesize::Int - maxfragmentation::Int - client::Bool - readbuffer::Vector{UInt8} - writebuffer::Vector{UInt8} - readclosed::Bool - writeclosed::Bool -end - -const DEFAULT_MAX_FRAG = 1024 - -IOExtras.tcpsocket(ws::WebSocket) = tcpsocket(ws.io) - -WebSocket(io::Connection, req=Request(), resp=Response(); client::Bool=true, maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG) = - WebSocket(uuid4(), io, req, resp, maxframesize, maxfragmentation, client, UInt8[], UInt8[], false, false) - -""" - WebSockets.isclosed(ws) -> Bool - -Check whether a `WebSocket` has sent and received CLOSE frames. -""" -isclosed(ws::WebSocket) = ws.readclosed && ws.writeclosed - -# Handshake -"Check whether a HTTP.Request or HTTP.Response is a websocket upgrade request/response" -function isupgrade(r::Message) - ((r isa Request && r.method == "GET") || - (r isa Response && r.status == 101)) && - (hasheader(r, "Connection", "upgrade") || - hasheader(r, "Connection", "keep-alive, upgrade")) && - hasheader(r, "Upgrade", "websocket") -end - -# Renamed in HTTP@1 -@deprecate is_upgrade isupgrade - -@noinline handshakeerror() = throw(WebSocketError(CloseFrameBody(1002, "Websocket handshake failed"))) - -function hashedkey(key) - hashkey = "$(strip(key))258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - return base64encode(digest(MD_SHA1, hashkey)) -end - -""" - WebSockets.open(handler, url; verbose=false, kw...) - -Initiate a websocket connection to `url` (which should have schema like `ws://` or `wss://`), -and call `handler(ws)` with the websocket connection. Passing `verbose=true` or `verbose=2` -will enable debug logging for the life of the websocket connection. -`handler` should be a function of the form `f(ws) -> nothing`, where `ws` is a [`WebSocket`](@ref). -Supported keyword arguments are the same as supported by [`HTTP.request`](@ref). -Typical websocket usage is: -```julia -WebSockets.open(url) do ws - # iterate incoming websocket messages - for msg in ws - # send message back to server or do other logic here - send(ws, msg) - end - # iteration ends when the websocket connection is closed by server or error -end -``` -""" -function open(f::Function, url; suppress_close_error::Bool=false, verbose=false, headers=[], maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG, kw...) - key = base64encode(rand(Random.RandomDevice(), UInt8, 16)) - headers = [ - "Upgrade" => "websocket", - "Connection" => "Upgrade", - "Sec-WebSocket-Key" => key, - "Sec-WebSocket-Version" => "13", - headers... - ] - # HTTP.open - open("GET", url, headers; verbose=verbose, kw...) do http - startread(http) - isupgrade(http.message) || handshakeerror() - if header(http, "Sec-WebSocket-Accept") != hashedkey(key) - throw(WebSocketError("Invalid Sec-WebSocket-Accept\n" * "$(http.message)")) - end - # later stream logic checks to see if the HTTP message is "complete" - # by seeing if ntoread is 0, which is typemax(Int) for websockets by default - # so set it to 0 so it's correctly viewed as "complete" once we're done - # doing websocket things - http.ntoread = 0 - io = http.stream - ws = WebSocket(io, http.message.request, http.message; maxframesize, maxfragmentation) - @debug "$(ws.id): WebSocket opened" - try - f(ws) - catch e - if !isok(e) - suppress_close_error || @error "$(ws.id): error" (e, catch_backtrace()) - end - if !isclosed(ws) - if e isa WebSocketError && e.message isa CloseFrameBody - close(ws, e.message) - else - close(ws, CloseFrameBody(1008, "Unexpected client websocket error")) - end - end - if !isok(e) - rethrow() - end - finally - if !isclosed(ws) - close(ws, CloseFrameBody(1000, "")) - end - end - end -end - -""" - WebSockets.listen(handler, host, port; verbose=false, kw...) - WebSockets.listen!(handler, host, port; verbose=false, kw...) -> HTTP.Server - -Listen for websocket connections on `host` and `port`, and call `handler(ws)`, -which should be a function taking a single `WebSocket` argument. -Keyword arguments `kw...` are the same as supported by [`HTTP.listen`](@ref). -Typical usage is like: -```julia -WebSockets.listen(host, port) do ws - # iterate incoming websocket messages - for msg in ws - # send message back to client or do other logic here - send(ws, msg) - end - # iteration ends when the websocket connection is closed by client or error -end -``` -""" -function listen end - -listen(f, args...; kw...) = Servers.listen(http -> upgrade(f, http; kw...), args...; kw...) -listen!(f, args...; kw...) = Servers.listen!(http -> upgrade(f, http; kw...), args...; kw...) - -function upgrade(f::Function, http::Streams.Stream; suppress_close_error::Bool=false, maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG, nagle=false, quickack=true, kw...) - @debug "Server websocket upgrade requested" - isupgrade(http.message) || handshakeerror() - if !hasheader(http, "Sec-WebSocket-Version", "13") - throw(WebSocketError("Expected \"Sec-WebSocket-Version: 13\"!\n" * "$(http.message)")) - end - if !hasheader(http, "Sec-WebSocket-Key") - throw(WebSocketError("Expected \"Sec-WebSocket-Key header\"!\n" * "$(http.message)")) - end - setstatus(http, 101) - setheader(http, "Upgrade" => "websocket") - setheader(http, "Connection" => "Upgrade") - key = header(http, "Sec-WebSocket-Key") - setheader(http, "Sec-WebSocket-Accept" => hashedkey(key)) - startwrite(http) - io = http.stream - req = http.message - - # tune websocket tcp connection for performance : https://github.com/JuliaWeb/HTTP.jl/issues/1140 - @static if VERSION >= v"1.3" - sock = tcpsocket(io) - # I don't understand why uninitializd sockets can get here, but they can - if sock.status ∉ (Base.StatusInit, Base.StatusUninit) && isopen(sock) - Sockets.nagle(sock, nagle) - Sockets.quickack(sock, quickack) - end - end - - ws = WebSocket(io, req, req.response; client=false, maxframesize, maxfragmentation) - @debug "$(ws.id): WebSocket upgraded; connection established" - try - f(ws) - catch e - if !isok(e) - suppress_close_error || @error begin - msg = current_exceptions_to_string() - "$(ws.id): Unexpected websocket server error. $msg" - end - end - if !isclosed(ws) - if e isa WebSocketError && e.message isa CloseFrameBody - close(ws, e.message) - else - close(ws, CloseFrameBody(1011, "Unexpected server websocket error")) - end - end - if !isok(e) - rethrow() - end - finally - if !isclosed(ws) - close(ws, CloseFrameBody(1000, "")) - end - end -end - -# Sending messages -isbinary(x) = x isa AbstractVector{UInt8} -istext(x) = x isa AbstractString -opcode(x) = isbinary(x) ? BINARY : TEXT - -function payload(ws, x) - if ws.client - # if we're client, we need to mask the payload, so use our writebuffer for masking - pload = isbinary(x) ? x : codeunits(string(x)) - len = length(pload) - resize!(ws.writebuffer, len) - copyto!(ws.writebuffer, pload) - return ws.writebuffer - else - # if we're server, we just need to make sure payload is AbstractVector{UInt8} - return isbinary(x) ? x : codeunits(string(x)) - end -end - -""" - send(ws::WebSocket, msg) - -Send a message on a websocket connection. If `msg` is an `AbstractString`, -a TEXT websocket message will be sent; if `msg` is an `AbstractVector{UInt8}`, -a BINARY websocket message will be sent. Otherwise, `msg` should be an iterable -of either `AbstractString` or `AbstractVector{UInt8}`, and a fragmented message -will be sent, one frame for each iterated element. - -Control frames can be sent by calling `ping(ws[, data])`, `pong(ws[, data])`, -or `close(ws[, body::WebSockets.CloseFrameBody])`. Calling `close` will initiate -the close sequence and close the underlying connection. -""" -function Sockets.send(ws::WebSocket, x) - @debug "$(ws.id): Writing non-control message" - @require !ws.writeclosed - if !isbinary(x) && !istext(x) - # if x is not single binary or text, then assume it's an iterable of binary or text - # and we'll send fragmented message - first = true - n = 0 - state = iterate(x) - if state === nothing - # x was not binary or text, but is an empty iterable, send single empty frame - x = "" - @goto write_single_frame - end - @debug "$(ws.id): Writing fragmented message" - item, st = state - # we prefetch next state so we know if we're on the last item or not - # so we can appropriately set the FIN bit for the last fragmented frame - nextstate = iterate(x, st) - while true - n += writeframe(ws.io, Frame(nextstate === nothing, first ? opcode(item) : CONTINUATION, ws.client, payload(ws, item))) - first = false - nextstate === nothing && break - item, st = nextstate - nextstate = iterate(x, st) - end - else - # single binary or text frame for message -@label write_single_frame - return writeframe(ws.io, Frame(true, opcode(x), ws.client, payload(ws, x))) - end -end - -# control frames -""" - ping(ws, data=[]) - -Send a PING control frame on a websocket connection. `data` is an optional -body to send with the message. PONG messages are automatically responded -to when a PING message is received by a websocket connection. -""" -function ping(ws::WebSocket, data=UInt8[]) - @require !ws.writeclosed - @debug "$(ws.id): sending ping" - return writeframe(ws.io, Frame(true, PING, ws.client, payload(ws, data))) -end - -""" - pong(ws, data=[]) - -Send a PONG control frame on a websocket connection. `data` is an optional -body to send with the message. Note that PING messages are automatically -responded to internally by the websocket connection with a corresponding -PONG message, but in certain cases, a unidirectional PONG message can be -used as a one-way heartbeat. -""" -function pong(ws::WebSocket, data=UInt8[]) - @require !ws.writeclosed - @debug "$(ws.id): sending pong" - return writeframe(ws.io, Frame(true, PONG, ws.client, payload(ws, data))) -end - -""" - close(ws, body::WebSockets.CloseFrameBody=nothing) - -Initiate a close sequence on a websocket connection. `body` is an optional -`WebSockets.CloseFrameBody` with a status code and optional reason message. -If a CLOSE frame has already been received, then a responding CLOSE frame is sent -and the connection is closed. If a CLOSE frame hasn't already been received, the -CLOSE frame is sent and `receive` is attempted to receive the responding CLOSE -frame. -""" -function Base.close(ws::WebSocket, body::CloseFrameBody=CloseFrameBody(1000, "")) - isclosed(ws) && return - @debug "$(ws.id): Closing websocket" - ws.writeclosed = true - data = Vector{UInt8}(body.message) - prepend!(data, reinterpret(UInt8, [hton(UInt16(body.status))])) - try - writeframe(ws.io, Frame(true, CLOSE, ws.client, data)) - catch - # ignore thrown errors here because we're closing anyway - end - # if we're initiating the close, wait until we receive the - # responding close frame or timeout - if !ws.readclosed - Timer(5) do t - ws.readclosed = true - !ws.client && isopen(ws.io) && close(ws.io) - end - end - while !ws.readclosed - try - receive(ws) - catch - # ignore thrown errors here because we're closing anyway - # but set readclosed so we don't keep trying to read - ws.readclosed = true - end - end - # we either recieved the responding CLOSE frame and readclosed was set - # or there was an error/timeout reading it; in any case, readclosed should be closed now - @assert ws.readclosed - # if we're the server, it's our job to close the underlying socket - !ws.client && isopen(ws.io) && close(ws.io) - return -end - -# Receiving messages - -# returns whether additional frames should be read -# true if fragmented message or a ping/pong frame was handled -@noinline control_len_check(len) = len > 125 && throw(WebSocketError(CloseFrameBody(1002, "Invalid length for control frame"))) -@noinline utf8check(x) = isvalid(x) || throw(WebSocketError(CloseFrameBody(1007, "Invalid UTF-8"))) - -function checkreadframe!(ws::WebSocket, frame::Frame) - if frame.flags.rsv1 || frame.flags.rsv2 || frame.flags.rsv3 - throw(WebSocketError(CloseFrameBody(1002, "Reserved bits set in control frame"))) - end - opcode = frame.flags.opcode - if iscontrol(opcode) && !frame.flags.final - throw(WebSocketError(CloseFrameBody(1002, "Fragmented control frame"))) - end - if opcode == CLOSE - ws.readclosed = true - # reply with Close control frame if we didn't initiate close - if !ws.writeclosed - close(ws) - end - throw(WebSocketError(frame.payload)) - elseif opcode == PING - control_len_check(frame.flags.len) - pong(ws, frame.payload) - return false - elseif opcode == PONG - control_len_check(frame.flags.len) - return false - elseif frame.flags.final && frame.flags.opcode == TEXT && frame.payload isa String - utf8check(frame.payload) - end - return frame.flags.final -end - -_append(x::AbstractVector{UInt8}, y::AbstractVector{UInt8}) = append!(x, y) -_append(x::String, y::String) = string(x, y) - -# low-level for reading a single frame -readframe(ws::WebSocket) = readframe(ws.io, Frame, ws.readbuffer) - -""" - receive(ws::WebSocket) -> Union{String, Vector{UInt8}} - -Receive a message from a websocket connection. Returns a `String` if -the message was TEXT, or a `Vector{UInt8}` if the message was BINARY. -If control frames (ping or pong) are received, they are handled -automatically and a non-control message is waited for. If a CLOSE -message is received, it is responded to and a `WebSocketError` is thrown -with the `WebSockets.CloseFrameBody` as the error value. This error can -be checked with `WebSockets.isok(err)` to see if the closing was "normal" -or if an actual error occurred. For fragmented messages, the incoming -frames will continue to be read until the final fragment is received. -The bodies of each fragment are concatenated into the final message -returned by `receive`. Note that `WebSocket` objects can be iterated, -where each iteration yields a message until the connection is closed. -""" -function receive(ws::WebSocket) - @debug "$(ws.id): Reading message" - @require !ws.readclosed - frame = readframe(ws.io, Frame, ws.readbuffer) - @debug "$(ws.id): Received frame: $frame" - done = checkreadframe!(ws, frame) - # common case of reading single non-control frame - done && return frame.payload - opcode = frame.flags.opcode - iscontrol(opcode) && return receive(ws) - # if we're here, we're reading a fragmented message - payload = frame.payload - while true - frame = readframe(ws.io, Frame, ws.readbuffer, opcode) - @debug "$(ws.id): Received frame: $frame" - done = checkreadframe!(ws, frame) - if !iscontrol(frame.flags.opcode) - payload = _append(payload, frame.payload) - @debug "$(ws.id): payload len = $(length(payload))" - end - done && break - end - payload isa String && utf8check(payload) - @debug "Read message: $(payload[1:min(1024, sizeof(payload))])" - return payload -end - -""" - iterate(ws) - -Continuously call `receive(ws)` on a `WebSocket` connection, with -each iteration yielding a message until the connection is closed. -E.g. -```julia -for msg in ws - # do something with msg -end -``` -""" -function Base.iterate(ws::WebSocket, st=nothing) - isclosed(ws) && return nothing - try - return receive(ws), nothing - catch e - isok(e) && return nothing - rethrow(e) - end -end - -end # module WebSockets diff --git a/src/access_log.jl b/src/access_log.jl index 9bc1f9550..bbef3b9c5 100644 --- a/src/access_log.jl +++ b/src/access_log.jl @@ -34,7 +34,6 @@ end function logfmt_parser(s) s = String(s) - vars = Symbol[] ex = Expr(:call, :print, :io) i = 1 while i <= lastindex(s) @@ -60,14 +59,14 @@ function symbol_mapping(s::Symbol) str = string(s) if (m = match(r"^http_(.+)$", str); m !== nothing) hdr = replace(String(m[1]), '_' => '-') - :(HTTP.header(http.message, $hdr, "-")) + :(something(HTTP.getheader(http.request.headers, $hdr), "-")) elseif (m = match(r"^sent_http_(.+)$", str); m !== nothing) hdr = replace(String(m[1]), '_' => '-') - :(HTTP.header(http.message.response, $hdr, "-")) + :(something(HTTP.getheader(http.response.headers, $hdr), "-")) elseif s === :remote_addr - :(http.stream.peerip) + :(HTTP.remote_address(http.connection)) elseif s === :remote_port - :(http.stream.peerport) + :(HTTP.remote_port(http.connection)) elseif s === :remote_user :("-") # TODO: find from Basic auth... elseif s === :time_iso8601 @@ -90,17 +89,17 @@ function symbol_mapping(s::Symbol) m = symbol_mapping(:request_method) t = symbol_mapping(:request_uri) p = symbol_mapping(:server_protocol) - (m, " ", t, " ", p...) + (m, " ", t, " ", p) elseif s === :request_method - :(http.message.method) + :(http.request.method) elseif s === :request_uri - :(http.message.target) + :(http.request.target) elseif s === :server_protocol - ("HTTP/", :(http.message.version.major), ".", :(http.message.version.minor)) + :(HTTP.http_version(http.connection)) elseif s === :status - :(http.message.response.status) + :(http.response.status) elseif s === :body_bytes_sent - return :(max(0, http.nwritten)) + :(HTTP.bodylen(http.response)) else error("unknown variable in logfmt: $s") end diff --git a/src/client/client.jl b/src/client/client.jl new file mode 100644 index 000000000..51d3ef16b --- /dev/null +++ b/src/client/client.jl @@ -0,0 +1,250 @@ +const DEFAULT_CONNECT_TIMEOUT = 3000 +const DEFAULT_MAX_RETRIES = 10 + +Base.@kwdef struct ClientSettings + scheme::String + host::String + port::UInt32 + allocator::Ptr{aws_allocator} = default_aws_allocator() + bootstrap::Ptr{aws_client_bootstrap} = default_aws_client_bootstrap() + event_loop_group::Ptr{aws_event_loop_group} = default_aws_event_loop_group() + socket_domain::Symbol = :ipv4 + connect_timeout_ms::Int = DEFAULT_CONNECT_TIMEOUT + keep_alive_interval_sec::Int = 0 + keep_alive_timeout_sec::Int = 0 + keep_alive_max_failed_probes::Int = 0 + keepalive::Bool = false + response_first_byte_timeout_ms::Int = 0 + ssl_cert::Union{Nothing, String} = nothing + ssl_key::Union{Nothing, String} = nothing + ssl_capath::Union{Nothing, String} = nothing + ssl_cacert::Union{Nothing, String} = nothing + ssl_insecure::Bool = false + ssl_alpn_list::String = "h2;http/1.1" + proxy_allow_env_var::Bool = true + proxy_connection_type::Union{Nothing, Symbol} = nothing + proxy_host::Union{Nothing, String} = nothing + proxy_port::Union{Nothing, UInt32} = nothing + proxy_ssl_cert::Union{Nothing, String} = nothing + proxy_ssl_key::Union{Nothing, String} = nothing + proxy_ssl_capath::Union{Nothing, String} = nothing + proxy_ssl_cacert::Union{Nothing, String} = nothing + proxy_ssl_insecure::Bool = false + proxy_ssl_alpn_list::String = "h2;http/1.1" + retry_partition::Union{Nothing, String} = nothing + max_retries::Int = DEFAULT_MAX_RETRIES + backoff_scale_factor_ms::Int = 25 + max_backoff_secs::Int = 20 + jitter_mode::aws_exponential_backoff_jitter_mode = AWS_EXPONENTIAL_BACKOFF_JITTER_DEFAULT + retry_timeout_ms::Int = 60000 + initial_bucket_capacity::Int = 500 + max_connections::Int = 512 + max_connection_idle_in_milliseconds::Int = 60000 + connection_acquisition_timeout_ms::Int = 0 + max_pending_connection_acquisitions::Int = 0 + enable_read_back_pressure::Bool = false + http2_prior_knowledge::Bool = false +end + +ClientSettings( + scheme::AbstractString, + host::AbstractString, + port::UInt32; + # HTTP.jl 1.0 compat keywords + connect_timeout=nothing, + connect_timeout_ms::Int=DEFAULT_CONNECT_TIMEOUT, + retry::Bool=true, + retries::Integer=DEFAULT_MAX_RETRIES, + max_retries::Integer=DEFAULT_MAX_RETRIES, + require_ssl_verification::Bool=true, + ssl_insecure::Bool=false, + kw...) = + ClientSettings(; + scheme=String(scheme), + host=String(host), + port=port, + connect_timeout_ms=(connect_timeout !== nothing ? connect_timeout * 1000 : connect_timeout_ms), + max_retries=(retry ? (retries != DEFAULT_MAX_RETRIES ? retries : max_retries) : 0), + ssl_insecure=(!require_ssl_verification || ssl_insecure), + kw...) + +# make a new ClientSettings object from an existing one w/ just different url values +@generated function ClientSettings(cs::ClientSettings, scheme::AbstractString, host::AbstractString, port::Integer) + ex = :(ClientSettings(String(scheme), String(host), port % UInt32)) + for i = 4:fieldcount(ClientSettings) + push!(ex.args, :(getfield(cs, $(Meta.quot(fieldname(ClientSettings, i)))))) + end + return ex +end + +mutable struct Client + settings::ClientSettings + socket_options::aws_socket_options + tls_options::Union{Nothing, aws_tls_connection_options} + # only 1 of proxy_options or proxy_env_settings is set + proxy_options::Union{Nothing, aws_http_proxy_options} + proxy_env_settings::Union{Nothing, proxy_env_var_settings} + retry_options::aws_standard_retry_options + retry_strategy::Ptr{aws_retry_strategy} + conn_manager_opts::aws_http_connection_manager_options + connection_manager::Ptr{aws_http_connection_manager} + + Client() = new() +end + +Client(scheme::AbstractString, host::AbstractString, port::Integer; kw...) = Client(ClientSettings(scheme, host, port % UInt32; kw...)) + +function Client(cs::ClientSettings) + client = Client() + client.settings = cs + # socket options + client.socket_options = aws_socket_options( + AWS_SOCKET_STREAM, # socket type + cs.socket_domain == :ipv4 ? AWS_SOCKET_IPV4 : AWS_SOCKET_IPV6, # socket domain + AWS_SOCKET_IMPL_PLATFORM_DEFAULT, # aws_socket_impl_type + cs.connect_timeout_ms, + cs.keep_alive_interval_sec, + cs.keep_alive_timeout_sec, + cs.keep_alive_max_failed_probes, + cs.keepalive, + ntuple(x -> Cchar(0), 16) # network_interface_name + ) + # tls options + if cs.scheme == "https" || cs.scheme == "wss" + client.tls_options = LibAwsIO.tlsoptions(cs.host; + cs.ssl_cert, + cs.ssl_key, + cs.ssl_capath, + cs.ssl_cacert, + cs.ssl_insecure, + cs.ssl_alpn_list + ) + else + client.tls_options = nothing + end + # proxy options + if cs.proxy_host !== nothing && cs.proxy_port !== nothing + client.proxy_options = aws_http_proxy_options( + cs.proxy_connection_type == :forward ? AWS_HPCT_HTTP_FORWARD : AWS_HPCT_HTTP_TUNNEL, + aws_byte_cursor_from_c_str(cs.proxy_host), + cs.proxy_port % UInt32, + cs.proxy_ssl_cert === nothing ? C_NULL : LibAwsIO.tlsoptions(cs.proxy_host; + cs.proxy_ssl_cert, + cs.proxy_ssl_key, + cs.proxy_ssl_capath, + cs.proxy_ssl_cacert, + cs.proxy_ssl_insecure, + cs.proxy_ssl_alpn_list + ), + #TODO: support proxy_strategy + C_NULL, # proxy_strategy::Ptr{aws_http_proxy_strategy} + 0, # auth_type::aws_http_proxy_authentication_type + aws_byte_cursor_from_c_str(""), # auth_username::aws_byte_cursor + aws_byte_cursor_from_c_str(""), # auth_password::aws_byte_cursor + ) + elseif cs.proxy_allow_env_var + client.proxy_env_settings = proxy_env_var_settings( + AWS_HPEV_ENABLE, + cs.proxy_connection_type == :forward ? AWS_HPCT_HTTP_FORWARD : AWS_HPCT_HTTP_TUNNEL, + cs.proxy_ssl_cert === nothing ? C_NULL : LibAwsIO.tlsoptions(cs.proxy_host; + cs.proxy_ssl_cert, + cs.proxy_ssl_key, + cs.proxy_ssl_capath, + cs.proxy_ssl_cacert, + cs.proxy_ssl_insecure, + cs.proxy_ssl_alpn_list + ) + ) + else + client.proxy_options = nothing + end + # retry strategy + exp_back_opts = aws_exponential_backoff_retry_options( + cs.event_loop_group, + cs.max_retries, + cs.backoff_scale_factor_ms, + cs.max_backoff_secs, + cs.jitter_mode, + C_NULL, # generate_random + C_NULL, # generate_random_impl + C_NULL, # generate_random_user_data + C_NULL, # shutdown_options::Ptr{aws_shutdown_callback_options} + ) + client.retry_options = aws_standard_retry_options( + exp_back_opts, + cs.initial_bucket_capacity + ) + client.retry_strategy = aws_retry_strategy_new_standard(cs.allocator, FieldRef(client, :retry_options)) + client.retry_strategy == C_NULL && aws_throw_error() + client.conn_manager_opts = aws_http_connection_manager_options( + cs.bootstrap, + typemax(Csize_t), # initial_window_size::Csize_t + pointer(FieldRef(client, :socket_options)), + cs.response_first_byte_timeout_ms, + (cs.scheme == "https" || cs.scheme == "wss") ? pointer(FieldRef(client, :tls_options)) : C_NULL, + cs.http2_prior_knowledge, + C_NULL, # monitoring_options::Ptr{aws_http_connection_monitoring_options} + aws_byte_cursor_from_c_str(cs.host), + cs.port % UInt32, + C_NULL, # initial_settings_array::Ptr{aws_http2_setting} + 0, # num_initial_settings::Csize_t + 0, # max_closed_streams::Csize_t + false, # http2_conn_manual_window_management::Bool + client.proxy_options === nothing ? C_NULL : pointer(FieldRef(client, :proxy_options)), # proxy_options::Ptr{aws_http_proxy_options} + client.proxy_env_settings === nothing ? C_NULL : pointer(FieldRef(client, :proxy_env_settings)), # proxy_env_settings::Ptr{proxy_env_var_settings} + cs.max_connections, # max_connections::Csize_t, 512 + C_NULL, # shutdown_complete_user_data::Ptr{Cvoid} + C_NULL, # shutdown_complete_callback::Ptr{aws_http_connection_manager_shutdown_complete_fn} + cs.enable_read_back_pressure, # enable_read_back_pressure::Bool + cs.max_connection_idle_in_milliseconds, + cs.connection_acquisition_timeout_ms, + cs.max_pending_connection_acquisitions, + C_NULL, # network_interface_names_array + 0, # num_network_interface_names + ) + client.connection_manager = aws_http_connection_manager_new(cs.allocator, FieldRef(client, :conn_manager_opts)) + client.connection_manager == C_NULL && aws_throw_error() + + finalizer(client) do x + if x.connection_manager != C_NULL + aws_http_connection_manager_release(x.connection_manager) + x.connection_manager = C_NULL + end + if x.retry_strategy != C_NULL + aws_retry_strategy_release(x.retry_strategy) + x.retry_strategy = C_NULL + end + end + return client +end + +#TODO: this should probably be a LRU cache to not grow indefinitely +struct Clients + lock::ReentrantLock + clients::Dict{ClientSettings, Client} +end + +Clients() = Clients(ReentrantLock(), Dict{ClientSettings, Client}()) + +const CLIENTS = Clients() + +function getclient(key::ClientSettings, clients::Clients=CLIENTS) + Base.@lock clients.lock begin + if haskey(clients.clients, key) + return clients.clients[key] + else + client = Client(key) + clients.clients[key] = client + return client + end + end +end + +function close_all_clients!(clients::Clients=CLIENTS) + Base.@lock clients.lock begin + for client in values(clients.clients) + finalize(client) + end + empty!(clients.clients) + end +end diff --git a/src/client/connection.jl b/src/client/connection.jl new file mode 100644 index 000000000..17071a053 --- /dev/null +++ b/src/client/connection.jl @@ -0,0 +1,26 @@ +const on_setup = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_setup(conn, error_code, fut_ptr) + fut = unsafe_pointer_to_objref(fut_ptr) + if error_code == AWS_IO_DNS_INVALID_NAME# || error_code == AWS_IO_TLS_ERROR_NEGOTIATION_FAILURE + notify(fut, DontRetry(CapturedException(aws_error(error_code), Base.backtrace()))) + elseif error_code != 0 + notify(fut, CapturedException(aws_error(error_code), Base.backtrace())) + else + notify(fut, conn) + end + return +end + +function with_connection(f::Function, client::Client) + fut = Future{Ptr{aws_http_connection}}() + GC.@preserve fut begin + aws_http_connection_manager_acquire_connection(client.connection_manager, on_setup[], pointer_from_objref(fut)) + connection = wait(fut) + end + try + return f(connection) + finally + aws_http_connection_manager_release_connection(client.connection_manager, connection) + end +end diff --git a/src/client/makerequest.jl b/src/client/makerequest.jl new file mode 100644 index 000000000..298363568 --- /dev/null +++ b/src/client/makerequest.jl @@ -0,0 +1,73 @@ +get(a...; kw...) = request("GET", a...; kw...) +put(a...; kw...) = request("PUT", a...; kw...) +post(a...; kw...) = request("POST", a...; kw...) +delete(a...; kw...) = request("DELETE", a...; kw...) +patch(a...; kw...) = request("PATCH", a...; kw...) +head(a...; kw...) = request("HEAD", a...; kw...) +options(a...; kw...) = request("OPTIONS", a...; kw...) + +const COOKIEJAR = CookieJar() + +_something(x, y) = x === nothing ? y : x + +# main entrypoint for making an HTTP request +# can provide method, url, headers, body, along with various keyword arguments +function request(method, url, h=Header[], b::RequestBodyTypes=nothing; + allocator=default_aws_allocator(), + headers=h, + body::RequestBodyTypes=b, + chunkedbody=nothing, + username=nothing, + password=nothing, + bearer=nothing, + query=nothing, + client::Union{Nothing, Client}=nothing, + # redirect options + redirect=true, + redirect_limit=3, + redirect_method=nothing, + forwardheaders=true, + # cookie options + cookies=true, + cookiejar::CookieJar=COOKIEJAR, + # response options + response_stream=nothing, # compat + response_body=response_stream, + decompress::Union{Nothing, Bool}=nothing, + status_exception::Bool=true, + readtimeout::Int=0, # only supported for HTTP 1.1, not HTTP 2 (current aws limitation) + retry_non_idempotent::Bool=false, + modifier=nothing, + verbose=0, + # only client keywords in catch-all + kw...) + uri = parseuri(url, query, allocator) + return with_redirect(allocator, method, uri, headers, body, redirect, redirect_limit, redirect_method, forwardheaders) do method, uri, headers, body + reqclient = @something(client, getclient(ClientSettings(scheme(uri), host(uri), getport(uri); allocator=allocator, kw...)))::Client + with_retry_token(reqclient) do + resp = with_connection(reqclient) do conn + http2 = aws_http_connection_get_version(conn) == AWS_HTTP_VERSION_2 + path = resource(uri) + with_request(reqclient, method, path, headers, body, chunkedbody, decompress, (username !== nothing && password !== nothing) ? "$username:$password" : userinfo(uri), bearer, modifier, http2, cookies, cookiejar, verbose) do req + if response_body isa AbstractVector{UInt8} + ref = Ref(1) + GC.@preserve ref begin + on_stream_response_body = BufferOnResponseBody(response_body, Base.unsafe_convert(Ptr{Int}, ref)) + with_stream(conn, req, chunkedbody, on_stream_response_body, decompress, http2, readtimeout, allocator) + end + elseif response_body isa IO + on_stream_response_body = IOOnResponseBody(response_body) + with_stream(conn, req, chunkedbody, on_stream_response_body, decompress, http2, readtimeout, allocator) + else + with_stream(conn, req, chunkedbody, response_body, decompress, http2, readtimeout, allocator) + end + end + end + # status error check + if status_exception && iserror(resp) + throw(StatusError(method, uri, resp)) + end + return resp + end + end +end diff --git a/src/client/redirects.jl b/src/client/redirects.jl new file mode 100644 index 000000000..e260939ec --- /dev/null +++ b/src/client/redirects.jl @@ -0,0 +1,79 @@ +const SENSITIVE_HEADERS = Set([ + "Authorization", + "Www-Authenticate", + "Cookie", + "Cookie2" +]) + +function isdomainorsubdomain(sub, parent) + sub == parent && return true + endswith(sub, parent) || return false + return sub[length(sub)-length(parent)] == '.' +end + +function newmethod(request_method, response_status, redirect_method) + # using https://everything.curl.dev/http/redirects#get-or-post as a reference + # also reference: https://github.com/curl/curl/issues/5237#issuecomment-618293609 + if response_status == 307 || response_status == 308 + # specific status codes that indicate an identical request should be made to new location + return request_method + elseif response_status == 303 + # 303 means it's a new/different URI, so only GET allowed + return "GET" + elseif redirect_method == :same + return request_method + elseif redirect_method !== nothing && String(redirect_method) in ("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH") + return redirect_method + elseif request_method == "HEAD" + # Unless otherwise specified (e.g. with `redirect_method`), be conservative and keep the + # same method, see: + # + # * + # * + # + # Turning a HEAD request through a redirect may be undesired: + # . + return request_method + end + return "GET" +end + +function with_redirect(f, allocator, method, uri, headers=nothing, body=nothing, redirect::Bool=true, redirect_limit::Int=3, redirect_method=nothing, forwardheaders::Bool=true) + if !redirect || redirect_limit == 0 + # no redirecting + return f(method, uri, headers, body) + end + count = 0 + while true + ret = f(method, uri, headers, body) + resp = getresponse(ret) + if (count == redirect_limit || !isredirect(resp) || (location = getheader(resp.headers, "Location")) == "") + return ret + end + + # follow redirect + olduri = uri + newuri = resolvereference(makeuri(uri), location) + uri = parseuri(newuri, nothing, allocator) + method = newmethod(method, resp.status, redirect_method) + body = method == "GET" ? nothing : body + if forwardheaders + headers = filter(headers) do (header, _) + # false return values are filtered out + if headereq(header, "host") + return false + elseif any(x -> headereq(x, header), SENSITIVE_HEADERS) && !isdomainorsubdomain(host(uri), host(olduri)) + return false + elseif method == "GET" && (headereq(header, "content-type") || headereq(header, "content-length")) + return false + else + return true + end + end + else + headers = Header[] + end + count += 1 + end + @assert false "Unreachable!" +end \ No newline at end of file diff --git a/src/client/request.jl b/src/client/request.jl new file mode 100644 index 000000000..1fc573f94 --- /dev/null +++ b/src/client/request.jl @@ -0,0 +1,78 @@ +const USER_AGENT = Ref{Union{String, Nothing}}("HTTP.jl/$VERSION") + +""" + setuseragent!(x::Union{String, Nothing}) + +Set the default User-Agent string to be used in each HTTP request. +Can be manually overridden by passing an explicit `User-Agent` header. +Setting `nothing` will prevent the default `User-Agent` header from being passed. +""" +function setuseragent!(x::Union{String, Nothing}) + USER_AGENT[] = x + return +end + +function with_request(f::Function, client::Client, method, path, headers=nothing, body=nothing, chunkedbody=nothing, decompress::Union{Nothing, Bool}=nothing, userinfo=nothing, bearer=nothing, modifier=nothing, http2::Bool=false, cookies=true, cookiejar=COOKIEJAR, verbose=false) + # create request + req = Request(method, path, headers, nothing, http2, client.settings.allocator) + # add headers to request + h = req.headers + if http2 + setscheme(h, client.settings.scheme) + setauthority(h, client.settings.host) + else + setheader(h, "host", client.settings.host) + end + setheaderifabsent(h, "accept", "*/*") + setheaderifabsent(h, "user-agent", something(USER_AGENT[], "-")) + if decompress === nothing || decompress + setheaderifabsent(h, "accept-encoding", "gzip") + end + if userinfo !== nothing + setheaderifabsent(h, "authorization", "Basic $(base64encode(unescapeuri(userinfo)))") + end + if bearer !== nothing + setheaderifabsent(h, "authorization", "Bearer $bearer") + end + if !http2 && chunkedbody !== nothing + setheaderifabsent(h, "transfer-encoding", "chunked") + end + if headers !== nothing + for (k, v) in headers + addheader(h, k, v) + end + end + if cookies === true || (cookies isa AbstractDict && !isempty(cookies)) + cookiestosend = Cookies.getcookies!(cookiejar, client.settings.scheme, client.settings.host, req.path) + if !(cookies isa Bool) + for (name, value) in cookies + push!(cookiestosend, Cookie(name, value)) + end + end + if !isempty(cookiestosend) + setheader(req.headers, "Cookie", stringify("", cookiestosend)) + end + end + # modifier + if modifier !== nothing + newbody = modifier(req, body) + if newbody !== nothing + setinputstream!(req, newbody) + else + setinputstream!(req, body) + end + elseif body !== nothing + try + setinputstream!(req, body) + catch e + @error "Failed to set input stream" exception=(e, catch_backtrace()) + end + end + # call user function + verbose > 0 && print_request(stdout, req) + ret = f(req) + resp = getresponse(ret) + verbose > 0 && print_response(stdout, resp) + cookies === false || Cookies.setcookies!(cookiejar, client.settings.scheme, client.settings.host, req.path, resp.headers) + return ret +end diff --git a/src/client/retry.jl b/src/client/retry.jl new file mode 100644 index 000000000..94b89e7bd --- /dev/null +++ b/src/client/retry.jl @@ -0,0 +1,89 @@ +struct DontRetry{T} <: Exception + error::T +end + +Base.showerror(io::IO, e::DontRetry) = print(io, e.error) + +struct StreamError{T} <: Exception + error::Exception + stream::T # Stream +end + +Base.showerror(io::IO, e::StreamError) = print(io, e.error) + +const on_acquired = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_acquired(retry_strategy, error_code, retry_token, fut_ptr) + fut = unsafe_pointer_to_objref(fut_ptr) + if error_code != 0 + notify(fut, DontRetry(CapturedException(aws_error(error_code), Base.backtrace()))) + else + notify(fut, retry_token) + end + return +end + +const retry_ready = Ref{Ptr{Cvoid}}(C_NULL) + +function c_retry_ready(token, error_code::Cint, fut_ptr) + fut = unsafe_pointer_to_objref(fut_ptr) + if error_code != 0 + notify(fut, DontRetry(CapturedException(aws_error(error_code), Base.backtrace()))) + else + notify(fut, token) + end + return +end + +function with_retry_token(f::Function, client::Client) + # If max_retries is 0, we don't need to bother with any retrying + client.settings.max_retries == 0 && return f() + retry_partition = client.settings.retry_partition === nothing ? C_NULL : aws_byte_cursor_from_c_str(client.settings.retry_partition) + fut = Future{Ptr{aws_retry_token}}() + GC.@preserve fut begin + if aws_retry_strategy_acquire_retry_token(client.retry_strategy, retry_partition, on_acquired[], pointer_from_objref(fut), client.settings.retry_timeout_ms) != 0 + aws_throw_error() + end + token = wait(fut) + end + try + while true + try + ret = f() + aws_retry_token_record_success(token) + return ret + catch e + stream = nothing + if e isa StreamError + stream = e.stream + e = e.error + end + if e isa DontRetry + if stream !== nothing && iserror(stream.response.status) && stream.bufferstream !== nothing + # for error responses, we need to commit the temporary body buffer + stream.response.body = readavailable(stream.bufferstream) + end + throw(e.error) + end + # note we assume any error that wasn't wrapped in DontRetry is retryable + retryReady = Future{Ptr{aws_retry_token}}() + GC.@preserve retryReady begin + if aws_retry_strategy_schedule_retry( + token, + #TODO: use different error types? + AWS_RETRY_ERROR_TYPE_TRANSIENT, + retry_ready[], + pointer_from_objref(retryReady) + ) != 0 + #TODO: do we need to commit a previous error body to the response here? + aws_throw_error() + end + #TODO: should we wrap this in try-catch to commit a previous stream bufferstream to the response body? + token = wait(retryReady) + end + end + end + finally + aws_retry_token_release(token) + end +end \ No newline at end of file diff --git a/src/client/stream.jl b/src/client/stream.jl new file mode 100644 index 000000000..90dfe5e96 --- /dev/null +++ b/src/client/stream.jl @@ -0,0 +1,229 @@ +const on_response_headers = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_response_headers(aws_stream_ptr, header_block, header_array::Ptr{aws_http_header}, num_headers, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + headers = stream.response.headers + addheaders(headers, header_array, num_headers) + return Cint(0) +end + +writebuf(body, maxsize=length(body) == 0 ? typemax(Int64) : length(body)) = Base.GenericIOBuffer{AbstractVector{UInt8}}(body, true, true, true, false, maxsize) + +const on_response_header_block_done = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_response_header_block_done(aws_stream_ptr, header_block, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + if aws_http_stream_get_incoming_response_status(aws_stream_ptr, FieldRef(stream, :status)) != 0 + return aws_raise_error(aws_last_error()) + end + stream.response.status = stream.status + # if this is the end of the main header block, prepare our response body to be written to, otherwise return + if header_block != AWS_HTTP_HEADER_BLOCK_MAIN + return Cint(0) + end + if stream.decompress !== false + val = getheader(stream.response.headers, "content-encoding") + stream.decompress = val !== nothing && val == "gzip" + end + return Cint(0) +end + +const on_response_body = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_response_body(aws_stream_ptr, data::Ptr{aws_byte_cursor}, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + bc = unsafe_load(data) + stream.response.metrics.response_body_length += bc.len + if stream.decompress + if stream.gzipstream === nothing + stream.bufferstream = b = Base.BufferStream() + stream.gzipstream = g = CodecZlib.GzipDecompressorStream(b) + unsafe_write(g, bc.ptr, bc.len) + else + unsafe_write(stream.gzipstream, bc.ptr, bc.len) + end + else + if stream.bufferstream === nothing + stream.bufferstream = b = Base.BufferStream() + unsafe_write(b, bc.ptr, bc.len) + else + unsafe_write(stream.bufferstream, bc.ptr, bc.len) + end + end + return Cint(0) +end + +const on_metrics = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_metrics(aws_stream_ptr, metrics::Ptr{aws_http_stream_metrics}, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + m = unsafe_load(metrics) + if m.send_start_timestamp_ns != -1 + stream.response.metrics.stream_metrics = m + end + return +end + +const on_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_complete(aws_stream_ptr, error_code, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + if stream.gzipstream !== nothing + close(stream.gzipstream) + end + if stream.bufferstream !== nothing + close(stream.bufferstream) + end + if error_code != 0 + notify(stream.fut, CapturedException(aws_error(error_code), Base.backtrace())) + else + notify(stream.fut, nothing) + end + return +end + +const on_destroy = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_destroy(stream) + return +end + +if !@isdefined aws_websocket_server_upgrade_options + const aws_websocket_server_upgrade_options = Ptr{Cvoid} +end + +mutable struct Stream{T} + allocator::Ptr{aws_allocator} + decompress::Union{Nothing, Bool} + http2::Bool + status::Cint # used as a ref + fut::Future{Nothing} + chunk::Union{Nothing, InputStream} + final_chunk_written::Bool + bufferstream::Union{Nothing, Base.BufferStream} + gzipstream::Union{Nothing, CodecZlib.GzipDecompressorStream} + # remaining fields are initially undefined + ptr::Ptr{aws_http_stream} + connection::T # Connection{F, S} (in servers.jl) + request_options::aws_http_make_request_options + response::Response + method::aws_byte_cursor + path::aws_byte_cursor + request_handler_options::aws_http_request_handler_options + request::Request + http2_stream_write_data_options::aws_http2_stream_write_data_options + chunk_options::aws_http1_chunk_options + websocket_options::aws_websocket_server_upgrade_options + Stream{T}(allocator, decompress, http2) where {T} = new{T}(allocator, decompress, http2, 0, Future{Nothing}(), nothing, false, nothing, nothing) +end + +Base.hash(s::Stream, h::UInt) = hash(s.ptr, h) + +const on_stream_write_on_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_stream_write_on_complete(aws_stream_ptr, error_code, fut_ptr) + fut = unsafe_pointer_to_objref(fut_ptr) + if error_code != 0 + notify(fut, CapturedException(aws_error(error_code), Base.backtrace())) + else + notify(fut, nothing) + end + return +end + +function writechunk(s::Stream, chunk::RequestBodyTypes) + @assert (isdefined(s, :response) && + isdefined(s.response, :request) && + s.response.request.method in ("POST", "PUT", "PATCH")) "write is only allowed for POST, PUT, and PATCH requests" + s.chunk = InputStream(s.allocator, chunk) + fut = Future{Nothing}() + if s.http2 + s.http2_stream_write_data_options = aws_http2_stream_write_data_options( + s.chunk.ptr, + chunk == "", + on_stream_write_on_complete[], + pointer_from_objref(fut) + ) + aws_http2_stream_write_data(s.ptr, FieldRef(s, :http2_stream_write_data_options)) != 0 && aws_throw_error() + else + s.chunk_options = aws_http1_chunk_options( + s.chunk.ptr, + s.chunk.bodylen, + C_NULL, + 0, + on_stream_write_on_complete[], + pointer_from_objref(fut) + ) + aws_http1_stream_write_chunk(s.ptr, FieldRef(s, :chunk_options)) != 0 && aws_throw_error() + end + wait(fut) + return s.chunk.bodylen +end + +function with_stream(conn::Ptr{aws_http_connection}, req::Request, chunkedbody, on_stream_response_body, decompress, http2, readtimeout, allocator) + stream = Stream{Nothing}(allocator, decompress, http2) + if on_stream_response_body !== nothing + stream.bufferstream = Base.BufferStream() + end + GC.@preserve stream begin + stream.request_options = aws_http_make_request_options( + 1, + req.ptr, + pointer_from_objref(stream), + on_response_headers[], + on_response_header_block_done[], + on_response_body[], + on_metrics[], + on_complete[], + on_destroy[], + http2 && chunkedbody !== nothing, # http2_use_manual_data_writes + readtimeout * 1000 # response_first_byte_timeout_ms + ) + stream_ptr = aws_http_connection_make_request(conn, FieldRef(stream, :request_options)) + stream_ptr == C_NULL && aws_throw_error() + stream.ptr = stream_ptr + http2 = aws_http_connection_get_version(conn) == AWS_HTTP_VERSION_2 + stream.response = resp = Response(0, nothing, nothing, http2, allocator) + resp.metrics = RequestMetrics() + resp.request = req + try + aws_http_stream_activate(stream_ptr) != 0 && aws_throw_error() + # write chunked body if provided + if chunkedbody !== nothing + foreach(chunk -> writechunk(stream, chunk), chunkedbody) + # write final chunk + writechunk(stream, "") + end + if on_stream_response_body !== nothing + try + while !eof(stream.bufferstream) + on_stream_response_body(resp, _readavailable(stream.bufferstream)) + end + catch e + rethrow(DontRetry(e)) + end + else + wait(stream.fut) + if stream.bufferstream !== nothing + resp.body = _readavailable(stream.bufferstream) + else + resp.body = UInt8[] + end + end + return resp + finally + aws_http_stream_release(stream_ptr) + end + end # GC.@preserve +end + +# can be removed once https://github.com/JuliaLang/julia/pull/57211 is fully released +function _readavailable(this::Base.BufferStream) + bytes = lock(this.cond) do + Base.wait_readnb(this, 1) + buf = this.buffer + @assert buf.seekable == false + take!(buf) + end + return bytes +end \ No newline at end of file diff --git a/src/clientlayers/ConnectionRequest.jl b/src/clientlayers/ConnectionRequest.jl deleted file mode 100644 index 62dcd5535..000000000 --- a/src/clientlayers/ConnectionRequest.jl +++ /dev/null @@ -1,227 +0,0 @@ -module ConnectionRequest - -using URIs, Sockets, Base64, ConcurrentUtilities, ExceptionUnwrapping -import MbedTLS -import OpenSSL -using ..Messages, ..IOExtras, ..Connections, ..Streams, ..Exceptions -import ..SOCKET_TYPE_TLS - -islocalhost(host::AbstractString) = host == "localhost" || host == "127.0.0.1" || host == "::1" || host == "0000:0000:0000:0000:0000:0000:0000:0001" || host == "0:0:0:0:0:0:0:1" - -# hasdotsuffix reports whether s ends in "."+suffix. -hasdotsuffix(s, suffix) = endswith(s, "." * suffix) - -function isnoproxy(host::AbstractString) - for x in NO_PROXY - (hasdotsuffix(host, x) || (host == x)) && return true - end - return false -end - -const NO_PROXY = String[] - -function __init__() - # check for no_proxy environment variable - if haskey(ENV, "no_proxy") - for x in split(ENV["no_proxy"], ","; keepempty=false) - push!(NO_PROXY, startswith(x, ".") ? x[2:end] : x) - end - end - return -end - -function getproxy(scheme, host) - (isnoproxy(host) || islocalhost(host)) && return nothing - if scheme == "http" && (p = get(ENV, "http_proxy", ""); !isempty(p)) - return p - elseif scheme == "http" && (p = get(ENV, "HTTP_PROXY", ""); !isempty(p)) - return p - elseif scheme == "https" && (p = get(ENV, "https_proxy", ""); !isempty(p)) - return p - elseif scheme == "https" && (p = get(ENV, "HTTPS_PROXY", ""); !isempty(p)) - return p - end - return nothing -end - -export connectionlayer - -const CLOSE_IMMEDIATELY = Ref{Bool}(false) - -""" - connectionlayer(handler) -> handler - -Retrieve an `IO` connection from the Connections. - -Close the connection if the request throws an exception. -Otherwise leave it open so that it can be reused. -""" -function connectionlayer(handler) - return function connections(req; proxy=getproxy(req.url.scheme, req.url.host), socket_type::Type=TCPSocket, socket_type_tls::Union{Nothing, Type}=nothing, readtimeout::Int=0, connect_timeout::Int=30, logerrors::Bool=false, logtag=nothing, closeimmediately::Bool=CLOSE_IMMEDIATELY[], kw...) - local io, stream - if proxy !== nothing - target_url = req.url - url = URI(proxy) - if target_url.scheme == "http" - req.target = string(target_url) - end - - userinfo = unescapeuri(url.userinfo) - if !isempty(userinfo) && !hasheader(req.headers, "Proxy-Authorization") - @debug "Adding Proxy-Authorization: Basic header." - setheader(req.headers, "Proxy-Authorization" => "Basic $(base64encode(userinfo))") - end - else - url = target_url = req.url - end - - connect_timeout = connect_timeout == 0 && readtimeout > 0 ? readtimeout : connect_timeout - IOType = sockettype(url, socket_type, socket_type_tls, get(kw, :sslconfig, nothing)) - start_time = time() - try - io = newconnection(IOType, url.host, url.port; readtimeout=readtimeout, connect_timeout=connect_timeout, kw...) - catch e - if logerrors - @error current_exceptions_to_string() type=Symbol("HTTP.ConnectError") method=req.method url=req.url context=req.context logtag=logtag - end - req.context[:connect_errors] = get(req.context, :connect_errors, 0) + 1 - throw(ConnectError(string(url), e)) - finally - req.context[:connect_duration_ms] = get(req.context, :connect_duration_ms, 0.0) + (time() - start_time) * 1000 - end - - shouldreuse = !(target_url.scheme in ("ws", "wss")) && !closeimmediately - try - if proxy !== nothing && target_url.scheme in ("https", "wss", "ws") - shouldreuse = false - # tunnel request - if target_url.scheme in ("https", "wss") - target_url = URI(target_url, port=443) - elseif target_url.scheme in ("ws", ) && target_url.port == "" - target_url = URI(target_url, port=80) # if there is no port info, connect_tunnel will fail - end - r = if readtimeout > 0 - try_with_timeout(readtimeout) do _ - connect_tunnel(io, target_url, req) - end - else - connect_tunnel(io, target_url, req) - end - if r.status != 200 - close(io) - return r - end - if target_url.scheme in ("https", "wss") - InnerIOType = sockettype(target_url, socket_type, socket_type_tls, get(kw, :sslconfig, nothing)) - io = Connections.sslupgrade(InnerIOType, io, target_url.host; readtimeout=readtimeout, kw...) - end - req.headers = filter(x->x.first != "Proxy-Authorization", req.headers) - end - - stream = Stream(req.response, io) - return handler(stream; readtimeout=readtimeout, logerrors=logerrors, logtag=logtag, kw...) - catch e - shouldreuse = false - # manually unwrap CompositeException since it's not defined as a "wrapper" exception by ExceptionUnwrapping - while e isa CompositeException - e = e.exceptions[1] - end - root_err = ExceptionUnwrapping.unwrap_exception_to_root(e) - # don't log if it's an HTTPError since we should have already logged it - if logerrors && root_err isa StatusError - @error current_exceptions_to_string() type=Symbol("HTTP.StatusError") method=req.method url=req.url context=req.context logtag=logtag - end - if logerrors && !ExceptionUnwrapping.has_wrapped_exception(e, HTTPError) - @error current_exceptions_to_string() type=Symbol("HTTP.ConnectionRequest") method=req.method url=req.url context=req.context logtag=logtag - end - @debug "❗️ ConnectionLayer $root_err. Closing: $io" - if @isdefined(stream) && stream.nwritten == -1 - # we didn't write anything, so don't need to worry about - # idempotency of the request - req.context[:nothingwritten] = true - end - root_err isa HTTPError || throw(RequestError(req, root_err)) - throw(root_err) - finally - releaseconnection(io, shouldreuse; kw...) - if !shouldreuse - @try Base.IOError close(io) - end - end - end -end - -function sockettype(url::URI, socket_type_tcp, socket_type_tls, sslconfig) - if url.scheme in ("wss", "https") - tls_socket_type(socket_type_tls, sslconfig) - else - socket_type_tcp - end -end - -""" - tls_socket_type(socket_type_tls, sslconfig)::Type - -Find the best TLS socket type, given the values of these keyword arguments. - -If both are `nothing` then we use the global default: `HTTP.SOCKET_TYPE_TLS[]`. -If both are not `nothing` then they must agree: -`sslconfig` must be of the right type to configure `socket_type_tls` or we throw an `ArgumentError`. -""" -function tls_socket_type(socket_type_tls::Union{Nothing, Type}, - sslconfig::Union{Nothing, MbedTLS.SSLConfig, OpenSSL.SSLContext} - )::Type - - socket_type_matching_sslconfig = - if sslconfig isa MbedTLS.SSLConfig - MbedTLS.SSLContext - elseif sslconfig isa OpenSSL.SSLContext - OpenSSL.SSLStream - else - nothing - end - - if socket_type_tls === socket_type_matching_sslconfig - # Use the global default TLS socket if they're both nothing, or use - # what they both specify if they're not nothing. - isnothing(socket_type_tls) ? SOCKET_TYPE_TLS[] : socket_type_tls - # If either is nothing, use the other one. - elseif isnothing(socket_type_tls) - socket_type_matching_sslconfig - elseif isnothing(socket_type_matching_sslconfig) - socket_type_tls - else - # If they specify contradictory types, throw an error. - # Error thrown in noinline closure to avoid speed penalty in common case - @noinline function err(socket_type_tls, sslconfig) - msg = """ - Incompatible values for keyword args `socket_type_tls` and `sslconfig`: - socket_type_tls=$socket_type_tls - typeof(sslconfig)=$(typeof(sslconfig)) - - Make them match or provide only one of them. - - the socket type MbedTLS.SSLContext is configured by MbedTLS.SSLConfig - - the socket type OpenSSL.SSLStream is configured by OpenSSL.SSLContext""" - throw(ArgumentError(msg)) - end - err(socket_type_tls, sslconfig) - end -end - -function connect_tunnel(io, target_url, req) - target = "$(URIs.hoststring(target_url.host)):$(target_url.port)" - @debug "📡 CONNECT HTTPS tunnel to $target" - headers = Dict("Host" => target) - if (auth = header(req, "Proxy-Authorization"); !isempty(auth)) - headers["Proxy-Authorization"] = auth - end - request = Request("CONNECT", target, headers) - # @debug "connect_tunnel: writing headers" - writeheaders(io, request) - # @debug "connect_tunnel: reading headers" - readheaders(io, request.response) - # @debug "connect_tunnel: done reading headers" - return request.response -end - -end # module ConnectionRequest diff --git a/src/clientlayers/CookieRequest.jl b/src/clientlayers/CookieRequest.jl deleted file mode 100644 index cb581f486..000000000 --- a/src/clientlayers/CookieRequest.jl +++ /dev/null @@ -1,52 +0,0 @@ -module CookieRequest - -using Dates, URIs -using ..Cookies, ..Messages, ..Strings - -# default global cookie jar -const COOKIEJAR = CookieJar() - -export cookielayer, COOKIEJAR - -""" - cookielayer(handler) -> handler - -Check for host-appropriate cookies to include in the outgoing request -from the `cookiejar` keyword argument (by default, a global cookiejar is used). -Store "Set-Cookie" cookies from the response headers. -""" -function cookielayer(handler) - return function managecookies(req::Request; cookies=true, cookiejar::CookieJar=COOKIEJAR, kw...) - if cookies === true || (cookies isa AbstractDict && !isempty(cookies)) - url = req.url - cookiestosend = Cookies.getcookies!(cookiejar, url) - if !(cookies isa Bool) - for (name, value) in cookies - push!(cookiestosend, Cookie(name, value)) - end - end - if !isempty(cookiestosend) - existingcookie = header(req.headers, "Cookie") - if existingcookie != "" && haskey(req.context, :includedCookies) - # this is a redirect where we previously included cookies - # we want to filter those out to avoid duplicate cookie sending - # and the case where a cookie was set to expire from the 1st request - previouscookies = Cookies.cookies(req) - previouslyincluded = req.context[:includedCookies] - filtered = filter(x -> !(x.name in previouslyincluded), previouscookies) - existingcookie = stringify("", filtered) - end - setheader(req.headers, "Cookie" => stringify(existingcookie, cookiestosend)) - req.context[:includedCookies] = map(x -> x.name, cookiestosend) - end - res = handler(req; kw...) - Cookies.setcookies!(cookiejar, url, res.headers) - return res - else - # skip - return handler(req; kw...) - end - end -end - -end # module CookieRequest diff --git a/src/clientlayers/ExceptionRequest.jl b/src/clientlayers/ExceptionRequest.jl deleted file mode 100644 index fb71d0cd2..000000000 --- a/src/clientlayers/ExceptionRequest.jl +++ /dev/null @@ -1,26 +0,0 @@ -module ExceptionRequest - -export exceptionlayer - -using ..IOExtras, ..Messages, ..Exceptions - -""" - exceptionlayer(handler) -> handler - -Throw a `StatusError` if the request returns an error response status. -""" -function exceptionlayer(handler) - return function exceptions(stream; status_exception::Bool=true, timedout=nothing, logerrors::Bool=false, logtag=nothing, kw...) - res = handler(stream; timedout=timedout, logerrors=logerrors, logtag=logtag, kw...) - if status_exception && iserror(res) - req = res.request - req.context[:status_errors] = get(req.context, :status_errors, 0) + 1 - e = StatusError(res.status, req.method, req.target, res) - throw(e) - else - return res - end - end -end - -end # module ExceptionRequest diff --git a/src/clientlayers/HeadersRequest.jl b/src/clientlayers/HeadersRequest.jl deleted file mode 100644 index eda77ca77..000000000 --- a/src/clientlayers/HeadersRequest.jl +++ /dev/null @@ -1,95 +0,0 @@ -module HeadersRequest - -export headerslayer, setuseragent! - -using Base64, URIs -using ..Messages, ..Forms, ..IOExtras, ..Sniff, ..Forms, ..Strings - -""" - headerslayer(handler) -> handler - -Sets default expected headers. -""" -function headerslayer(handler) - return function defaultheaders(req; iofunction=nothing, decompress=nothing, - basicauth::Bool=true, detect_content_type::Bool=false, canonicalize_headers::Bool=false, kw...) - headers = req.headers - ## basicauth - if basicauth - userinfo = unescapeuri(req.url.userinfo) - if !isempty(userinfo) && !hasheader(headers, "Authorization") - @debug "Adding Authorization: Basic header." - setheader(headers, "Authorization" => "Basic $(base64encode(userinfo))") - end - end - ## content type detection - if detect_content_type && (!hasheader(headers, "Content-Type") - && !isa(req.body, Form) - && isbytes(req.body)) - - sn = sniff(bytes(req.body)) - setheader(headers, "Content-Type" => sn) - @debug "setting Content-Type header to: $sn" - end - ## default headers - if isempty(req.url.port) || - (req.url.scheme == "http" && req.url.port == "80") || - (req.url.scheme == "https" && req.url.port == "443") - hostheader = req.url.host - else - hostheader = req.url.host * ":" * req.url.port - end - defaultheader!(headers, "Host" => hostheader) - defaultheader!(headers, "Accept" => "*/*") - if USER_AGENT[] !== nothing - defaultheader!(headers, "User-Agent" => USER_AGENT[]) - end - - if !hasheader(headers, "Content-Length") && - !hasheader(headers, "Transfer-Encoding") && - !hasheader(headers, "Upgrade") - l = nbytes(req.body) - if l !== nothing - setheader(headers, "Content-Length" => string(l)) - elseif req.method == "GET" && iofunction isa Function - setheader(headers, "Content-Length" => "0") - end - end - if !hasheader(headers, "Content-Type") && req.body isa Form && req.method in ("POST", "PUT", "PATCH") - # "Content-Type" => "multipart/form-data; boundary=..." - setheader(headers, content_type(req.body)) - elseif !hasheader(headers, "Content-Type") && (req.body isa Union{AbstractDict, NamedTuple}) && req.method in ("POST", "PUT", "PATCH") - setheader(headers, "Content-Type" => "application/x-www-form-urlencoded") - end - if decompress === nothing || decompress - defaultheader!(headers, "Accept-Encoding" => "gzip") - end - ## canonicalize headers - if canonicalize_headers - req.headers = canonicalizeheaders(headers) - end - res = handler(req; iofunction, decompress, kw...) - if canonicalize_headers - res.headers = canonicalizeheaders(res.headers) - end - return res - end -end - -canonicalizeheaders(h::T) where {T} = T([tocameldash(k) => v for (k,v) in h]) - -const USER_AGENT = Ref{Union{String, Nothing}}("HTTP.jl/$VERSION") - -""" - setuseragent!(x::Union{String, Nothing}) - -Set the default User-Agent string to be used in each HTTP request. -Can be manually overridden by passing an explicit `User-Agent` header. -Setting `nothing` will prevent the default `User-Agent` header from being passed. -""" -function setuseragent!(x::Union{String, Nothing}) - USER_AGENT[] = x - return -end - -end # module diff --git a/src/clientlayers/MessageRequest.jl b/src/clientlayers/MessageRequest.jl deleted file mode 100644 index 79bda0e08..000000000 --- a/src/clientlayers/MessageRequest.jl +++ /dev/null @@ -1,59 +0,0 @@ -module MessageRequest - -using URIs, LoggingExtras -using ..IOExtras, ..Messages, ..Parsers, ..Exceptions -using ..Messages, ..Parsers -using ..Strings: HTTPVersion -import ..DEBUG_LEVEL - -export messagelayer - -# like Messages.mkheaders, but we want to make a copy of user-provided headers -# and avoid double copy when no headers provided (very common) -mkreqheaders(::Nothing, ch) = Header[] -mkreqheaders(headers::Headers, ch) = ch ? copy(headers) : headers -mkreqheaders(h, ch) = mkheaders(h) - -""" - messagelayer(handler) -> handler - -Construct a [`Request`](@ref) object from method, url, headers, and body. -Hard-coded as the first layer in the request pipeline. -""" -function messagelayer(handler) - return function makerequest(method::String, url::URI, headers, body; copyheaders::Bool=true, response_stream=nothing, http_version=HTTPVersion(1, 1), verbose=DEBUG_LEVEL[], kw...) - req = Request(method, resource(url), mkreqheaders(headers, copyheaders), body; url=url, version=http_version, responsebody=response_stream) - local resp - start_time = time() - try - # if debugging, enable by wrapping request in custom logger logic - resp = if verbose > 0 - LoggingExtras.withlevel(Logging.Debug) do - handler(req; verbose, response_stream, kw...) - end - else - handler(req; verbose, response_stream, kw...) - end - catch e - if e isa CapturedException - e = e.ex - end - if e isa StatusError - resp = e.response - end - rethrow(e) - finally - dur = (time() - start_time) * 1000 - req.context[:total_request_duration_ms] = dur - if @isdefined(resp) && iserror(resp) && haskey(resp.request.context, :response_body) - if isbytes(resp.body) - resp.body = resp.request.context[:response_body] - else - write(resp.body, resp.request.context[:response_body]) - end - end - end - end -end - -end # module MessageRequest diff --git a/src/clientlayers/RedirectRequest.jl b/src/clientlayers/RedirectRequest.jl deleted file mode 100644 index 5f4a62a40..000000000 --- a/src/clientlayers/RedirectRequest.jl +++ /dev/null @@ -1,118 +0,0 @@ -module RedirectRequest - -using URIs -using ..Messages, ..Pairs - -export redirectlayer, nredirects - -""" - redirectlayer(handler) -> handler - -Redirects the request in the case of 3xx response status. -""" -function redirectlayer(handler) - return function redirects(req; redirect::Bool=true, redirect_limit::Int=3, redirect_method=nothing, forwardheaders::Bool=true, response_stream=nothing, kw...) - if !redirect || redirect_limit == 0 - # no redirecting - return handler(req; kw...) - end - req.context[:allow_redirects] = true - count = 0 - while true - # Verify the url before making the request. Verification is done in - # the redirect loop to also catch bad redirect URLs. - verify_url(req.url) - res = handler(req; kw...) - - if (count == redirect_limit || !isredirect(res) - || (location = header(res, "Location")) == "") - return res - end - - # follow redirect - oldurl = req.url - url = resolvereference(req.url, location) - method = newmethod(req.method, res.status, redirect_method) - body = method == "GET" ? UInt8[] : req.body - req = Request(method, resource(url), copy(req.headers), body; - url=url, version=req.version, responsebody=response_stream, parent=res, context=req.context) - if forwardheaders - req.headers = filter(req.headers) do (header, _) - # false return values are filtered out - if header == "Host" - return false - elseif (header in SENSITIVE_HEADERS && !isdomainorsubdomain(url.host, oldurl.host)) - return false - elseif method == "GET" && header in ("Content-Type", "Content-Length") - return false - else - return true - end - end - else - req.headers = Header[] - end - @debug "➡️ Redirect: $url" - count += 1 - if count == redirect_limit - req.context[:redirectlimitreached] = true - end - end - @assert false "Unreachable!" - end -end - -function nredirects(req) - return req.parent === nothing ? 0 : (1 + nredirects(req.parent.request)) -end - -const SENSITIVE_HEADERS = Set([ - "Authorization", - "Www-Authenticate", - "Cookie", - "Cookie2" -]) - -function isdomainorsubdomain(sub, parent) - sub == parent && return true - endswith(sub, parent) || return false - return sub[length(sub)-length(parent)] == '.' -end - -function verify_url(url::URI) - if !(url.scheme in ("http", "https", "ws", "wss")) - throw(ArgumentError("missing or unsupported scheme in URL (expected http(s) or ws(s)): $(url)")) - end - if isempty(url.host) - throw(ArgumentError("missing host in URL: $(url)")) - end -end - -function newmethod(request_method, response_status, redirect_method) - # using https://everything.curl.dev/http/redirects#get-or-post as a reference - # also reference: https://github.com/curl/curl/issues/5237#issuecomment-618293609 - if response_status == 307 || response_status == 308 - # specific status codes that indicate an identical request should be made to new location - return request_method - elseif response_status == 303 - # 303 means it's a new/different URI, so only GET allowed - return "GET" - elseif redirect_method == :same - return request_method - elseif redirect_method !== nothing && String(redirect_method) in ("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH") - return redirect_method - elseif request_method == "HEAD" - # Unless otherwise specified (e.g. with `redirect_method`), be conservative and keep the - # same method, see: - # - # * - # * - # - # Turning a HEAD request through a redirect may be undesired: - # . - return request_method - end - return "GET" -end - -end # module RedirectRequest diff --git a/src/clientlayers/RetryRequest.jl b/src/clientlayers/RetryRequest.jl deleted file mode 100644 index ab62b5272..000000000 --- a/src/clientlayers/RetryRequest.jl +++ /dev/null @@ -1,107 +0,0 @@ -module RetryRequest - -using Sockets, MbedTLS, OpenSSL, ExceptionUnwrapping -using ..IOExtras, ..Messages, ..Strings, ..ExceptionRequest, ..Exceptions - -export retrylayer - -FALSE(x...) = false - -""" - retrylayer(handler) -> handler - -Retry the request if it throws a recoverable exception. - -`Base.retry` and `Base.ExponentialBackOff` implement a randomised exponentially -increasing delay is introduced between attempts to avoid exacerbating network -congestion. - -By default, requests that have a retryable body, where the request wasn't written -or is idempotent will be retried. If the request is made and a response is received -with a status code of 403, 408, 409, 429, or 5xx, the request will be retried. - -`retries` controls the # of total retries that will be attempted. - -`retry_check` allows passing a custom retry check in the case where the default -retry check _wouldn't_ retry, if `retry_check` returns true, then the request -will be retried anyway. -""" -function retrylayer(handler) - return function manageretries(req::Request; retry::Bool=true, retries::Int=4, - retry_delays=ExponentialBackOff(n = retries, factor=3.0), retry_check=FALSE, - retry_non_idempotent::Bool=false, kw...) - if !retry || retries == 0 - # no retry - return handler(req; kw...) - end - req.context[:allow_retries] = true - req.context[:retryattempt] = 0 - if retry_non_idempotent - req.context[:retry_non_idempotent] = true - end - req_body_is_marked = false - if req.body isa IO && Messages.supportsmark(req.body) - @debug "Marking request body stream" - req_body_is_marked = true - mark(req.body) - end - retryattempt = Ref(0) - retry_request = Base.retry(handler, - delays=retry_delays, - check=(s, ex) -> begin - retryattempt[] += 1 - req.context[:retryattempt] = retryattempt[] - retry = ( - (isrecoverable(ex) && retryable(req)) || - (retryablebody(req) && !retrylimitreached(req) && _retry_check(s, ex, req, retry_check)) - ) - if retryattempt[] == retries - req.context[:retrylimitreached] = true - end - if retry - @debug "🔄 Retry $ex: $(sprintcompact(req))" - reset!(req.response) - if req_body_is_marked - @debug "Resetting request body stream" - reset(req.body) - mark(req.body) - end - else - @debug "🚷 No Retry: $(no_retry_reason(ex, req))" - end - return s, retry - end - ) - return retry_request(req; kw...) - end -end - -isrecoverable(ex) = is_wrapped_exception(ex) ? isrecoverable(unwrap_exception(ex)) : false -isrecoverable(::Union{Base.EOFError, Base.IOError, MbedTLS.MbedException, OpenSSL.OpenSSLError}) = true -isrecoverable(ex::ArgumentError) = ex.msg == "stream is closed or unusable" -isrecoverable(ex::CompositeException) = all(isrecoverable, ex.exceptions) -# Treat all DNS errors except `EAI_AGAIN`` as non-recoverable -# Ref: https://github.com/JuliaLang/julia/blob/ec8df3da3597d0acd503ff85ac84a5f8f73f625b/stdlib/Sockets/src/addrinfo.jl#L108-L112 -isrecoverable(ex::Sockets.DNSError) = (ex.code == Base.UV_EAI_AGAIN) -isrecoverable(ex::StatusError) = retryable(ex.status) - -function _retry_check(s, ex, req, check) - resp = req.response - resp_body = get(req.context, :response_body, nothing) - return check(s, ex, req, resp_body !== nothing ? resp : nothing, resp_body) -end - -function no_retry_reason(ex, req) - buf = IOBuffer() - unwrapped_ex = unwrap_exception(ex) - show(IOContext(buf, :compact => true), req) - print(buf, ", ", - unwrapped_ex isa StatusError ? "HTTP $(ex.status): " : - !isrecoverable(unwrapped_ex) ? "unrecoverable exception: " : - !isbytes(req.body) ? "request streamed, " : "", - !isbytes(req.response.body) ? "response streamed, " : "", - !isidempotent(req) ? "$(req.method) non-idempotent" : "") - return String(take!(buf)) -end - -end # module RetryRequest diff --git a/src/clientlayers/StreamRequest.jl b/src/clientlayers/StreamRequest.jl deleted file mode 100644 index 9d8c48a40..000000000 --- a/src/clientlayers/StreamRequest.jl +++ /dev/null @@ -1,191 +0,0 @@ -module StreamRequest - -using ..IOExtras, ..Messages, ..Streams, ..Connections, ..Strings, ..RedirectRequest, ..Exceptions -using CodecZlib, URIs -using SimpleBufferStream: BufferStream -using ConcurrentUtilities: @samethreadpool_spawn - -export streamlayer - -""" - streamlayer(stream) -> HTTP.Response - -Create a [`Stream`](@ref) to send a `Request` and `body` to an `IO` -stream and read the response. - -Send the `Request` body in a background task and begins reading the response -immediately so that the transmission can be aborted if the `Response` status -indicates that the server does not wish to receive the message body. -[RFC7230 6.5](https://tools.ietf.org/html/rfc7230#section-6.5). -""" -function streamlayer(stream::Stream; iofunction=nothing, decompress::Union{Nothing, Bool}=nothing, logerrors::Bool=false, logtag=nothing, timedout=nothing, kw...)::Response - response = stream.message - req = response.request - @debug sprintcompact(req) - @debug "client startwrite" - write_start = time() - startwrite(stream) - - @debug sprint(show, req) - if iofunction === nothing && !isbytes(req.body) - @debug "$(typeof(req)).body: $(sprintcompact(req.body))" - end - - try - @sync begin - if iofunction === nothing - # use a lock here for request.context changes (this is currently the only places - # where multiple threads may modify/change context at the same time) - lock = ReentrantLock() - @samethreadpool_spawn try - writebody(stream, req, lock) - finally - Base.@lock lock begin - req.context[:write_duration_ms] = get(req.context, :write_duration_ms, 0.0) + ((time() - write_start) * 1000) - end - @debug "client closewrite" - closewrite(stream) - end - read_start = time() - @samethreadpool_spawn try - @debug "client startread" - startread(stream) - if !isaborted(stream) - readbody(stream, response, decompress, lock) - end - finally - Base.@lock lock begin - req.context[:read_duration_ms] = get(req.context, :read_duration_ms, 0.0) + ((time() - read_start) * 1000) - end - @debug "client closeread" - closeread(stream) - end - else - try - iofunction(stream) - finally - closewrite(stream) - closeread(stream) - end - end - end - catch - if timedout === nothing || !timedout[] - req.context[:io_errors] = get(req.context, :io_errors, 0) + 1 - if logerrors - @error current_exceptions_to_string() type=Symbol("HTTP.IOError") method=req.method url=req.url context=req.context logtag=logtag - end - end - rethrow() - end - - @debug sprintcompact(response) - @debug sprint(show, response) - return response -end - -function writebody(stream::Stream, req::Request, lock) - if !isbytes(req.body) - n = writebodystream(stream, req.body) - closebody(stream) - else - n = write(stream, req.body) - end - Base.@lock lock begin - req.context[:nbytes_written] = n - end - return n -end - -function writebodystream(stream, body) - n = 0 - for chunk in body - n += writechunk(stream, chunk) - end - return n -end - -function writebodystream(stream, body::IO) - return write(stream, body) -end - -function writebodystream(stream, body::Union{AbstractDict, NamedTuple}) - # application/x-www-form-urlencoded - return write(stream, URIs.escapeuri(body)) -end - -writechunk(stream, body::IO) = writebodystream(stream, body) -writechunk(stream, body::Union{AbstractDict, NamedTuple}) = writebodystream(stream, body) -writechunk(stream, body) = write(stream, body) - -function readbody(stream::Stream, res::Response, decompress::Union{Nothing, Bool}, lock) - if decompress === true || (decompress === nothing && header(res, "Content-Encoding") == "gzip") - # Plug in a buffer stream in between so that we can (i) read the http stream in - # chunks instead of byte-by-byte and (ii) make sure to stop reading the http stream - # at eof. - buf = BufferStream() - gzstream = GzipDecompressorStream(buf) - tsk = @async begin - try - write(gzstream, stream) - finally - # Close here to (i) deallocate resources in zlib and (ii) make sure that - # read(buf)/write(..., buf) below don't block forever. Note that this will - # close the stream wrapped by the decompressor (buf) but *not* the http - # stream, which should be left open. - close(gzstream) - end - end - readbody!(stream, res, buf, lock) - wait(tsk) - else - readbody!(stream, res, stream, lock) - end -end - -function readbody!(stream::Stream, res::Response, buf_or_stream, lock) - n = 0 - if !iserror(res) - if isbytes(res.body) - if length(res.body) > 0 - # user-provided buffer to read response body into - # specify write=true to make the buffer writable - # but also specify maxsize, which means it won't be grown - # (we don't want to be changing the user's buffer for them) - body = IOBuffer(res.body; write=true, maxsize=length(res.body)) - if buf_or_stream isa BufferStream - # if it's a BufferStream, the response body was gzip encoded - # so using the default write is fastest because it utilizes - # readavailable under the hood, for which BufferStream is optimized - n = write(body, buf_or_stream) - elseif buf_or_stream isa Stream{Response} - # for HTTP.Stream, there's already an optimized read method - # that just needs an IOBuffer to write into - n = readall!(buf_or_stream, body) - else - error("unreachable") - end - else - res.body = read(buf_or_stream) - n = length(res.body) - end - elseif res.body isa Base.GenericIOBuffer && buf_or_stream isa Stream{Response} - # optimization for IOBuffer response_stream to avoid temporary allocations - n = readall!(buf_or_stream, res.body) - else - n = write(res.body, buf_or_stream) - end - else - # read the response body into the request context so that it can be - # read by the user if they want to or set later if - # we end up not retrying/redirecting/etc. - Base.@lock lock begin - res.request.context[:response_body] = read(buf_or_stream) - end - end - Base.@lock lock begin - res.request.context[:nbytes] = n - end -end - -end # module StreamRequest diff --git a/src/clientlayers/TimeoutRequest.jl b/src/clientlayers/TimeoutRequest.jl deleted file mode 100644 index 5ca728d34..000000000 --- a/src/clientlayers/TimeoutRequest.jl +++ /dev/null @@ -1,38 +0,0 @@ -module TimeoutRequest - -using ..Connections, ..Streams, ..Exceptions, ..Messages -using ConcurrentUtilities -using ..Exceptions: current_exceptions_to_string - -export timeoutlayer - -""" - timeoutlayer(handler) -> handler - -Close the `HTTP.Stream` if no data has been received for `readtimeout` seconds. -""" -function timeoutlayer(handler) - return function timeouts(stream::Stream; readtimeout::Int=0, logerrors::Bool=false, logtag=nothing, kw...) - if readtimeout <= 0 - # skip - return handler(stream; logerrors=logerrors, kw...) - end - return try - try_with_timeout(readtimeout, Response) do timedout - handler(stream; logerrors=logerrors, logtag=logtag, timedout=timedout, kw...) - end - catch e - if e isa ConcurrentUtilities.TimeoutException - req = stream.message.request - req.context[:timeout_errors] = get(req.context, :timeout_errors, 0) + 1 - if logerrors - @error current_exceptions_to_string() type=Symbol("HTTP.TimeoutError") method=req.method url=req.url context=req.context timeout=readtimeout logtag=logtag - end - e = Exceptions.TimeoutError(readtimeout) - end - rethrow(e) - end - end -end - -end # module TimeoutRequest diff --git a/src/cookiejar.jl b/src/cookiejar.jl index 0a358dbfd..e6649b36c 100644 --- a/src/cookiejar.jl +++ b/src/cookiejar.jl @@ -51,7 +51,7 @@ function pathmatch(cookie::Cookie, requestpath) end """ - Cookies.getcookies!(jar::CookieJar, url::URI) + Cookies.getcookies!(jar::CookieJar, scheme, host, path) Retrieve valid `Cookie`s from the `CookieJar` according to the provided `url`. Cookies will be returned as a `Vector{Cookie}`. Only cookies for `http` or `https` @@ -59,18 +59,17 @@ scheme in the url will be returned. Cookies will be checked according to the can host of the url and any cookie max age or expiration will be accounted for. Expired cookies will not be returned and will be removed from the cookie jar. """ -function getcookies!(jar::CookieJar, url::URI, now::DateTime=Dates.now(Dates.UTC))::Vector{Cookie} +function getcookies!(jar::CookieJar, scheme::String, host::String, path::String, now::DateTime=Dates.now(Dates.UTC))::Vector{Cookie} cookies = Cookie[] - if url.scheme != "http" && url.scheme != "https" + if scheme != "http" && scheme != "https" return cookies end - host = canonicalhost(url.host) + host = canonicalhost(host) host == "" && return cookies Base.@lock jar.lock begin !haskey(jar.entries, host) && return cookies entries = jar.entries[host] - https = url.scheme == "https" - path = url.path + https = scheme == "https" if path == "" path = "/" end @@ -78,7 +77,6 @@ function getcookies!(jar::CookieJar, url::URI, now::DateTime=Dates.now(Dates.UTC expired = Cookie[] for (id, e) in entries if e.persistent && e.expires != DateTime(1) && e.expires < now - @debug "Deleting expired cookie: $(e.name)" push!(expired, e) continue end @@ -86,7 +84,6 @@ function getcookies!(jar::CookieJar, url::URI, now::DateTime=Dates.now(Dates.UTC continue end e.lastaccess = now - @debug "Including cookie in request: $(e.name) to $(url.host)" push!(cookies, e) end for c in expired @@ -106,21 +103,21 @@ function getcookies!(jar::CookieJar, url::URI, now::DateTime=Dates.now(Dates.UTC end """ - Cookies.setcookies!(jar::CookieJar, url::URI, headers::Headers) + Cookies.setcookies!(jar::CookieJar, scheme, host, path, headers::Headers) Identify, "Set-Cookie" response headers from `headers`, parse the `Cookie`s, and store valid entries in the cookie `jar` according to the canonical host in `url`. Cookies can be retrieved from the `jar` via [`Cookies.getcookies!`](@ref). """ -function setcookies!(jar::CookieJar, url::URI, headers::Headers) +function setcookies!(jar::CookieJar, scheme::String, host::String, path::String, headers::Headers) cookies = readsetcookies(headers) isempty(cookies) && return - if url.scheme != "http" && url.scheme != "https" + if scheme != "http" && scheme != "https" return end - host = canonicalhost(url.host) + host = canonicalhost(host) host == "" && return - defPath = defaultPath(url.path) + defPath = defaultPath(path) now = Dates.now(Dates.UTC) Base.@lock jar.lock begin entries = get!(() -> Dict{String, Cookie}(), jar.entries, host) @@ -141,7 +138,6 @@ function setcookies!(jar::CookieJar, url::URI, headers::Headers) c.persistent = false else if c.expires < now - @debug "Cookie expired: $(c.name)" @goto remove end c.persistent = true @@ -189,7 +185,7 @@ function defaultPath(path) if i === nothing || i == 1 return "/" end - return path[1:i] + return path[1:(i - 1)] end const endOfTime = DateTime(9999, 12, 31, 23, 59, 59, 0) diff --git a/src/cookies.jl b/src/cookies.jl index a38852ec6..7cc09e6cf 100644 --- a/src/cookies.jl +++ b/src/cookies.jl @@ -33,8 +33,8 @@ module Cookies export Cookie, CookieJar, cookies, stringify, getcookies!, setcookies!, addcookie! import Base: == -using Dates, URIs, Sockets -using ..IOExtras, ..Parsers, ..Messages +using Dates, Sockets +import ..addheader, ..headereq, ..Headers, ..Request, ..Response @enum SameSite SameSiteDefaultMode=1 SameSiteLaxMode SameSiteStrictMode SameSiteNoneMode @@ -184,21 +184,8 @@ For requests, the cookie will be stringified and concatenated to any existing """ function addcookie! end -function addcookie!(r::Request, c::Cookie) - cstr = stringify(c) - chead = header(r, "Cookie", nothing) - if chead !== nothing - setheader(r, "Cookie" => "$chead; $cstr") - else - appendheader(r, SubString("Cookie") => SubString(cstr)) - end - return r -end - -function addcookie!(r::Response, c::Cookie) - appendheader(r, SubString("Set-Cookie") => SubString(stringify(c, false))) - return r -end +addcookie!(r::Request, c::Cookie) = addheader(r.headers, "Cookie", stringify(c)) +addcookie!(r::Response, c::Cookie) = addheader(r.headers, "Set-Cookie", stringify(c, false)) validcookiepathbyte(b) = (' ' <= b < '\x7f') && b != ';' validcookievaluebyte(b) = (' ' <= b < '\x7f') && b != '"' && b != ';' && b != '\\' @@ -221,9 +208,11 @@ const RFC1123GMTFormat = gmtformat(Dates.RFC1123Format) # readSetCookies parses all "Set-Cookie" values from # the header h and returns the successfully parsed Cookies. -function readsetcookies(h::Headers) +function readsetcookies(headers::Headers) result = Cookie[] - for line in headers(h, "Set-Cookie") + for h in headers + headereq(h.name, "Set-Cookie") || continue + line = h.value if length(line) == 0 continue end @@ -311,8 +300,6 @@ function readsetcookies(h::Headers) return result end -readsetcookies(h) = readsetcookies(mkheaders(h)) - function isIP(host) try Base.parse(IPAddr, host) @@ -336,9 +323,11 @@ cookies(r::Request) = readcookies(r.headers, "") # readCookies parses all "Cookie" values from the header h and # returns the successfully parsed Cookies. # if filter isn't empty, only cookies of that name are returned -function readcookies(h::Headers, filter::String="") +function readcookies(headers::Headers, filter::String="") result = Cookie[] - for line in headers(h, "Cookie") + for h in headers + headereq(h.name, "Cookie") || continue + line = h.value for part in split(strip(line), ';'; keepempty=false) part = strip(part) length(part) <= 1 && continue @@ -358,8 +347,6 @@ function readcookies(h::Headers, filter::String="") return result end -readcookies(h, f) = readcookies(mkheaders(h), f) - # validCookieExpires returns whether v is a valid cookie expires-value. function validCookieExpires(dt) # IETF RFC 6265 Section 5.1.1.5, the year must not be less than 1601 @@ -419,7 +406,7 @@ sanitizeCookieName(n) = sanitizeCookieName(String(n)) # with a comma or space. # See https:#golang.org/issue/7243 for the discussion. function sanitizeCookieValue(v::String) - v = String(filter(validcookievaluebyte, [Char(b) for b in bytes(v)])) + v = String(filter(validcookievaluebyte, [c for c in v])) length(v) == 0 && return v if contains(v, ' ') || contains(v, ',') return string('"', v, '"') diff --git a/src/download.jl b/src/download.jl deleted file mode 100644 index a92b88fde..000000000 --- a/src/download.jl +++ /dev/null @@ -1,160 +0,0 @@ -using .Pairs -using CodecZlib - -""" - safer_joinpath(basepart, parts...) -A variation on `joinpath`, that is more resistant to directory traversal attacks. -The parts to be joined (excluding the `basepart`), -are not allowed to contain `..`, or begin with a `/`. -If they do then this throws an `DomainError`. -""" -function safer_joinpath(basepart, parts...) - explain = "Possible directory traversal attack detected." - for part in parts - occursin("..", part) && throw(DomainError(part, "contains \"..\". $explain")) - startswith(part, '/') && throw(DomainError(part, "begins with \"/\". $explain")) - end - joinpath(basepart, parts...) -end - -function try_get_filename_from_headers(hdrs) - for content_disp in hdrs - # extract out of Content-Disposition line - # rough version of what is needed in https://github.com/JuliaWeb/HTTP.jl/issues/179 - filename_part = match(r"filename\s*=\s*(.*)", content_disp) - if filename_part !== nothing - filename = filename_part[1] - quoted_filename = match(r"\"(.*)\"", filename) - if quoted_filename !== nothing - # It was in quotes, so it will be double escaped - filename = unescape_string(quoted_filename[1]) - end - return filename == "" ? nothing : filename - end - end - return nothing -end - -function try_get_filename_from_request(req) - function file_from_target(t) - (t == "" || t == "/") && return nothing - f = basename(URI(t).path) # URI(...).path to strip out e.g. query parts - return (f == "" ? file_from_target(dirname(t)) : f) - end - - # First try to get file from the original request URI - oreq = req - while oreq.parent !== nothing - oreq = oreq.parent.request - end - f = file_from_target(oreq.target) - f !== nothing && return f - - # Secondly try to get file from the last request URI - return file_from_target(req.target) -end - - -determine_file(::Nothing, resp, hdrs) = determine_file(tempdir(), resp, hdrs) -# ^ We want to the filename if possible because extension is useful for FileIO.jl - -function determine_file(path, resp, hdrs) - if isdir(path) - # we have been given a path to a directory - # got to to workout what file to put there - filename = something( - try_get_filename_from_headers(hdrs), - try_get_filename_from_request(resp.request), - basename(tempname()) # fallback, basically a random string - ) - - - safer_joinpath(path, filename) - else - # We have been given a full filepath - path - end -end - -""" - download(url, [local_path], [headers]; update_period=1, kw...) - -Similar to `Base.download` this downloads a file, returning the filename. -If the `local_path`: - - is not provided, then it is saved in a temporary directory - - if part to a directory is provided then it is saved into that directory - - otherwise the local path is uses as the filename to save to. - -When saving into a directory, the filename is determined (where possible), -from the rules of the HTTP. - - - `update_period` controls how often (in seconds) to report the progress. - - set to `Inf` to disable reporting - - `headers` specifies headers to be used for the HTTP GET request - - any additional keyword args (`kw...`) are passed on to the HTTP request. -""" -function download(url::AbstractString, local_path=nothing, headers=Header[]; update_period=1, kw...) - format_progress(x) = round(x, digits=4) - format_bytes(x) = !isfinite(x) ? "∞ B" : Base.format_bytes(round(Int, x)) - format_seconds(x) = "$(round(x; digits=2)) s" - format_bytes_per_second(x) = format_bytes(x) * "/s" - - - @debug "downloading $url" - local file - hdrs = String[] - HTTP.open("GET", url, headers; kw...) do stream - resp = startread(stream) - # Store intermediate header from redirects to use for filename detection - content_disp = header(resp, "Content-Disposition") - !isempty(content_disp) && push!(hdrs, content_disp) - eof(stream) && return # don't do anything for streams we can't read (yet) - - file = determine_file(local_path, resp, hdrs) - total_bytes = parse(Float64, header(resp, "Content-Length", "NaN")) - downloaded_bytes = 0 - start_time = now() - prev_time = now() - - if header(resp, "Content-Encoding") == "gzip" - stream = GzipDecompressorStream(stream) # auto decoding - total_bytes = NaN # We don't know actual total bytes if the content is zipped. - end - - function report_callback() - prev_time = now() - taken_time = (prev_time - start_time).value / 1000 # in seconds - average_speed = downloaded_bytes / taken_time - remaining_bytes = total_bytes - downloaded_bytes - remaining_time = remaining_bytes / average_speed - completion_progress = downloaded_bytes / total_bytes - - @info("Downloading", - source=url, - dest = file, - progress = completion_progress |> format_progress, - time_taken = taken_time |> format_seconds, - time_remaining = remaining_time |> format_seconds, - average_speed = average_speed |> format_bytes_per_second, - downloaded = downloaded_bytes |> format_bytes, - remaining = remaining_bytes |> format_bytes, - total = total_bytes |> format_bytes, - ) - end - - Base.open(file, "w") do fh - while(!eof(stream)) - downloaded_bytes += write(fh, readavailable(stream)) - if !isinf(update_period) - if now() - prev_time > Millisecond(round(1000update_period)) - report_callback() - end - end - end - end - if !isinf(update_period) - report_callback() - end - end - file -end diff --git a/src/forms.jl b/src/forms.jl new file mode 100644 index 000000000..8823de2ae --- /dev/null +++ b/src/forms.jl @@ -0,0 +1,434 @@ +module Forms + +export Form, Multipart, content_type, parse_multipart_form + +import ..sniff + +# Form request body +mutable struct Form <: IO + data::Vector{IO} + index::Int + mark::Int + boundary::String +end + +Form(f::Form) = f +Base.eof(f::Form) = f.index > length(f.data) +Base.isopen(f::Form) = false +Base.close(f::Form) = nothing +Base.length(f::Form) = sum(x->isa(x, IOStream) ? filesize(x) - position(x) : bytesavailable(x), f.data) + +function Base.mark(f::Form) + foreach(mark, f.data) + f.mark = f.index + return +end + +function Base.reset(f::Form) + foreach(reset, f.data) + f.index = f.mark + f.mark = -1 + return +end + +function Base.unmark(f::Form) + foreach(unmark, f.data) + f.mark = -1 + return +end + +function Base.position(f::Form) + index = f.index + foreach(mark, f.data) + return index +end +function Base.seek(f::Form, pos) + f.index = pos + foreach(reset, f.data) + return +end + +Base.readavailable(f::Form) = read(f) +function Base.read(f::Form) + result = UInt8[] + for io in f.data + append!(result, read(io)) + end + f.index = length(f.data) + 1 + return result +end + +function Base.read(f::Form, n::Integer) + nb = 0 + result = UInt8[] + while nb < n + d = read(f.data[f.index], n - nb) + nb += length(d) + append!(result, d) + eof(f.data[f.index]) && (f.index += 1) + f.index > length(f.data) && break + end + return result +end + +""" + HTTP.Form(data; boundary=string(rand(UInt128), base=16)) + +Construct a request body for multipart/form-data encoding from `data`. + +`data` must iterate key-value pairs (e.g. `AbstractDict` or `Vector{Pair}`) where the key/value of the +iterator is the key/value of each mutipart boundary chunk. +Files and other large data arguments can be provided as values as IO arguments: either an `IOStream` +such as returned via `open(file)`, or an `IOBuffer` for in-memory data. + +For complete control over a multipart chunk's details, an +[`HTTP.Multipart`](@ref) type is provided to support setting the `filename`, `Content-Type`, +and `Content-Transfer-Encoding`. + +# Examples +```julia +data = Dict( + "text" => "text data", + # filename (cat.png) and content-type (image/png) inferred from the IOStream + "file1" => open("cat.png"), + # manully controlled chunk + "file2" => HTTP.Multipart("dog.jpeg", open("mydog.jpg"), "image/jpeg"), +) +body = HTTP.Form(data) +headers = [] +HTTP.post(url, headers, body) +``` +""" +function Form(d; boundary=string(rand(UInt128), base=16)) + # https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html + bcharsnospace = raw"\w'\(\)\+,-\./:=\?" + boundary_re = Regex("^[$bcharsnospace ]{0,69}[$bcharsnospace]\$") + @assert match(boundary_re, boundary) !== nothing + @assert eltype(d) <: Pair + data = IO[] + io = IOBuffer() + len = length(d) + for (i, (k, v)) in enumerate(d) + write(io, (i == 1 ? "" : "\r\n") * "--" * boundary * "\r\n") + write(io, "Content-Disposition: form-data; name=\"$k\"") + if isa(v, IO) + writemultipartheader(io, v) + seekstart(io) + push!(data, io) + push!(data, v) + io = IOBuffer() + else + write(io, "\r\n\r\n") + write(io, v) + end + end + # write final boundary + write(io, "\r\n--" * boundary * "--" * "\r\n") + seekstart(io) + push!(data, io) + return Form(data, 1, -1, boundary) +end + +function writemultipartheader(io::IOBuffer, i::IOStream) + write(io, "; filename=\"$(basename(i.name[7:end-1]))\"\r\n") + write(io, "Content-Type: $(sniff(i))\r\n\r\n") + return +end +function writemultipartheader(io::IOBuffer, i::IO) + write(io, "\r\n\r\n") + return +end + +""" + HTTP.Multipart(filename::String, data::IO, content_type=HTTP.sniff(data), content_transfer_encoding="") + +A type to represent a single multipart upload chunk for a file. This type would be used as the value in a +key-value pair when constructing a [`HTTP.Form`](@ref) for a request body (see example below). +The `data` argument must be an `IO` type such as `IOStream`, or `IOBuffer`. +The `content_type` and `content_transfer_encoding` arguments allow manual setting of these multipart headers. +`Content-Type` will default to the result of the `HTTP.sniff(data)` mimetype detection algorithm, whereas +`Content-Transfer-Encoding` will be left out if not specified. + +# Examples +```julia +body = HTTP.Form(Dict( + "key" => HTTP.Multipart("File.txt", open("MyFile.txt"), "text/plain"), +)) +headers = [] +HTTP.post(url, headers, body) +``` + +# Extended help + +Filename SHOULD be included when the Multipart represents the contents of a file +[RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) + +Content-Disposition set to "form-data" MUST be included with each Multipart. +An additional "name" parameter MUST be included +An optional "filename" parameter SHOULD be included if the contents of a file are sent +This will be formatted such as: + Content-Disposition: form-data; name="user"; filename="myfile.txt" +[RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) + +Content-Type for each Multipart is optional, but SHOULD be included if the contents +of a file are sent. +[RFC7578 4.4](https://tools.ietf.org/html/rfc7578#section-4.4) + +Content-Transfer-Encoding for each Multipart is deprecated +[RFC7578 4.7](https://tools.ietf.org/html/rfc7578#section-4.7) + +Other Content- header fields MUST be ignored +[RFC7578 4.8](https://tools.ietf.org/html/rfc7578#section-4.8) +""" +mutable struct Multipart{T <: IO} <: IO + filename::Union{String, Nothing} + data::T + contenttype::String + contenttransferencoding::String + name::String +end + +function Multipart(f::Union{AbstractString, Nothing}, data::T, ct::AbstractString="", cte::AbstractString="", name::AbstractString="") where {T<:IO} + f = f !== nothing ? String(f) : nothing + return Multipart{T}(f, data, String(ct), String(cte), String(name)) +end + +function Base.show(io::IO, m::Multipart{T}) where {T} + items = ["data=::$T", "contenttype=\"$(m.contenttype)\"", "contenttransferencoding=\"$(m.contenttransferencoding)\")"] + m.filename === nothing || pushfirst!(items, "filename=\"$(m.filename)\"") + print(io, "HTTP.Multipart($(join(items, ", ")))") +end + +Base.bytesavailable(m::Multipart{T}) where {T} = isa(m.data, IOStream) ? filesize(m.data) - position(m.data) : bytesavailable(m.data) +Base.eof(m::Multipart{T}) where {T} = eof(m.data) +Base.read(m::Multipart{T}, n::Integer) where {T} = read(m.data, n) +Base.read(m::Multipart{T}) where {T} = read(m.data) +Base.mark(m::Multipart{T}) where {T} = mark(m.data) +Base.reset(m::Multipart{T}) where {T} = reset(m.data) +Base.seekstart(m::Multipart{T}) where {T} = seekstart(m.data) + +function writemultipartheader(io::IOBuffer, i::Multipart) + if i.filename === nothing + write(io, "\r\n") + else + write(io, "; filename=\"$(i.filename)\"\r\n") + end + contenttype = i.contenttype == "" ? sniff(i.data) : i.contenttype + write(io, "Content-Type: $(contenttype)\r\n") + write(io, i.contenttransferencoding == "" ? "\r\n" : "Content-Transfer-Encoding: $(i.contenttransferencoding)\r\n\r\n") + return +end + +content_type(f::Form) = "multipart/form-data; boundary=$(f.boundary)" + +const CR_BYTE = 0x0d # \r +const LF_BYTE = 0x0a # \n +const DASH_BYTE = 0x2d # - +const HTAB_BYTE = 0x09 # \t +const SPACE_BYTE = 0x20 +const SEMICOLON_BYTE = UInt8(';') +const CRLFCRLF = (CR_BYTE, LF_BYTE, CR_BYTE, LF_BYTE) + +"compare byte buffer `a` from index `i` to index `j` with `b` and check if they are byte-equal" +function byte_buffers_eq(a, i, j, b) + l = 1 + @inbounds for k = i:j + a[k] == b[l] || return false + l += 1 + end + return true +end + +""" + find_multipart_boundary(bytes, boundaryDelimiter; start::Int=1) + +Find the first and last index of the next boundary delimiting a part, and if +the discovered boundary is the terminating boundary. +""" +function find_multipart_boundary(bytes::AbstractVector{UInt8}, boundaryDelimiter::AbstractVector{UInt8}, start::Int=1) + # The boundary delimiter line is prepended with two '-' characters + # The boundary delimiter line starts on a new line, so must be preceded by a \r\n. + # The boundary delimiter line ends with \r\n, and can have "optional linear whitespace" between + # the end of the boundary delimiter, and the \r\n. + # The last boundary delimiter line has an additional '--' at the end of the boundary delimiter + # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) + i = start + end_index = i + length(boundaryDelimiter) + 1 + while end_index <= length(bytes) + if bytes[i] == DASH_BYTE && bytes[i + 1] == DASH_BYTE && byte_buffers_eq(bytes, i + 2, end_index, boundaryDelimiter) + # boundary delimiter line start on a new line ... + if i > 1 + (i == 2 || bytes[i-2] != CR_BYTE || bytes[i-1] != LF_BYTE) && error("boundary delimiter found, but it was not the start of a line") + # the CRLF preceding the boundary delimiter is "conceptually attached + # to the boundary", so account for this with the index + i -= 2 + end + # need to check if there are enough characters for the CRLF or for two dashes + end_index < length(bytes)-1 || error("boundary delimiter found, but did not end with new line") + is_terminating_delimiter = bytes[end_index+1] == DASH_BYTE && bytes[end_index+2] == DASH_BYTE + is_terminating_delimiter && (end_index += 2) + # ... there can be arbitrary SP and HTAB space between the boundary delimiter ... + while end_index < length(bytes) && (bytes[end_index+1] in (HTAB_BYTE, SPACE_BYTE)) + end_index += 1 + end + # ... and ends with a new line + newlineEnd = end_index < length(bytes)-1 && + bytes[end_index+1] == CR_BYTE && + bytes[end_index+2] == LF_BYTE + if !newlineEnd + error("boundary delimiter found, but did not end with new line") + end + end_index += 2 + return (is_terminating_delimiter, i, end_index) + end + i += 1 + end_index += 1 + end + error("boundary delimiter not found") +end + +""" + find_multipart_boundaries(bytes, boundary; start=1) + +Find the start and end indexes of all the parts of the multipart object. Ultimately this method is +looking for the data between the boundary delimiters in the byte array. A vector containing all +the start/end pairs is returned. +""" +function find_multipart_boundaries(bytes::AbstractVector{UInt8}, boundary::AbstractVector{UInt8}, start=1) + idxs = Tuple{Int, Int}[] + while true + (is_terminating_delimiter, i, end_index) = find_multipart_boundary(bytes, boundary, start) + push!(idxs, (i, end_index)) + is_terminating_delimiter && break + start = end_index + 1 + end + return idxs +end + +""" + find_header_boundary(bytes) + +Find the end of the multipart header in the byte array. Returns a Tuple with the +start index(1) and the end index. Headers are separated from the body by CRLFCRLF. + +[RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1) +[RFC822 3.1](https://tools.ietf.org/html/rfc822#section-3.1) +""" +function find_header_boundary(bytes::AbstractVector{UInt8}) + length(CRLFCRLF) > length(bytes) && return nothing + l = length(bytes) - length(CRLFCRLF) + 1 + i = 1 + end_index = length(CRLFCRLF) + while (i <= l) + byte_buffers_eq(bytes, i, end_index, CRLFCRLF) && return (1, end_index) + i += 1 + end_index += 1 + end + error("no delimiter found separating header from multipart body") +end + +""" + parse_multipart_chunk(chunk) + +Parse a single multi-part chunk into a Multipart object. This will decode +the header and extract the contents from the byte array. +""" +function parse_multipart_chunk(chunk) + _, end_index = find_header_boundary(chunk) + ind = end_index + header = unsafe_string(pointer(chunk), ind) + content = view(chunk, end_index+1:lastindex(chunk)) + + # find content disposition + re = match(r"^[Cc]ontent-[Dd]isposition:[ \t]*form-data;[ \t]*(.*)\r\n"mx, header) + if re === nothing + @warn "Content disposition is not specified dropping the chunk." String(chunk) + return nothing # Specifying content disposition is mandatory + end + content_disposition = SubString(re.match, sizeof("Content-Disposition: form-data;")+1) + + name = nothing + filename = nothing + while !isempty(content_disposition) + re_pair = match(r"""^ + [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+)[ \t]*=[ \t]*"(.*?)";? + """x, content_disposition) + if re_pair !== nothing + key = re_pair.captures[1] + value = re_pair.captures[2] + if key == "name" + name = value + elseif key == "filename" + filename = value + else + # do stuff with other content disposition key-value pairs + end + content_disposition = SubString(content_disposition, sizeof(re_pair.match)+1) + continue + end + re_flag = match(r"""^ + [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+);? + """x, content_disposition) + if re_flag !== nothing + # do stuff with content disposition flags + content_disposition = SubString(content_disposition, sizeof(re_flag.match)+1) + continue + end + break + end + + if name === nothing + @warn "Content disposition is missing the name field. Dropping the chunk." String(chunk) + return nothing + end + + re_ct = match(r"(?i)Content-Type: (\S*[^;\s])", header) + contenttype = re_ct === nothing ? "text/plain" : re_ct.captures[1] + return Multipart(filename, IOBuffer(content), contenttype, "", name) +end + +""" + parse_multipart_body(body, boundary)::Vector{Multipart} + +Parse the multipart body received from the client breaking it into the various +chunks which are returned as an array of Multipart objects. +""" +function parse_multipart_body(body::AbstractVector{UInt8}, boundary::AbstractString)::Vector{Multipart} + multiparts = Multipart[] + idxs = find_multipart_boundaries(body, codeunits(boundary)) + length(idxs) > 1 || (return multiparts) + + for i in 1:length(idxs)-1 + chunk = view(body, idxs[i][2]+1:idxs[i+1][1]-1) + push!(multiparts, parse_multipart_chunk(chunk)) + end + return multiparts +end + + +""" + parse_multipart_form(content_type, body)::Vector{Multipart} + +Parse the full mutipart form submission from the client returning and +array of Multipart objects containing all the data. + +The order of the multipart form data in the request should be preserved. +[RFC7578 5.2](https://tools.ietf.org/html/rfc7578#section-5.2). + +The boundary delimiter MUST NOT appear inside any of the encapsulated parts. Note +that the boundary delimiter does not need to have '-' characters, but a line using +the boundary delimiter will start with '--' and end in \r\n. +[RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) +""" +function parse_multipart_form(content_type::Union{String, Nothing}, body::Union{AbstractVector{UInt8}, Nothing})::Union{Vector{Multipart}, Nothing} + # parse boundary from Content-Type + (content_type === nothing || body === nothing) && return nothing + m = match(r"multipart/form-data; boundary=(.*)$", content_type) + m === nothing && return nothing + boundary_delimiter = m[1] + # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) + length(boundary_delimiter) > 70 && error("boundary delimiter must not be greater than 70 characters") + return parse_multipart_body(body, boundary_delimiter) +end + +end # module \ No newline at end of file diff --git a/src/Handlers.jl b/src/handlers.jl similarity index 70% rename from src/Handlers.jl rename to src/handlers.jl index 63fba3457..9c06b20f2 100644 --- a/src/Handlers.jl +++ b/src/handlers.jl @@ -1,10 +1,8 @@ module Handlers -export Handler, Middleware, serve, serve!, Router, register!, getroute, getparams, getparam, getcookies, streamhandler +export Handler, Middleware, serve, serve!, Router, register!, getroute, getparams, getparam, getcookies -using URIs -using ..Messages, ..Streams, ..IOExtras, ..Servers, ..Sockets, ..Cookies -import ..HTTP # for doc references +import ..Request, ..Cookies """ Handler @@ -41,104 +39,6 @@ then an input to the `auth_middlware`, which further enhances/modifies the handl """ abstract type Middleware end -""" - streamhandler(request_handler) -> stream handler - -Middleware that takes a request handler and returns a stream handler. Used by default -in `HTTP.serve` to take the user-provided request handler and process the `Stream` -from `HTTP.listen` and pass the parsed `Request` to the handler. - -Is included by default in `HTTP.serve` as the base "middleware" when `stream=false` is passed. -""" -function streamhandler(handler) - return function(stream::Stream) - request::Request = stream.message - request.body = read(stream) - closeread(stream) - request.response::Response = handler(request) - request.response.request = request - startwrite(stream) - write(stream, request.response.body) - return - end -end - -# Interface change in HTTP@1 -@deprecate RequestHandlerFunction streamhandler - -""" - HTTP.serve(handler, host=Sockets.localhost, port=8081; kw...) - HTTP.serve(handler, port::Integer=8081; kw...) - HTTP.serve(handler, server::Base.IOServer; kw...) - HTTP.serve!(args...; kw...) -> HTTP.Server - -Listen for HTTP connections and execute the `handler` function for each request. -Listening details can be passed as `host`/`port` pair, a single `port` (`host` will -default to `localhost`), or an already listening `server` object, as returned from -`Sockets.listen`. To open up a server to external requests, the `host` argument is -typically `"0.0.0.0"`. - -The `HTTP.serve!` form is non-blocking and returns an `HTTP.Server` object which can be -`wait(server)`ed on manually, or `close(server)`ed to gracefully shut down the server. -Calling `HTTP.forceclose(server)` will immediately force close the server and all active -connections. `HTTP.serve` will block on the server listening loop until interrupted or -and an irrecoverable error occurs. - -The `handler` function should be of the form `f(req::HTTP.Request)::HTTP.Response`. -Alternatively, passing `stream=true` requires the `handler` to be of the form -`f(stream::HTTP.Stream) -> Nothing`. See [`HTTP.Router`](@ref) for details on using -it as a request handler. - -Optional keyword arguments: -- `sslconfig=nothing`, Provide an `MbedTLS.SSLConfig` object to handle ssl - connections. Pass `sslconfig=MbedTLS.SSLConfig(false)` to disable ssl - verification (useful for testing). Construct a custom `SSLConfig` object - with `MbedTLS.SSLConfig(certfile, keyfile)`. -- `tcpisvalid = tcp->true`, function `f(::TCPSocket)::Bool` to check if accepted - connections are valid before processing requests. e.g. to do source IP filtering. -- `readtimeout::Int=0`, close the connection if no data is received for this - many seconds. Use readtimeout = 0 to disable. -- `reuseaddr::Bool=false`, allow multiple servers to listen on the same port. - Not supported on some OS platforms. Can check `HTTP.Servers.supportsreuseaddr()`. -- `server::Base.IOServer=nothing`, provide an `IOServer` object to listen on; - allows manually closing or configuring the server socket. -- `verbose::Bool=false`, log connection information to `stdout`. -- `access_log::Function`, function for formatting access log messages. The - function should accept two arguments, `io::IO` to which the messages should - be written, and `http::HTTP.Stream` which can be used to query information - from. See also [`@logfmt_str`](@ref). -- `on_shutdown::Union{Function, Vector{<:Function}, Nothing}=nothing`, one or - more functions to be run if the server is closed (for example by an - `InterruptException`). Note, shutdown function(s) will not run if an - `IOServer` object is supplied to the `server` keyword argument and closed - by `close(server)`. - -```julia -# start a blocking echo server -HTTP.serve("127.0.0.1", 8081) do req - return HTTP.Response(200, req.body) -end - -# non-blocking server -server = HTTP.serve!(8081) do req - return HTTP.Response(200, "response body") -end -# can gracefully close server manually -close(server) -``` -""" -function serve end - -""" - HTTP.serve!(args...; kw...) -> HTTP.Server - -Non-blocking version of [`HTTP.serve`](@ref); see that function for details. -""" -function serve! end - -serve(f, args...; stream::Bool=false, kw...) = Servers.listen(stream ? f : streamhandler(f), args...; kw...) -serve!(f, args...; stream::Bool=false, kw...) = Servers.listen!(stream ? f : streamhandler(f), args...; kw...) - # tree-based router handler mutable struct Variable name::String @@ -165,7 +65,6 @@ end Base.show(io::IO, x::Leaf) = print(io, "Leaf($(x.method))") -export Node mutable struct Node segment::Union{String, Variable} exact::Vector{Node} # sorted alphabetically, all x.segment are String @@ -352,8 +251,8 @@ end default404(::Request) = Response(404) default405(::Request) = Response(405) -default404(s::Stream) = setstatus(s, 404) -default405(s::Stream) = setstatus(s, 405) +# default404(s::Stream) = setstatus(s, 404) +# default405(s::Stream) = setstatus(s, 405) Router(_404=default404, _405=default405, middleware=nothing) = Router(_404, _405, Node(), middleware) @@ -388,8 +287,7 @@ register!(r::Router, path, handler) = register!(r, "*", path, handler) const Params = Dict{String, String} function gethandler(r::Router, req::Request) - url = URI(req.target) - segments = split(url.path, '/'; keepempty=false) + segments = split(req.path, '/'; keepempty=false) leaf = match(r.routes, req.method, segments, 1) params = Params() if leaf isa Leaf @@ -405,23 +303,23 @@ function gethandler(r::Router, req::Request) return leaf, "", params end -function (r::Router)(stream::Stream{<:Request}) - req = stream.message - handler, route, params = gethandler(r, req) - if handler === nothing - # didn't match a registered route - return r._404(stream) - elseif handler === missing - # matched the path, but method not supported - return r._405(stream) - else - req.context[:route] = route - if !isempty(params) - req.context[:params] = params - end - return handler(stream) - end -end +# function (r::Router)(stream::Stream{<:Request}) +# req = stream.message +# handler, route, params = gethandler(r, req) +# if handler === nothing +# # didn't match a registered route +# return r._404(stream) +# elseif handler === missing +# # matched the path, but method not supported +# return r._405(stream) +# else +# req.context[:route] = route +# if !isempty(params) +# req.context[:params] = params +# end +# return handler(stream) +# end +# end function (r::Router)(req::Request) handler, route, params = gethandler(r, req) @@ -432,9 +330,9 @@ function (r::Router)(req::Request) # matched the path, but method not supported return r._405(req) else - req.context[:route] = route + req.route = route if !isempty(params) - req.context[:params] = params + req.params = params end return handler(req) end @@ -447,7 +345,7 @@ Retrieve the original route registration string for a request after its url has matched against a router. Helpful for metric logging to ignore matched variables in a path and only see the registered routes. """ -getroute(req) = get(req.context, :route, nothing) +getroute(req) = req.route """ HTTP.getparams(req) -> Dict{String, String} @@ -457,7 +355,7 @@ If a path was registered with a router via `HTTP.register!` like "/api/widget/{id}", then the path parameters are available in the request context and can be retrieved like `id = HTTP.getparams(req)["id"]`. """ -getparams(req) = get(req.context, :params, nothing) +getparams(req) = req.params """ HTTP.getparam(req, name, default=nothing) -> String @@ -469,7 +367,7 @@ If a path was registered with a router via `HTTP.register!` like function getparam(req, name, default=nothing) params = getparams(req) params === nothing && return default - return get(params, name, default) + return Base.get(params, name, default) end """ @@ -481,8 +379,8 @@ request in the request context. Cookies can then be retrieved by calling """ function cookie_middleware(handler) function (req) - if !haskey(req.context, :cookies) - req.context[:cookies] = Cookies.cookies(req) + if req.cookies === nothing + req.cookies = Cookies.cookies(req) end return handler(req) end @@ -496,6 +394,6 @@ are expected to be stored in the `req.context[:cookies]` of the request context as implemented in the [`HTTP.Handlers.cookie_middleware`](@ref) middleware. """ -getcookies(req) = get(() -> Cookie[], req.context, :cookies) +getcookies(req) = req.cookies === nothing ? Cookies.Cookie[] : req.cookies::Vector{Cookies.Cookie} -end # module +end # module Handlers \ No newline at end of file diff --git a/src/multipart.jl b/src/multipart.jl deleted file mode 100644 index f03f2bbbb..000000000 --- a/src/multipart.jl +++ /dev/null @@ -1,227 +0,0 @@ -module Forms - -export Form, Multipart, content_type - -using ..IOExtras, ..Sniff, ..Conditions -import ..HTTP # for doc references - -# Form request body -mutable struct Form <: IO - data::Vector{IO} - index::Int - mark::Int - boundary::String -end - -Form(f::Form) = f -Base.eof(f::Form) = f.index > length(f.data) -Base.isopen(f::Form) = false -Base.close(f::Form) = nothing -Base.length(f::Form) = sum(x->isa(x, IOStream) ? filesize(x) - position(x) : bytesavailable(x), f.data) -IOExtras.nbytes(x::Form) = length(x) - -function Base.mark(f::Form) - foreach(mark, f.data) - f.mark = f.index - return -end - -function Base.reset(f::Form) - foreach(reset, f.data) - f.index = f.mark - f.mark = -1 - return -end - -function Base.unmark(f::Form) - foreach(unmark, f.data) - f.mark = -1 - return -end - -function Base.position(f::Form) - index = f.index - foreach(mark, f.data) - return index -end -function Base.seek(f::Form, pos) - f.index = pos - foreach(reset, f.data) - return -end - -Base.readavailable(f::Form) = read(f) -function Base.read(f::Form) - result = UInt8[] - for io in f.data - append!(result, read(io)) - end - f.index = length(f.data) + 1 - return result -end - -function Base.read(f::Form, n::Integer) - nb = 0 - result = UInt8[] - while nb < n - d = read(f.data[f.index], n - nb) - nb += length(d) - append!(result, d) - eof(f.data[f.index]) && (f.index += 1) - f.index > length(f.data) && break - end - return result -end - -""" - HTTP.Form(data; boundary=string(rand(UInt128), base=16)) - -Construct a request body for multipart/form-data encoding from `data`. - -`data` must iterate key-value pairs (e.g. `AbstractDict` or `Vector{Pair}`) where the key/value of the -iterator is the key/value of each mutipart boundary chunk. -Files and other large data arguments can be provided as values as IO arguments: either an `IOStream` -such as returned via `open(file)`, or an `IOBuffer` for in-memory data. - -For complete control over a multipart chunk's details, an -[`HTTP.Multipart`](@ref) type is provided to support setting the `filename`, `Content-Type`, -and `Content-Transfer-Encoding`. - -# Examples -```julia -data = Dict( - "text" => "text data", - # filename (cat.png) and content-type (image/png) inferred from the IOStream - "file1" => open("cat.png"), - # manully controlled chunk - "file2" => HTTP.Multipart("dog.jpeg", open("mydog.jpg"), "image/jpeg"), -) -body = HTTP.Form(data) -headers = [] -HTTP.post(url, headers, body) -``` -""" -function Form(d; boundary=string(rand(UInt128), base=16)) - # https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html - bcharsnospace = raw"\w'\(\)\+,-\./:=\?" - boundary_re = Regex("^[$bcharsnospace ]{0,69}[$bcharsnospace]\$") - @require match(boundary_re, boundary) !== nothing - @require eltype(d) <: Pair - data = IO[] - io = IOBuffer() - len = length(d) - for (i, (k, v)) in enumerate(d) - write(io, (i == 1 ? "" : "\r\n") * "--" * boundary * "\r\n") - write(io, "Content-Disposition: form-data; name=\"$k\"") - if isa(v, IO) - writemultipartheader(io, v) - seekstart(io) - push!(data, io) - push!(data, v) - io = IOBuffer() - else - write(io, "\r\n\r\n") - write(io, v) - end - end - # write final boundary - write(io, "\r\n--" * boundary * "--" * "\r\n") - seekstart(io) - push!(data, io) - return Form(data, 1, -1, boundary) -end - -function writemultipartheader(io::IOBuffer, i::IOStream) - write(io, "; filename=\"$(basename(i.name[7:end-1]))\"\r\n") - write(io, "Content-Type: $(sniff(i))\r\n\r\n") - return -end -function writemultipartheader(io::IOBuffer, i::IO) - write(io, "\r\n\r\n") - return -end - -""" - HTTP.Multipart(filename::String, data::IO, content_type=HTTP.sniff(data), content_transfer_encoding="") - -A type to represent a single multipart upload chunk for a file. This type would be used as the value in a -key-value pair when constructing a [`HTTP.Form`](@ref) for a request body (see example below). -The `data` argument must be an `IO` type such as `IOStream`, or `IOBuffer`. -The `content_type` and `content_transfer_encoding` arguments allow manual setting of these multipart headers. -`Content-Type` will default to the result of the `HTTP.sniff(data)` mimetype detection algorithm, whereas -`Content-Transfer-Encoding` will be left out if not specified. - -# Examples -```julia -body = HTTP.Form(Dict( - "key" => HTTP.Multipart("File.txt", open("MyFile.txt"), "text/plain"), -)) -headers = [] -HTTP.post(url, headers, body) -``` - -# Extended help - -Filename SHOULD be included when the Multipart represents the contents of a file -[RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) - -Content-Disposition set to "form-data" MUST be included with each Multipart. -An additional "name" parameter MUST be included -An optional "filename" parameter SHOULD be included if the contents of a file are sent -This will be formatted such as: - Content-Disposition: form-data; name="user"; filename="myfile.txt" -[RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) - -Content-Type for each Multipart is optional, but SHOULD be included if the contents -of a file are sent. -[RFC7578 4.4](https://tools.ietf.org/html/rfc7578#section-4.4) - -Content-Transfer-Encoding for each Multipart is deprecated -[RFC7578 4.7](https://tools.ietf.org/html/rfc7578#section-4.7) - -Other Content- header fields MUST be ignored -[RFC7578 4.8](https://tools.ietf.org/html/rfc7578#section-4.8) -""" -mutable struct Multipart{T <: IO} <: IO - filename::Union{String, Nothing} - data::T - contenttype::String - contenttransferencoding::String - name::String -end - -function Multipart(f::Union{AbstractString, Nothing}, data::T, ct::AbstractString="", cte::AbstractString="", name::AbstractString="") where {T<:IO} - f = f !== nothing ? String(f) : nothing - return Multipart{T}(f, data, String(ct), String(cte), String(name)) -end - -function Base.show(io::IO, m::Multipart{T}) where {T} - items = ["data=::$T", "contenttype=\"$(m.contenttype)\"", "contenttransferencoding=\"$(m.contenttransferencoding)\")"] - m.filename === nothing || pushfirst!(items, "filename=\"$(m.filename)\"") - print(io, "HTTP.Multipart($(join(items, ", ")))") -end - -Base.bytesavailable(m::Multipart{T}) where {T} = isa(m.data, IOStream) ? filesize(m.data) - position(m.data) : bytesavailable(m.data) -Base.eof(m::Multipart{T}) where {T} = eof(m.data) -Base.read(m::Multipart{T}, n::Integer) where {T} = read(m.data, n) -Base.read(m::Multipart{T}) where {T} = read(m.data) -Base.mark(m::Multipart{T}) where {T} = mark(m.data) -Base.reset(m::Multipart{T}) where {T} = reset(m.data) -Base.seekstart(m::Multipart{T}) where {T} = seekstart(m.data) - -function writemultipartheader(io::IOBuffer, i::Multipart) - if i.filename === nothing - write(io, "\r\n") - else - write(io, "; filename=\"$(i.filename)\"\r\n") - end - contenttype = i.contenttype == "" ? sniff(i.data) : i.contenttype - write(io, "Content-Type: $(contenttype)\r\n") - write(io, i.contenttransferencoding == "" ? "\r\n" : "Content-Transfer-Encoding: $(i.contenttransferencoding)\r\n\r\n") - return -end - -content_type(f::Form) = "Content-Type" => - "multipart/form-data; boundary=$(f.boundary)" - -end # module \ No newline at end of file diff --git a/src/parsemultipart.jl b/src/parsemultipart.jl deleted file mode 100644 index 5fbf6e2ee..000000000 --- a/src/parsemultipart.jl +++ /dev/null @@ -1,257 +0,0 @@ -module MultiPartParsing - -import ..access_threaded -using ..Messages, ..Forms, ..Parsers - -export parse_multipart_form - -const CR_BYTE = 0x0d # \r -const LF_BYTE = 0x0a # \n -const DASH_BYTE = 0x2d # - -const HTAB_BYTE = 0x09 # \t -const SPACE_BYTE = 0x20 -const SEMICOLON_BYTE = UInt8(';') -const CRLFCRLF = (CR_BYTE, LF_BYTE, CR_BYTE, LF_BYTE) - -"compare byte buffer `a` from index `i` to index `j` with `b` and check if they are byte-equal" -function byte_buffers_eq(a, i, j, b) - l = 1 - @inbounds for k = i:j - a[k] == b[l] || return false - l += 1 - end - return true -end - -""" - find_multipart_boundary(bytes, boundaryDelimiter; start::Int=1) - -Find the first and last index of the next boundary delimiting a part, and if -the discovered boundary is the terminating boundary. -""" -function find_multipart_boundary(bytes::AbstractVector{UInt8}, boundaryDelimiter::AbstractVector{UInt8}; start::Int=1) - # The boundary delimiter line is prepended with two '-' characters - # The boundary delimiter line starts on a new line, so must be preceded by a \r\n. - # The boundary delimiter line ends with \r\n, and can have "optional linear whitespace" between - # the end of the boundary delimiter, and the \r\n. - # The last boundary delimiter line has an additional '--' at the end of the boundary delimiter - # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) - - i = start - end_index = i + length(boundaryDelimiter) + 1 - while end_index <= length(bytes) - if bytes[i] == DASH_BYTE && bytes[i + 1] == DASH_BYTE && byte_buffers_eq(bytes, i + 2, end_index, boundaryDelimiter) - # boundary delimiter line start on a new line ... - if i > 1 - (i == 2 || bytes[i-2] != CR_BYTE || bytes[i-1] != LF_BYTE) && error("boundary delimiter found, but it was not the start of a line") - # the CRLF preceding the boundary delimiter is "conceptually attached - # to the boundary", so account for this with the index - i -= 2 - end - - # need to check if there are enough characters for the CRLF or for two dashes - end_index < length(bytes)-1 || error("boundary delimiter found, but did not end with new line") - - is_terminating_delimiter = bytes[end_index+1] == DASH_BYTE && bytes[end_index+2] == DASH_BYTE - is_terminating_delimiter && (end_index += 2) - - # ... there can be arbitrary SP and HTAB space between the boundary delimiter ... - while end_index < length(bytes) && (bytes[end_index+1] in (HTAB_BYTE, SPACE_BYTE)) - end_index += 1 - end - # ... and ends with a new line - newlineEnd = end_index < length(bytes)-1 && - bytes[end_index+1] == CR_BYTE && - bytes[end_index+2] == LF_BYTE - if !newlineEnd - error("boundary delimiter found, but did not end with new line") - end - - end_index += 2 - - return (is_terminating_delimiter, i, end_index) - end - - i += 1 - end_index += 1 - end - - error("boundary delimiter not found") -end - -""" - find_multipart_boundaries(bytes, boundary; start=1) - -Find the start and end indexes of all the parts of the multipart object. Ultimately this method is -looking for the data between the boundary delimiters in the byte array. A vector containing all -the start/end pairs is returned. -""" -function find_multipart_boundaries(bytes::AbstractVector{UInt8}, boundary::AbstractVector{UInt8}; start=1) - idxs = Tuple{Int, Int}[] - while true - (is_terminating_delimiter, i, end_index) = find_multipart_boundary(bytes, boundary; start = start) - push!(idxs, (i, end_index)) - is_terminating_delimiter && break - start = end_index + 1 - end - return idxs -end - -""" - find_header_boundary(bytes) - -Find the end of the multipart header in the byte array. Returns a Tuple with the -start index(1) and the end index. Headers are separated from the body by CRLFCRLF. - -[RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1) -[RFC822 3.1](https://tools.ietf.org/html/rfc822#section-3.1) -""" -function find_header_boundary(bytes::AbstractVector{UInt8}) - length(CRLFCRLF) > length(bytes) && return nothing - - l = length(bytes) - length(CRLFCRLF) + 1 - i = 1 - end_index = length(CRLFCRLF) - while (i <= l) - byte_buffers_eq(bytes, i, end_index, CRLFCRLF) && return (1, end_index) - i += 1 - end_index += 1 - end - error("no delimiter found separating header from multipart body") -end - -const content_disposition_regex = Parsers.RegexAndMatchData[] -function content_disposition_regex_f() - r = Parsers.RegexAndMatchData(r"^[Cc]ontent-[Dd]isposition:[ \t]*form-data;[ \t]*(.*)\r\n"mx) - Parsers.init!(r) -end - -const content_disposition_flag_regex = Parsers.RegexAndMatchData[] -function content_disposition_flag_regex_f() - r = Parsers.RegexAndMatchData(r"""^ - [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+);? - """x) - Parsers.init!(r) -end - -const content_disposition_pair_regex = Parsers.RegexAndMatchData[] -function content_disposition_pair_regex_f() - r = Parsers.RegexAndMatchData(r"""^ - [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+)[ \t]*=[ \t]*"(.*?)";? - """x) - Parsers.init!(r) -end - -const content_type_regex = Parsers.RegexAndMatchData[] -function content_type_regex_f() - r = Parsers.RegexAndMatchData(r"(?i)Content-Type: (\S*[^;\s])") - Parsers.init!(r) -end - -""" - parse_multipart_chunk(chunk) - -Parse a single multi-part chunk into a Multipart object. This will decode -the header and extract the contents from the byte array. -""" -function parse_multipart_chunk(chunk) - startIndex, end_index = find_header_boundary(chunk) - header = SubString(unsafe_string(pointer(chunk, startIndex), end_index - startIndex + 1)) - content = view(chunk, end_index+1:lastindex(chunk)) - - # find content disposition - re = access_threaded(content_disposition_regex_f, content_disposition_regex) - if !Parsers.exec(re, header) - @warn "Content disposition is not specified dropping the chunk." String(chunk) - return nothing # Specifying content disposition is mandatory - end - content_disposition = Parsers.group(1, re, header) - - re_flag = access_threaded(content_disposition_flag_regex_f, content_disposition_flag_regex) - re_pair = access_threaded(content_disposition_pair_regex_f, content_disposition_pair_regex) - name = nothing - filename = nothing - while !isempty(content_disposition) - if Parsers.exec(re_pair, content_disposition) - key = Parsers.group(1, re_pair, content_disposition) - value = Parsers.group(2, re_pair, content_disposition) - if key == "name" - name = value - elseif key == "filename" - filename = value - else - # do stuff with other content disposition key-value pairs - end - content_disposition = Parsers.nextbytes(re_pair, content_disposition) - elseif Parsers.exec(re_flag, content_disposition) - # do stuff with content disposition flags - content_disposition = Parsers.nextbytes(re_flag, content_disposition) - else - break - end - end - - name === nothing && return - - re_ct = access_threaded(content_type_regex_f, content_type_regex) - contenttype = Parsers.exec(re_ct, header) ? Parsers.group(1, re_ct, header) : "text/plain" - - return Multipart(filename, IOBuffer(content), contenttype, "", name) -end - -""" - parse_multipart_body(body, boundary)::Vector{Multipart} - -Parse the multipart body received from the client breaking it into the various -chunks which are returned as an array of Multipart objects. -""" -function parse_multipart_body(body::AbstractVector{UInt8}, boundary::AbstractString)::Vector{Multipart} - multiparts = Multipart[] - idxs = find_multipart_boundaries(body, codeunits(boundary)) - length(idxs) > 1 || (return multiparts) - - for i in 1:length(idxs)-1 - chunk = view(body, idxs[i][2]+1:idxs[i+1][1]-1) - push!(multiparts, parse_multipart_chunk(chunk)) - end - return multiparts -end - - -""" - parse_multipart_form(req::Request)::Vector{Multipart} - -Parse the full mutipart form submission from the client returning and -array of Multipart objects containing all the data. - -The order of the multipart form data in the request should be preserved. -[RFC7578 5.2](https://tools.ietf.org/html/rfc7578#section-5.2). - -The boundary delimiter MUST NOT appear inside any of the encapsulated parts. Note -that the boundary delimiter does not need to have '-' characters, but a line using -the boundary delimiter will start with '--' and end in \r\n. -[RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) -""" -function parse_multipart_form(msg::Message)::Union{Vector{Multipart}, Nothing} - # parse boundary from Content-Type - m = match(r"multipart/form-data; boundary=(.*)$", msg["Content-Type"]) - m === nothing && return nothing - - boundary_delimiter = m[1] - - # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) - length(boundary_delimiter) > 70 && error("boundary delimiter must not be greater than 70 characters") - - return parse_multipart_body(payload(msg), boundary_delimiter) -end - -function __init__() - nt = isdefined(Base.Threads, :maxthreadid) ? Threads.maxthreadid() : Threads.nthreads() - resize!(empty!(content_disposition_regex), nt) - resize!(empty!(content_disposition_flag_regex), nt) - resize!(empty!(content_disposition_pair_regex), nt) - resize!(empty!(content_type_regex), nt) - return -end - -end # module MultiPartParsing diff --git a/src/parseutils.jl b/src/parseutils.jl deleted file mode 100644 index 1849e22fc..000000000 --- a/src/parseutils.jl +++ /dev/null @@ -1,48 +0,0 @@ -# we define our own RegexAndMatchData, similar to the definition in Base -# but we call create_match_data once in __init__ -mutable struct RegexAndMatchData - re::Regex - match_data::Ptr{Cvoid} - RegexAndMatchData(re::Regex) = new(re) # must create_match_data in __init__ -end - -function initialize!(re::RegexAndMatchData) - re.match_data = Base.PCRE.create_match_data(re.re.regex) - return -end - -@static if isdefined(Base, :RegexAndMatchData) - -""" -Execute a regular expression without the overhead of `Base.Regex` -""" -exec(re::RegexAndMatchData, bytes, offset::Int=1) = - Base.PCRE.exec(re.re.regex, bytes, offset-1, re.re.match_options, re.match_data) - -""" -`SubString` containing the bytes following the matched regular expression. -""" -nextbytes(re::RegexAndMatchData, bytes) = SubString(bytes, unsafe_load(Base.PCRE.ovec_ptr(re.match_data), 2) + 1) - -""" -`SubString` containing a regular expression match group. -""" -function group(i, re::RegexAndMatchData, bytes) - p = Base.PCRE.ovec_ptr(re.match_data) - SubString(bytes, unsafe_load(p, 2i+1) + 1, prevind(bytes, unsafe_load(p, 2i+2) + 1)) -end - -function group(i, re::RegexAndMatchData, bytes, default) - p = Base.PCRE.ovec_ptr(re.match_data) - return unsafe_load(p, 2i+1) == Base.PCRE.UNSET ? default : - SubString(bytes, unsafe_load(p, 2i+1) + 1, prevind(bytes, unsafe_load(p, 2i+2) + 1)) -end - -else # old Regex style - -exec(re::RegexAndMatchData, bytes, offset::Int=1) = Base.PCRE.exec(re.re.regex, bytes, offset-1, re.re.match_options, re.re.match_data) -nextbytes(re::RegexAndMatchData, bytes) = SubString(bytes, re.re.ovec[2]+1) -group(i, re::RegexAndMatchData, bytes) = SubString(bytes, re.re.ovec[2i+1]+1, prevind(bytes, re.re.ovec[2i+2]+1)) -group(i, re::RegexAndMatchData, bytes, default) = re.re.ovec[2i+1] == Base.PCRE.UNSET ? default : SubString(bytes, re.re.ovec[2i+1]+1, prevind(bytes, re.re.ovec[2i+2]+1)) - -end diff --git a/src/precompile.jl b/src/precompile.jl index d6cc822df..c382dbacc 100644 --- a/src/precompile.jl +++ b/src/precompile.jl @@ -2,44 +2,31 @@ using PrecompileTools: @setup_workload, @compile_workload try @setup_workload begin - # These need to be safe to call here and bake into the pkgimage, i.e. called twice. - Connections.__init__() - MultiPartParsing.__init__() - Parsers.__init__() - - # Doesn't seem to be needed here, and might not be safe to call twice (here and during runtime) - # ConnectionRequest.__init__() - + HTTP.__init__() + # HTTP.set_log_level!(4) gzip_data(data::String) = read(GzipCompressorStream(IOBuffer(data))) - - # random port in the dynamic/private range (49152–65535) which are are - # least likely to be used by well-known services - _port = 57813 - - cert, key = joinpath.(@__DIR__, "../test", "resources", ("cert.pem", "key.pem")) - sslconfig = MbedTLS.SSLConfig(cert, key) - - server = HTTP.serve!("0.0.0.0", _port; verbose = -1, listenany=true, sslconfig=sslconfig) do req - HTTP.Response(200, ["Content-Encoding" => "gzip"], gzip_data("dummy response")) - end - try + @compile_workload begin + # random port in the dynamic/private range (49152–65535) which are are + # least likely to be used by well-known services + _port = 57813 + server = HTTP.serve!("0.0.0.0", _port; listenany=true) do req + HTTP.Response(200, ["Content-Encoding" => "gzip"], gzip_data("dummy response")) + end # listenany allows changing port if that one is already in use, so check the actual port _port = HTTP.port(server) - url = "https://localhost:$_port" - - env = ["JULIA_NO_VERIFY_HOSTS" => "localhost", - "JULIA_SSL_NO_VERIFY_HOSTS" => nothing, - "JULIA_ALWAYS_VERIFY_HOSTS" => nothing] - - @compile_workload begin - withenv(env...) do - HTTP.get(url); - end + url = "http://localhost:$_port" + try + HTTP.get(url) + finally + close(server) + yield() # needed on 1.9 to avoid some issue where it seems a task doesn't stop before serialization + server = nothing + close_all_clients!() + close_default_aws_server_bootstrap!() + close_default_aws_client_bootstrap!() + close_default_aws_host_resolver!() + close_default_aws_event_loop_group!() end - finally - HTTP.forceclose(server) - yield() # needed on 1.9 to avoid some issue where it seems a task doesn't stop before serialization - server = nothing end end catch e diff --git a/src/requestresponse.jl b/src/requestresponse.jl new file mode 100644 index 000000000..8b97edefe --- /dev/null +++ b/src/requestresponse.jl @@ -0,0 +1,416 @@ +# working with headers +headereq(a::String, b::String) = GC.@preserve a b aws_http_header_name_eq(aws_byte_cursor_from_c_str(a), aws_byte_cursor_from_c_str(b)) + +mutable struct Header + header::aws_http_header + Header() = new() + Header(header::aws_http_header) = new(header) +end + +function Base.getproperty(x::Header, s::Symbol) + if s == :name + return str(getfield(x, :header).name) + elseif s == :value + return str(getfield(x, :header).value) + else + return getfield(x, s) + end +end + +Base.show(io::IO, h::Header) = print_header(io, h) + +mutable struct Headers <: AbstractVector{Header} + const ptr::Ptr{aws_http_headers} + function Headers(allocator=default_aws_allocator()) + x = new(aws_http_headers_new(allocator)) + x.ptr == C_NULL && aws_throw_error() + return finalizer(_ -> aws_http_headers_release(x.ptr), x) + end + # no finalizer in this constructor because whoever called aws_http_headers_new needs to do that + Headers(ptr::Ptr{aws_http_headers}) = new(ptr) +end + +Base.size(h::Headers) = (Int(aws_http_headers_count(h.ptr)),) + +function Base.getindex(h::Headers, i::Int) + header = Header() + aws_http_headers_get_index(h.ptr, i - 1, FieldRef(header, :header)) != 0 && aws_throw_error() + return header +end + +Base.Dict(h::Headers) = Dict(((h.name, h.value) for h in h)) + +addheader(headers::Headers, h::Header) = aws_http_headers_add_header(headers.ptr, FieldRef(h, :header)) != 0 && aws_throw_error() +addheader(headers::Headers, k, v) = GC.@preserve k v aws_http_headers_add(headers.ptr, aws_byte_cursor_from_c_str(k), aws_byte_cursor_from_c_str(v)) != 0 && aws_throw_error() +addheaders(headers::Headers, h::Vector{aws_http_header}) = GC.@preserve h aws_http_headers_add_array(headers.ptr, pointer(h), length(h)) != 0 && aws_throw_error() +addheaders(headers::Headers, h::Ptr{aws_http_header}, count::Integer) = aws_http_headers_add_array(headers.ptr, h, count) != 0 && aws_throw_error() + +function addheaders(headers::Headers, h::Vector{Pair{String, String}}) + for (k, v) in h + addheader(headers, k, v) + end +end + +setheader(headers::Headers, k, v) = GC.@preserve k v aws_http_headers_set(headers.ptr, aws_byte_cursor_from_c_str(k), aws_byte_cursor_from_c_str(v)) != 0 && aws_throw_error() +setscheme(headers::Headers, scheme) = GC.@preserve scheme aws_http2_headers_set_request_scheme(headers.ptr, aws_byte_cursor_from_c_str(scheme)) != 0 && aws_throw_error() +setauthority(headers::Headers, authority) = GC.@preserve authority aws_http2_headers_set_request_authority(headers.ptr, aws_byte_cursor_from_c_str(authority)) != 0 && aws_throw_error() + +#TODO: struct aws_string *aws_http_headers_get_all(const struct aws_http_headers *headers, struct aws_byte_cursor name); +function getheader(headers::Headers, k) + out = Ref{aws_byte_cursor}() + GC.@preserve k out begin + aws_http_headers_get(headers.ptr, aws_byte_cursor_from_c_str(k), out) != 0 && return nothing + return str(out[]) + end +end + +hasheader(headers::Headers, k) = + GC.@preserve k aws_http_headers_has(headers.ptr, aws_byte_cursor_from_c_str(k)) + +removeheader(headers::Headers, k) = + GC.@preserve k aws_http_headers_erase(headers.ptr, aws_byte_cursor_from_c_str(k)) != 0 && aws_throw_error() + +removeheader(headers::Headers, k, v) = + GC.@preserve k v aws_http_headers_erase_value(headers.ptr, aws_byte_cursor_from_c_str(k), aws_byte_cursor_from_c_str(v)) != 0 && aws_throw_error() + +Base.deleteat!(h::Headers, i::Int) = aws_http_headers_erase_index(h.ptr, i - 1) != 0 && aws_throw_error() +Base.empty!(h::Headers) = aws_http_headers_clear(h.ptr) != 0 && aws_throw_error() + +setheaderifabsent(headers, k, v) = !hasheader(headers, k) && setheader(headers, k, v) + +# request/response +abstract type Message end + +mutable struct InputStream + ptr::Ptr{aws_input_stream} + bodyref::Any + bodylen::Int64 + bodycursor::aws_byte_cursor + InputStream() = new() +end + +ischunked(is::InputStream) = is.ptr == C_NULL && is.bodyref !== nothing + +const RequestBodyTypes = Union{AbstractString, AbstractVector{UInt8}, IO, AbstractDict, NamedTuple, Nothing} + +function InputStream(allocator::Ptr{aws_allocator}, body::RequestBodyTypes) + is = InputStream() + if body !== nothing + if body isa RequestBodyTypes + if (body isa AbstractVector{UInt8}) || (body isa AbstractString) + is.bodyref = body + is.bodycursor = aws_byte_cursor(sizeof(body), pointer(body)) + is.ptr = aws_input_stream_new_from_cursor(allocator, FieldRef(is, :bodycursor)) + elseif body isa Union{AbstractDict, NamedTuple} + # hold a reference to the request body in order to gc-preserve it + is.bodyref = URIs.escapeuri(body) + is.bodycursor = aws_byte_cursor_from_c_str(is.bodyref) + is.ptr = aws_input_stream_new_from_cursor(allocator, FieldRef(is, :bodycursor)) + elseif body isa IOStream + is.bodyref = body + is.ptr = aws_input_stream_new_from_open_file(allocator, Libc.FILE(body)) + elseif body isa Form + # we set the request.body to the Form bytes in order to gc-preserve them + is.bodyref = read(body) + is.bodycursor = aws_byte_cursor(sizeof(is.bodyref), pointer(is.bodyref)) + is.ptr = aws_input_stream_new_from_cursor(allocator, FieldRef(is, :bodycursor)) + elseif body isa IO + # we set the request.body to the IO bytes in order to gc-preserve them + bytes = readavailable(body) + while !eof(body) + append!(bytes, readavailable(body)) + end + is.bodyref = bytes + is.bodycursor = aws_byte_cursor(sizeof(is.bodyref), pointer(is.bodyref)) + is.ptr = aws_input_stream_new_from_cursor(allocator, FieldRef(is, :bodycursor)) + else + throw(ArgumentError("request body must be a string, vector of UInt8, NamedTuple, AbstractDict, HTTP.Form, or IO")) + end + aws_input_stream_get_length(is.ptr, FieldRef(is, :bodylen)) != 0 && aws_throw_error() + if !(is.bodylen > 0) + aws_input_stream_release(is.ptr) + is.ptr = C_NULL + end + else + # assume a chunked request body; any kind of iterable where elements are RequestBodyTypes + @assert Base.isiterable(typeof(body)) "chunked request body must be an iterable" + is.bodyref = body + end + end + return finalizer(x -> aws_input_stream_release(x.ptr), is) +end + +function setinputstream!(msg::Message, body) + aws_http_message_set_body_stream(msg.ptr, C_NULL) + msg.inputstream = nothing + body === nothing && return + input_stream = InputStream(msg.allocator, body) + setfield!(msg, :inputstream, input_stream) + if input_stream.ptr != C_NULL + aws_http_message_set_body_stream(msg.ptr, input_stream.ptr) + if body isa Union{AbstractDict, NamedTuple} + setheaderifabsent(msg.headers, "content-type", "application/x-www-form-urlencoded") + elseif body isa Form + setheaderifabsent(msg.headers, "content-type", content_type(body)) + end + setheader(msg.headers, "content-length", string(input_stream.bodylen)) + end + return +end + +mutable struct Request <: Message + allocator::Ptr{aws_allocator} + ptr::Ptr{aws_http_message} + inputstream::Union{Nothing, InputStream} # used for outgoing request body + # only set in server-side request handlers + body::Union{Nothing, Vector{UInt8}} + route::Union{Nothing, String} + params::Union{Nothing, Dict{String, String}} + cookies::Any # actually Union{Nothing, Vector{Cookie}} + + function Request(method, path, headers=nothing, body=nothing, http2::Bool=false, allocator=default_aws_allocator()) + ptr = http2 ? + aws_http2_message_new_request(allocator) : + aws_http_message_new_request(allocator) + ptr == C_NULL && aws_throw_error() + try + GC.@preserve method aws_http_message_set_request_method(ptr, aws_byte_cursor_from_c_str(method)) != 0 && aws_throw_error() + GC.@preserve path aws_http_message_set_request_path(ptr, aws_byte_cursor_from_c_str(path)) != 0 && aws_throw_error() + request_headers = Headers(aws_http_message_get_headers(ptr)) + if headers !== nothing + for (k, v) in headers + addheader(request_headers, k, v) + end + end + req = new(allocator, ptr) + req.body = nothing + req.inputstream = nothing + req.route = nothing + req.params = nothing + req.cookies = nothing + body !== nothing && setinputstream!(req, body) + return finalizer(_ -> aws_http_message_release(ptr), req) + catch + aws_http_message_release(ptr) + rethrow() + end + end +end + +ptr(x) = getfield(x, :ptr) + +function Base.getproperty(x::Request, s::Symbol) + if s == :method + out = Ref{aws_byte_cursor}() + GC.@preserve out begin + aws_http_message_get_request_method(ptr(x), out) != 0 && return nothing + return str(out[]) + end + elseif s == :path || s == :target || s == :uri + out = Ref{aws_byte_cursor}() + GC.@preserve out begin + aws_http_message_get_request_path(ptr(x), out) != 0 && return nothing + path = str(out[]) + return s == :uri ? URI(path) : path + end + elseif s == :headers + return Headers(aws_http_message_get_headers(ptr(x))) + elseif s == :version + return aws_http_message_get_protocol_version(ptr(x)) == AWS_HTTP_VERSION_2 ? "2" : "1.1" + else + return getfield(x, s) + end +end + +function Base.setproperty!(x::Request, s::Symbol, v) + if s == :method + GC.@preserve v aws_http_message_set_request_method(x.ptr, aws_byte_cursor_from_c_str(v)) != 0 && aws_throw_error() + elseif s == :path + GC.@preserve v aws_http_message_set_request_path(x.ptr, aws_byte_cursor_from_c_str(v)) != 0 && aws_throw_error() + elseif s == :headers + addheaders(x.headers, v) + else + setfield!(x, s, v) + end +end + +function print_header(io, h) + key = h.name + val = h.value + if headereq(key, "authorization") + write(io, string(key, ": ", "******", "\r\n")) + return + elseif headereq(key, "proxy-authorization") + write(io, string(key, ": ", "******", "\r\n")) + elseif headereq(key, "cookie") + write(io, string(key, ": ", "******", "\r\n")) + return + elseif headereq(key, "set-cookie") + write(io, string(key, ": ", "******", "\r\n")) + else + write(io, string(key, ": ", val, "\r\n")) + return + end +end + +function print_request(io, method, version, path, headers, body) + write(io, "\"\"\"\n") + write(io, string(method, " ", path, " HTTP/$version\r\n")) + for h in headers + print_header(io, h) + end + write(io, "\r\n") + body !== nothing ? write(io, body) : write(io, "[request body streamed]") + write(io, "\n\"\"\"\n") + return +end + +getbody(r::Message) = isdefined(r, :inputstream) ? r.inputstream.bodyref : r.body + +print_request(io::IO, r::Request) = print_request(io, r.method, r.version, r.path, r.headers, getbody(r)) + +function Base.show(io::IO, r::Request) + println(io, "HTTP.Request:") + print_request(io, r) +end + +method(r::Request) = r.method +target(r::Request) = r.path +headers(r::Request) = r.headers +body(r::Request) = r.body + +resource(uri::URI) = string(isempty(uri.path) ? "/" : uri.path, + !isempty(uri.query) ? "?" : "", uri.query, + !isempty(uri.fragment) ? "#" : "", uri.fragment) + +mutable struct RequestMetrics + request_body_length::Int + response_body_length::Int + nretries::Int + stream_metrics::Union{Nothing, aws_http_stream_metrics} +end + +RequestMetrics() = RequestMetrics(0, 0, 0, nothing) + +mutable struct Response <: Message + allocator::Ptr{aws_allocator} + ptr::Ptr{aws_http_message} + inputstream::Union{Nothing, InputStream} + body::Union{Nothing, Vector{UInt8}} # only set for client-side response body when no user-provided response_body + metrics::RequestMetrics + request::Request + + function Response(status::Integer, headers, body, http2::Bool=false, allocator=default_aws_allocator()) + ptr = http2 ? + aws_http2_message_new_response(allocator) : + aws_http_message_new_response(allocator) + ptr == C_NULL && aws_throw_error() + try + GC.@preserve status aws_http_message_set_response_status(ptr, status) != 0 && aws_throw_error() + response_headers = Headers(aws_http_message_get_headers(ptr)) + if headers !== nothing + for (k, v) in headers + addheader(response_headers, k, v) + end + end + resp = new(allocator, ptr) + resp.body = nothing + resp.inputstream = nothing + body !== nothing && setinputstream!(resp, body) + return finalizer(_ -> aws_http_message_release(ptr), resp) + catch + aws_http_message_release(ptr) + rethrow() + end + end + Response() = new(C_NULL, C_NULL, nothing, nothing) +end + +Response(status::Integer, body) = Response(status, nothing, Vector{UInt8}(string(body))) +Response(status::Integer) = Response(status, nothing, nothing) + +getresponse(r::Response) = r + +bodylen(m::Message) = isdefined(m, :inputstream) && m.inputstream !== nothing ? m.inputstream.bodylen : 0 + +function Base.getproperty(x::Response, s::Symbol) + if s == :status + ref = Ref{Cint}() + aws_http_message_get_response_status(x.ptr, ref) != 0 && return nothing + return Int(ref[]) + elseif s == :headers + return Headers(aws_http_message_get_headers(x.ptr)) + elseif s == :version + return aws_http_message_get_protocol_version(x.ptr) == AWS_HTTP_VERSION_2 ? "2" : "1.1" + else + return getfield(x, s) + end +end + +function Base.setproperty!(x::Response, s::Symbol, v) + if s == :status + GC.@preserve v aws_http_message_set_response_status(x.ptr, v) != 0 && aws_throw_error() + elseif s == :headers + addheaders(x.headers, v) + else + setfield!(x, s, v) + end +end + +function print_response(io, status, version, headers, body) + write(io, "\"\"\"\n") + write(io, string("HTTP/$version ", status, "\r\n")) + for h in headers + print_header(io, h) + end + write(io, "\r\n") + body !== nothing ? write(io, body) : write(io, "[response body streamed]") + write(io, "\n\"\"\"\n") + return +end + +print_response(io::IO, r::Response) = print_response(io, r.status, r.version, r.headers, r.body) + +function Base.show(io::IO, r::Response) + println(io, "HTTP.Response:") + print_response(io, r) +end + +status(r::Response) = r.status +headers(r::Response) = r.headers +body(r::Response) = r.body + +""" + issafe(::Request) + +https://tools.ietf.org/html/rfc7231#section-4.2.1 +""" +issafe(r::Request) = issafe(r.method) +issafe(method::String) = method in ("GET", "HEAD", "OPTIONS", "TRACE") + +""" + iserror(::Response) + +Does this `Response` have an error status? +""" +iserror(r::Response) = iserror(r.status) +iserror(status::Integer) = status != 0 && status != 100 && status != 101 && + (status < 200 || status >= 300) && !isredirect(status) + +""" + isidempotent(::Request) + +https://tools.ietf.org/html/rfc7231#section-4.2.2 +""" +isidempotent(r::Request) = isidempotent(r.method) +isidempotent(method::String) = issafe(method) || method in ("PUT", "DELETE") + +""" + isredirect(::Response) + +Does this `Response` have a redirect status? +""" +isredirect(r::Response) = isredirect(r.status) +isredirect(status::Integer) = status in (301, 302, 303, 307, 308) + +Forms.parse_multipart_form(m::Message) = parse_multipart_form(getheader(m.headers, "content-type"), m.body) \ No newline at end of file diff --git a/src/server.jl b/src/server.jl new file mode 100644 index 000000000..cd67d3c40 --- /dev/null +++ b/src/server.jl @@ -0,0 +1,369 @@ +socket_endpoint(host, port) = aws_socket_endpoint( + ntuple(i -> i > sizeof(host) ? 0x00 : codeunit(host, i), Base._counttuple(fieldtype(aws_socket_endpoint, :address))), + port % UInt32 +) + +mutable struct Connection{S} + const server::S # Server{F, C} + const allocator::Ptr{aws_allocator} + const connection::Ptr{aws_http_connection} + const streams_lock::ReentrantLock + const streams::Set{Stream} + connection_options::aws_http_server_connection_options + + Connection( + server::S, + allocator::Ptr{aws_allocator}, + connection::Ptr{aws_http_connection}, + ) where {S} = new{S}(server, allocator, connection, ReentrantLock(), Set{Stream}()) +end + +Base.hash(c::Connection, h::UInt) = hash(c.connection, h) + +function remote_address(c::Connection) + socket_ptr = aws_http_connection_get_remote_endpoint(c.connection) + addr = unsafe_load(socket_ptr).address + bytes = Vector{UInt8}(undef, length(addr)) + nul_i = 0 + for i in eachindex(bytes) + b = addr[i] + @inbounds bytes[i] = b + if b == 0x00 + nul_i = i + break + end + end + resize!(bytes, nul_i == 0 ? length(addr) : nul_i - 1) + return String(bytes) +end +remote_port(c::Connection) = Int(unsafe_load(aws_http_connection_get_remote_endpoint(c.connection)).port) +function http_version(c::Connection) + v = aws_http_connection_get_version(c.connection) + return v == AWS_HTTP_VERSION_2 ? "HTTP/2" : "HTTP/1.1" +end + +getinet(host::String, port::Integer) = Sockets.InetAddr(parse(IPAddr, host), port) +getinet(host::IPAddr, port::Integer) = Sockets.InetAddr(host, port) + +mutable struct Server{F, C} + const f::F + const on_stream_complete::C + const fut::Future{Symbol} + const allocator::Ptr{aws_allocator} + const endpoint::aws_socket_endpoint + const socket_options::aws_socket_options + const tls_options::Union{aws_tls_connection_options, Nothing} + const connections_lock::ReentrantLock + const connections::Set{Connection} + const closed::Threads.Event + const access_log::Union{Nothing, Function} + const logstate::Base.CoreLogging.LogState + @atomic state::Symbol # :initializing, :running, :closed + server::Ptr{aws_http_server} + server_options::aws_http_server_options + + Server{F, C}( + f::F, + on_stream_complete::C, + fut::Future{Symbol}, + allocator::Ptr{aws_allocator}, + endpoint::aws_socket_endpoint, + socket_options::aws_socket_options, + tls_options::Union{aws_tls_connection_options, Nothing}, + connections_lock::ReentrantLock, + connections::Set{Connection}, + closed::Threads.Event, + access_log::Union{Nothing, Function}, + logstate::Base.CoreLogging.LogState, + state::Symbol, + ) where {F, C} = new{F, C}(f, on_stream_complete, fut, allocator, endpoint, socket_options, tls_options, connections_lock, connections, closed, access_log, logstate, state) +end + +Base.wait(s::Server) = wait(s.closed) +ftype(::Server{F}) where {F} = F +port(s::Server) = Int(s.endpoint.port) + +function serve!(f, host="127.0.0.1", port=8080; + allocator=default_aws_allocator(), + bootstrap::Ptr{aws_server_bootstrap}=default_aws_server_bootstrap(), + endpoint=nothing, + listenany::Bool=false, + on_stream_complete=nothing, + access_log::Union{Nothing, Function}=nothing, + # socket options + socket_options=nothing, + socket_domain=:ipv4, + connect_timeout_ms::Integer=3000, + keep_alive_interval_sec::Integer=0, + keep_alive_timeout_sec::Integer=0, + keep_alive_max_failed_probes::Integer=0, + keepalive::Bool=false, + # tls options + tls_options=nothing, + ssl_cert=nothing, + ssl_key=nothing, + ssl_capath=nothing, + ssl_cacert=nothing, + ssl_insecure=false, + ssl_alpn_list="h2;http/1.1", + initial_window_size=typemax(UInt64), + ) + addr = getinet(host, port) + if listenany + @show :pre port + port, sock = Sockets.listenany(addr.host, addr.port) + @show :post Int(port) + close(sock) + end + server = Server{typeof(f), typeof(on_stream_complete)}( + f, # RequestHandler + on_stream_complete, + Future{Symbol}(), + allocator, + endpoint !== nothing ? endpoint : socket_endpoint(host, port), + socket_options !== nothing ? socket_options : aws_socket_options( + AWS_SOCKET_STREAM, # socket type + socket_domain == :ipv4 ? AWS_SOCKET_IPV4 : AWS_SOCKET_IPV6, # socket domain + AWS_SOCKET_IMPL_PLATFORM_DEFAULT, # aws_socket_impl_type + connect_timeout_ms, + keep_alive_interval_sec, + keep_alive_timeout_sec, + keep_alive_max_failed_probes, + keepalive, + ntuple(x -> Cchar(0), 16) # network_interface_name + ), + tls_options !== nothing ? tls_options : + any(x -> x !== nothing, (ssl_cert, ssl_key, ssl_capath, ssl_cacert)) ? LibAwsIO.tlsoptions(host; + ssl_cert, + ssl_key, + ssl_capath, + ssl_cacert, + ssl_insecure, + ssl_alpn_list + ) : nothing, + ReentrantLock(), # connections_lock + Set{Connection}(), # connections + Threads.Event(), # closed + access_log, + Base.CoreLogging.current_logstate(), + :initializing, # state + ) + server.server_options = aws_http_server_options( + 1, + allocator, + bootstrap, + pointer(FieldRef(server, :endpoint)), + pointer(FieldRef(server, :socket_options)), + server.tls_options === nothing ? C_NULL : pointer(FieldRef(server, :tls_options)), + initial_window_size, + pointer_from_objref(server), + on_incoming_connection[], + on_destroy_complete[], + false # manual_window_management + ) + server.server = aws_http_server_new(FieldRef(server, :server_options)) + @assert server.server != C_NULL "failed to create server" + @atomic server.state = :running + return server +end + +const on_incoming_connection = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_incoming_connection(aws_server, aws_conn, error_code, server_ptr) + server = unsafe_pointer_to_objref(server_ptr) + Base.CoreLogging.with_logstate(server.logstate) do + if error_code != 0 + @error "incoming connection error" exception=(aws_error(error_code), Base.backtrace()) + return + end + conn = Connection( + server, + server.allocator, + aws_conn, + ) + conn.connection_options = aws_http_server_connection_options( + 1, + pointer_from_objref(conn), + on_incoming_request[], + on_connection_shutdown[] + ) + if aws_http_connection_configure_server( + aws_conn, + FieldRef(conn, :connection_options) + ) != 0 + @error "failed to configure connection" exception=(aws_error(), Base.backtrace()) + return + end + @lock server.connections_lock begin + push!(server.connections, conn) + end + return + end +end + +const on_connection_shutdown = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_connection_shutdown(aws_conn, error_code, conn_ptr) + conn = unsafe_pointer_to_objref(conn_ptr) + Base.CoreLogging.with_logstate(conn.server.logstate) do + if error_code != 0 + @error "connection shutdown error" exception=(aws_error(error_code), Base.backtrace()) + end + @lock conn.server.connections_lock begin + delete!(conn.server.connections, conn) + end + return + end +end + +const on_incoming_request = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_incoming_request(aws_conn, conn_ptr) + conn = unsafe_pointer_to_objref(conn_ptr) + Base.CoreLogging.with_logstate(conn.server.logstate) do + stream = Stream{typeof(conn)}( + conn.allocator, + false, # decompress + aws_http_connection_get_version(aws_conn) == AWS_HTTP_VERSION_2 # http2 + ) + stream.connection = conn + stream.request_handler_options = aws_http_request_handler_options( + 1, + aws_conn, + pointer_from_objref(stream), + on_request_headers[], + on_request_header_block_done[], + on_request_body[], + on_request_done[], + on_server_stream_complete[], + on_destroy[] + ) + stream.request = Request("", "") + stream.ptr = aws_http_stream_new_server_request_handler( + FieldRef(stream, :request_handler_options) + ) + if stream.ptr == C_NULL + @error "failed to create stream" exception=(aws_error(), Base.backtrace()) + else + @lock conn.streams_lock begin + push!(conn.streams, stream) + end + end + return stream.ptr + end +end + +const on_request_headers = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_request_headers(aws_stream_ptr, header_block, header_array::Ptr{aws_http_header}, num_headers, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + headers = stream.request.headers + addheaders(headers, header_array, num_headers) + return Cint(0) +end + +const on_request_header_block_done = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_request_header_block_done(aws_stream_ptr, header_block, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + ret = aws_http_stream_get_incoming_request_method(aws_stream_ptr, FieldRef(stream, :method)) + ret != 0 && return ret + aws_http_message_set_request_method(stream.request.ptr, stream.method) + ret = aws_http_stream_get_incoming_request_uri(aws_stream_ptr, FieldRef(stream, :path)) + ret != 0 && return ret + aws_http_message_set_request_path(stream.request.ptr, stream.path) + return Cint(0) +end + +const on_request_body = Ref{Ptr{Cvoid}}(C_NULL) + +#TODO: how could we allow for streaming request bodies? +function c_on_request_body(aws_stream_ptr, data::Ptr{aws_byte_cursor}, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + bc = unsafe_load(data) + body = stream.request.body + if body === nothing + body = Vector{UInt8}(undef, bc.len) + GC.@preserve body unsafe_copyto!(pointer(body), bc.ptr, bc.len) + stream.request.body = body + else + newlen = length(body) + bc.len + resize!(body, newlen) + GC.@preserve body unsafe_copyto!(pointer(body, length(body) - bc.len + 1), bc.ptr, bc.len) + end + return Cint(0) +end + +const on_request_done = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_request_done(aws_stream_ptr, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + Base.CoreLogging.with_logstate(stream.connection.server.logstate) do + try + stream.response = Base.invokelatest(stream.connection.server.f, stream.request)::Response + if stream.request.method == "HEAD" + setinputstream!(stream.response, nothing) + end + #TODO: is it possible to stream the response body? + #TODO: support transfer-encoding: gzip + catch e + @error "Request handler error; sending 500" exception=(e, catch_backtrace()) + stream.response = Response(500) + end + ret = aws_http_stream_send_response(aws_stream_ptr, stream.response.ptr) + if ret != 0 + @error "failed to send response" exception=(aws_error(ret), Base.backtrace()) + return Cint(AWS_ERROR_HTTP_UNKNOWN) + end + return Cint(0) + end +end + +const on_server_stream_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_server_stream_complete(aws_stream_ptr, error_code, stream_ptr) + stream = unsafe_pointer_to_objref(stream_ptr) + Base.CoreLogging.with_logstate(stream.connection.server.logstate) do + if error_code != 0 + @error "server complete error" exception=(aws_error(error_code), Base.backtrace()) + end + if stream.connection.server.on_stream_complete !== nothing + try + Base.invokelatest(stream.connection.server.on_stream_complete, stream) + catch e + @error "on_stream_complete error" exception=(e, catch_backtrace()) + end + end + if stream.connection.server.access_log !== nothing + try + @info sprint(stream.connection.server.access_log, stream) _group=:access + catch e + @error "access log error" exception=(e, catch_backtrace()) + end + end + @lock stream.connection.streams_lock begin + delete!(stream.connection.streams, stream) + end + return Cint(0) + end +end + +const on_destroy_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_destroy_complete(server_ptr) + server = unsafe_pointer_to_objref(server_ptr) + notify(server.fut, :destroyed) + return +end + +function Base.close(server::Server) + state = @atomicswap server.state = :closed + if state == :running + aws_http_server_release(server.server) + @assert wait(server.fut) == :destroyed + notify(server.closed) + end + return +end + +Base.isopen(server::Server) = @atomic(server.state) != :closed diff --git a/src/status_messages.jl b/src/status_messages.jl deleted file mode 100644 index 628f9933e..000000000 --- a/src/status_messages.jl +++ /dev/null @@ -1,98 +0,0 @@ -""" - statustext(::Int) -> String - -`String` representation of a HTTP status code. - -## Examples -```julia -julia> statustext(200) -"OK" - -julia> statustext(404) -"Not Found" -``` -""" -statustext(status) = Base.get(STATUS_MESSAGES, status, "Unknown Code") - -const STATUS_MESSAGES = (()->begin - v = fill("Unknown Code", 530) - v[CONTINUE] = "Continue" - v[SWITCHING_PROTOCOLS] = "Switching Protocols" - v[PROCESSING] = "Processing" # RFC 2518 => obsoleted by RFC 4918 - v[EARLY_HINTS] = "Early Hints" - v[OK] = "OK" - v[CREATED] = "Created" - v[ACCEPTED] = "Accepted" - v[NON_AUTHORITATIVE_INFORMATION] = "Non-Authoritative Information" - v[NO_CONTENT] = "No Content" - v[RESET_CONTENT] = "Reset Content" - v[PARTIAL_CONTENT] = "Partial Content" - v[MULTI_STATUS] = "Multi-Status" # RFC4918 - v[ALREADY_REPORTED] = "Already Reported" # RFC5842 - v[IM_USED] = "IM Used" # RFC3229 - v[MULTIPLE_CHOICES] = "Multiple Choices" - v[MOVED_PERMANENTLY] = "Moved Permanently" - v[MOVED_TEMPORARILY] = "Moved Temporarily" - v[SEE_OTHER] = "See Other" - v[NOT_MODIFIED] = "Not Modified" - v[USE_PROXY] = "Use Proxy" - v[TEMPORARY_REDIRECT] = "Temporary Redirect" - v[PERMANENT_REDIRECT] = "Permanent Redirect" # RFC7238 - v[BAD_REQUEST] = "Bad Request" - v[UNAUTHORIZED] = "Unauthorized" - v[PAYMENT_REQUIRED] = "Payment Required" - v[FORBIDDEN] = "Forbidden" - v[NOT_FOUND] = "Not Found" - v[METHOD_NOT_ALLOWED] = "Method Not Allowed" - v[NOT_ACCEPTABLE] = "Not Acceptable" - v[PROXY_AUTHENTICATION_REQUIRED] = "Proxy Authentication Required" - v[REQUEST_TIME_OUT] = "Request Time-out" - v[CONFLICT] = "Conflict" - v[GONE] = "Gone" - v[LENGTH_REQUIRED] = "Length Required" - v[PRECONDITION_FAILED] = "Precondition Failed" - v[REQUEST_ENTITY_TOO_LARGE] = "Request Entity Too Large" - v[REQUEST_URI_TOO_LARGE] = "Request-URI Too Large" - v[UNSUPPORTED_MEDIA_TYPE] = "Unsupported Media Type" - v[REQUESTED_RANGE_NOT_SATISFIABLE] = "Requested Range Not Satisfiable" - v[EXPECTATION_FAILED] = "Expectation Failed" - v[IM_A_TEAPOT] = "I'm a teapot" # RFC 2324 - v[MISDIRECTED_REQUEST] = "Misdirected Request" # RFC 7540 - v[UNPROCESSABLE_ENTITY] = "Unprocessable Entity" # RFC 4918 - v[LOCKED] = "Locked" # RFC 4918 - v[FAILED_DEPENDENCY] = "Failed Dependency" # RFC 4918 - v[UNORDERED_COLLECTION] = "Unordered Collection" # RFC 4918 - v[UPGRADE_REQUIRED] = "Upgrade Required" # RFC 2817 - v[PRECONDITION_REQUIRED] = "Precondition Required" # RFC 6585 - v[TOO_MANY_REQUESTS] = "Too Many Requests" # RFC 6585 - v[REQUEST_HEADER_FIELDS_TOO_LARGE] = "Request Header Fields Too Large" # RFC 6585 - v[LOGIN_TIMEOUT] = "Login Timeout" - v[NGINX_ERROR_NO_RESPONSE] = "nginx error: No Response" - v[UNAVAILABLE_FOR_LEGAL_REASONS] = "Unavailable For Legal Reasons" # RFC7725 - v[NGINX_ERROR_SSL_CERTIFICATE_ERROR] = "nginx error: SSL Certificate Error" - v[NGINX_ERROR_SSL_CERTIFICATE_REQUIRED] = "nginx error: SSL Certificate Required" - v[NGINX_ERROR_HTTP_TO_HTTPS] = "nginx error: HTTP -> HTTPS" - v[NGINX_ERROR_OR_ANTIVIRUS_INTERCEPTED_REQUEST_OR_ARCGIS_ERROR] = "nginx error or Antivirus intercepted request or ArcGIS error" - v[INTERNAL_SERVER_ERROR] = "Internal Server Error" - v[NOT_IMPLEMENTED] = "Not Implemented" - v[BAD_GATEWAY] = "Bad Gateway" - v[SERVICE_UNAVAILABLE] = "Service Unavailable" - v[GATEWAY_TIME_OUT] = "Gateway Time-out" - v[HTTP_VERSION_NOT_SUPPORTED] = "HTTP Version Not Supported" - v[VARIANT_ALSO_NEGOTIATES] = "Variant Also Negotiates" # RFC 2295 - v[INSUFFICIENT_STORAGE] = "Insufficient Storage" # RFC 4918 - v[LOOP_DETECTED] = "Loop Detected" # RFC5842 - v[BANDWIDTH_LIMIT_EXCEEDED] = "Bandwidth Limit Exceeded" - v[NOT_EXTENDED] = "Not Extended" # RFC 2774 - v[NETWORK_AUTHENTICATION_REQUIRED] = "Network Authentication Required" # RFC 6585 - v[CLOUDFLARE_SERVER_ERROR_UNKNOWN] = "CloudFlare Server Error: Unknown" - v[CLOUDFLARE_SERVER_ERROR_CONNECTION_REFUSED] = "CloudFlare Server Error: Connection Refused" - v[CLOUDFLARE_SERVER_ERROR_CONNECTION_TIMEOUT] = "CloudFlare Server Error: Connection Timeout" - v[CLOUDFLARE_SERVER_ERROR_ORIGIN_SERVER_UNREACHABLE] = "CloudFlare Server Error: Origin Server Unreachable" - v[CLOUDFLARE_SERVER_ERROR_A_TIMEOUT] = "CloudFlare Server Error: A Timeout" - v[CLOUDFLARE_SERVER_ERROR_CONNECTION_FAILED] = "CloudFlare Server Error: Connection Failed" - v[CLOUDFLARE_SERVER_ERROR_INVALID_SSL_CERITIFICATE] = "CloudFlare Server Error: Invalid SSL Ceritificate" - v[CLOUDFLARE_SERVER_ERROR_RAILGUN_ERROR] = "CloudFlare Server Error: Railgun Error" - v[SITE_FROZEN] = "Site Frozen" - return v -end)() diff --git a/src/StatusCodes.jl b/src/statuses.jl similarity index 52% rename from src/StatusCodes.jl rename to src/statuses.jl index d686672f1..46c418dac 100644 --- a/src/StatusCodes.jl +++ b/src/statuses.jl @@ -85,6 +85,103 @@ const CLOUDFLARE_SERVER_ERROR_INVALID_SSL_CERITIFICATE = 526 const CLOUDFLARE_SERVER_ERROR_RAILGUN_ERROR = 527 const SITE_FROZEN = 530 -include("status_messages.jl") +""" + statustext(::Int) -> String + +`String` representation of a HTTP status code. + +## Examples +```julia +julia> statustext(200) +"OK" + +julia> statustext(404) +"Not Found" +``` +""" +statustext(status) = Base.get(STATUS_MESSAGES, status, "Unknown Code") + +const STATUS_MESSAGES = (()->begin + v = fill("Unknown Code", 530) + v[CONTINUE] = "Continue" + v[SWITCHING_PROTOCOLS] = "Switching Protocols" + v[PROCESSING] = "Processing" # RFC 2518 => obsoleted by RFC 4918 + v[EARLY_HINTS] = "Early Hints" + v[OK] = "OK" + v[CREATED] = "Created" + v[ACCEPTED] = "Accepted" + v[NON_AUTHORITATIVE_INFORMATION] = "Non-Authoritative Information" + v[NO_CONTENT] = "No Content" + v[RESET_CONTENT] = "Reset Content" + v[PARTIAL_CONTENT] = "Partial Content" + v[MULTI_STATUS] = "Multi-Status" # RFC4918 + v[ALREADY_REPORTED] = "Already Reported" # RFC5842 + v[IM_USED] = "IM Used" # RFC3229 + v[MULTIPLE_CHOICES] = "Multiple Choices" + v[MOVED_PERMANENTLY] = "Moved Permanently" + v[MOVED_TEMPORARILY] = "Moved Temporarily" + v[SEE_OTHER] = "See Other" + v[NOT_MODIFIED] = "Not Modified" + v[USE_PROXY] = "Use Proxy" + v[TEMPORARY_REDIRECT] = "Temporary Redirect" + v[PERMANENT_REDIRECT] = "Permanent Redirect" # RFC7238 + v[BAD_REQUEST] = "Bad Request" + v[UNAUTHORIZED] = "Unauthorized" + v[PAYMENT_REQUIRED] = "Payment Required" + v[FORBIDDEN] = "Forbidden" + v[NOT_FOUND] = "Not Found" + v[METHOD_NOT_ALLOWED] = "Method Not Allowed" + v[NOT_ACCEPTABLE] = "Not Acceptable" + v[PROXY_AUTHENTICATION_REQUIRED] = "Proxy Authentication Required" + v[REQUEST_TIME_OUT] = "Request Time-out" + v[CONFLICT] = "Conflict" + v[GONE] = "Gone" + v[LENGTH_REQUIRED] = "Length Required" + v[PRECONDITION_FAILED] = "Precondition Failed" + v[REQUEST_ENTITY_TOO_LARGE] = "Request Entity Too Large" + v[REQUEST_URI_TOO_LARGE] = "Request-URI Too Large" + v[UNSUPPORTED_MEDIA_TYPE] = "Unsupported Media Type" + v[REQUESTED_RANGE_NOT_SATISFIABLE] = "Requested Range Not Satisfiable" + v[EXPECTATION_FAILED] = "Expectation Failed" + v[IM_A_TEAPOT] = "I'm a teapot" # RFC 2324 + v[MISDIRECTED_REQUEST] = "Misdirected Request" # RFC 7540 + v[UNPROCESSABLE_ENTITY] = "Unprocessable Entity" # RFC 4918 + v[LOCKED] = "Locked" # RFC 4918 + v[FAILED_DEPENDENCY] = "Failed Dependency" # RFC 4918 + v[UNORDERED_COLLECTION] = "Unordered Collection" # RFC 4918 + v[UPGRADE_REQUIRED] = "Upgrade Required" # RFC 2817 + v[PRECONDITION_REQUIRED] = "Precondition Required" # RFC 6585 + v[TOO_MANY_REQUESTS] = "Too Many Requests" # RFC 6585 + v[REQUEST_HEADER_FIELDS_TOO_LARGE] = "Request Header Fields Too Large" # RFC 6585 + v[LOGIN_TIMEOUT] = "Login Timeout" + v[NGINX_ERROR_NO_RESPONSE] = "nginx error: No Response" + v[UNAVAILABLE_FOR_LEGAL_REASONS] = "Unavailable For Legal Reasons" # RFC7725 + v[NGINX_ERROR_SSL_CERTIFICATE_ERROR] = "nginx error: SSL Certificate Error" + v[NGINX_ERROR_SSL_CERTIFICATE_REQUIRED] = "nginx error: SSL Certificate Required" + v[NGINX_ERROR_HTTP_TO_HTTPS] = "nginx error: HTTP -> HTTPS" + v[NGINX_ERROR_OR_ANTIVIRUS_INTERCEPTED_REQUEST_OR_ARCGIS_ERROR] = "nginx error or Antivirus intercepted request or ArcGIS error" + v[INTERNAL_SERVER_ERROR] = "Internal Server Error" + v[NOT_IMPLEMENTED] = "Not Implemented" + v[BAD_GATEWAY] = "Bad Gateway" + v[SERVICE_UNAVAILABLE] = "Service Unavailable" + v[GATEWAY_TIME_OUT] = "Gateway Time-out" + v[HTTP_VERSION_NOT_SUPPORTED] = "HTTP Version Not Supported" + v[VARIANT_ALSO_NEGOTIATES] = "Variant Also Negotiates" # RFC 2295 + v[INSUFFICIENT_STORAGE] = "Insufficient Storage" # RFC 4918 + v[LOOP_DETECTED] = "Loop Detected" # RFC5842 + v[BANDWIDTH_LIMIT_EXCEEDED] = "Bandwidth Limit Exceeded" + v[NOT_EXTENDED] = "Not Extended" # RFC 2774 + v[NETWORK_AUTHENTICATION_REQUIRED] = "Network Authentication Required" # RFC 6585 + v[CLOUDFLARE_SERVER_ERROR_UNKNOWN] = "CloudFlare Server Error: Unknown" + v[CLOUDFLARE_SERVER_ERROR_CONNECTION_REFUSED] = "CloudFlare Server Error: Connection Refused" + v[CLOUDFLARE_SERVER_ERROR_CONNECTION_TIMEOUT] = "CloudFlare Server Error: Connection Timeout" + v[CLOUDFLARE_SERVER_ERROR_ORIGIN_SERVER_UNREACHABLE] = "CloudFlare Server Error: Origin Server Unreachable" + v[CLOUDFLARE_SERVER_ERROR_A_TIMEOUT] = "CloudFlare Server Error: A Timeout" + v[CLOUDFLARE_SERVER_ERROR_CONNECTION_FAILED] = "CloudFlare Server Error: Connection Failed" + v[CLOUDFLARE_SERVER_ERROR_INVALID_SSL_CERITIFICATE] = "CloudFlare Server Error: Invalid SSL Ceritificate" + v[CLOUDFLARE_SERVER_ERROR_RAILGUN_ERROR] = "CloudFlare Server Error: Railgun Error" + v[SITE_FROZEN] = "Site Frozen" + return v +end)() end # module StatusCodes diff --git a/src/utils.jl b/src/utils.jl new file mode 100644 index 000000000..374cd9c7a --- /dev/null +++ b/src/utils.jl @@ -0,0 +1,219 @@ +""" + escapehtml(i::String) + +Returns a string with special HTML characters escaped: &, <, >, ", ' +""" +function escapehtml(i::AbstractString) + # Refer to http://stackoverflow.com/a/7382028/3822752 for spec. links + o = replace(i, "&" =>"&") + o = replace(o, "\""=>""") + o = replace(o, "'" =>"'") + o = replace(o, "<" =>"<") + o = replace(o, ">" =>">") + return o +end + +""" + iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) + +Convert from ISO8859_1 to UTF8. +""" +function iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) + io = IOBuffer() + for b in bytes + if b < 0x80 + write(io, b) + else + write(io, 0xc0 | (b >> 6)) + write(io, 0x80 | (b & 0x3f)) + end + end + return String(take!(io)) +end + +""" + tocameldash(s::String) + +Ensure the first character and characters that follow a '-' are uppercase. +""" +function tocameldash(s::String) + toUpper = UInt8('A') - UInt8('a') + v = Vector{UInt8}(codeunits(s)) + upper = true + for i = 1:length(v) + @inbounds b = v[i] + if upper + islower(b) && (v[i] = b + toUpper) + else + isupper(b) && (v[i] = lower(b)) + end + upper = b == UInt8('-') + end + return String(v) +end + +tocameldash(s::AbstractString) = tocameldash(String(s)) + +@inline islower(b::UInt8) = UInt8('a') <= b <= UInt8('z') +@inline isupper(b::UInt8) = UInt8('A') <= b <= UInt8('Z') +@inline lower(c::UInt8) = c | 0x20 + +function parseuri(url, query, allocator) + uri_ref = Ref{aws_uri}() + if url isa AbstractString + url_str = String(url) * (query === nothing ? "" : ("?" * URIs.escapeuri(query))) + elseif url isa URI + url_str = string(url) + else + throw(ArgumentError("url must be an AbstractString or URI")) + end + GC.@preserve url_str begin + url_ref = Ref(aws_byte_cursor(sizeof(url_str), pointer(url_str))) + aws_uri_init_parse(uri_ref, allocator, url_ref) + end + return uri_ref[] +end + +isbytes(x) = x isa AbstractVector{UInt8} || x isa AbstractString + +str(bc::aws_byte_cursor) = bc.ptr == C_NULL || bc.len == 0 ? "" : unsafe_string(bc.ptr, bc.len) + +function print_uri(io, uri::aws_uri) + print(io, "scheme: ", str(uri.scheme), "\n") + print(io, "userinfo: ", str(uri.userinfo), "\n") + print(io, "host_name: ", str(uri.host_name), "\n") + print(io, "port: ", Int(uri.port), "\n") + print(io, "path: ", str(uri.path), "\n") + print(io, "query: ", str(uri.query_string), "\n") + return +end + +scheme(uri::aws_uri) = str(uri.scheme) +userinfo(uri::aws_uri) = str(uri.userinfo) +host(uri::aws_uri) = str(uri.host_name) +port(uri::aws_uri) = uri.port +path(uri::aws_uri) = str(uri.path) +query(uri::aws_uri) = str(uri.query_string) + +function resource(uri::aws_uri) + ref = Ref(uri) + GC.@preserve ref begin + bc = aws_uri_path_and_query(ref) + path = str(unsafe_load(bc)) + return isempty(path) ? "/" : path + end +end + +const URI_SCHEME_HTTPS = "https" +const URI_SCHEME_WSS = "wss" +ishttps(sch) = aws_byte_cursor_eq_c_str_ignore_case(sch, URI_SCHEME_HTTPS) +iswss(sch) = aws_byte_cursor_eq_c_str_ignore_case(sch, URI_SCHEME_WSS) +function getport(uri::aws_uri) + sch = Ref(uri.scheme) + GC.@preserve sch begin + return UInt32(uri.port != 0 ? uri.port : (ishttps(sch) || iswss(sch)) ? 443 : 80) + end +end + +function makeuri(u::aws_uri) + return URIs.URI( + scheme=str(u.scheme), + userinfo=isempty(str(u.userinfo)) ? URIs.absent : str(u.userinfo), + host=str(u.host_name), + port=u.port == 0 ? URIs.absent : u.port, + path=isempty(str(u.path)) ? URIs.absent : str(u.path), + query=isempty(str(u.query_string)) ? URIs.absent : str(u.query_string), + ) +end + +struct AWSError <: Exception + msg::String +end + +aws_error() = AWSError(unsafe_string(aws_error_debug_str(aws_last_error()))) +aws_error(error_code) = AWSError(unsafe_string(aws_error_str(error_code))) +aws_throw_error() = throw(aws_error()) + +struct FieldRef{T, S} + x::T + field::Symbol +end + +FieldRef(x::T, field::Symbol) where {T} = FieldRef{T, fieldtype(T, field)}(x, field) + +function Base.unsafe_convert(P::Union{Type{Ptr{S}},Type{Ptr{Cvoid}}}, x::FieldRef{T, S}) where {T, S} + @assert isconcretetype(T) && ismutabletype(T) "only fields of mutable types are supported with FieldRef" + return P(pointer_from_objref(x.x) + fieldoffset(T, Base.fieldindex(T, x.field))) +end + +Base.pointer(x::FieldRef{S, T}) where {S, T} = Base.unsafe_convert(Ptr{T}, x) + +mutable struct Future{T} + const notify::Threads.Condition + @atomic set::Int8 # if 0, result is undefined, 1 means result is T, 2 means result is an exception + result::Union{Exception, T} # undefined initially + Future{T}() where {T} = new{T}(Threads.Condition(), 0) +end + +function Base.wait(f::Future{T}) where {T} + set = @atomic f.set + set == 1 && return f.result::T + set == 2 && throw(f.result::Exception) + lock(f.notify) # acquire barrier + try + set = f.set + set == 1 && return f.result::T + set == 2 && throw(f.result::Exception) + wait(f.notify) + finally + unlock(f.notify) # release barrier + end + if f.set == 1 + return f.result::T + else + @assert isdefined(f, :result) + throw(f.result::Exception) + end +end + +function Base.notify(f::Future{T}, x::Union{Exception, T}) where {T} + lock(f.notify) # acquire barrier + try + if f.set == Int8(0) + if x isa Exception + set = Int8(2) + f.result = x + else + set = Int8(1) + f.result = x + end + @atomic :release f.set = set + notify(f.notify) + end + finally + unlock(f.notify) + end + nothing +end + +struct BufferOnResponseBody{T <: AbstractVector{UInt8}} + buffer::T + pos::Ptr{Int} +end + +function (f::BufferOnResponseBody)(resp, buf) + len = length(buf) + pos = unsafe_load(f.pos) + copyto!(f.buffer, pos, buf, 1, len) + unsafe_store!(f.pos, pos + len) + return len +end + +struct IOOnResponseBody{T <: IO} + io::T +end + +function (f::IOOnResponseBody)(resp, buf) + write(f.io, buf) + return length(buf) +end diff --git a/src/websockets.jl b/src/websockets.jl new file mode 100644 index 000000000..82e58ea25 --- /dev/null +++ b/src/websockets.jl @@ -0,0 +1,446 @@ +module WebSockets + +using Base64, Random, LibAwsHTTPFork, LibAwsCommon, LibAwsIO + +import ..FieldRef, ..iswss, ..getport, ..makeuri, ..aws_throw_error, ..resource, ..Headers, ..Header, ..str, ..aws_error, ..aws_throw_error, ..Future, ..parseuri, ..with_redirect, ..with_request, ..getclient, ..ClientSettings, ..scheme, ..host, ..getport, ..userinfo, ..Client, ..Request, ..Response, ..setinputstream!, ..getresponse, ..CookieJar, ..COOKIEJAR, ..addheaders, ..Stream, ..HTTP, ..getheader + +export WebSocket, send, receive, ping, pong + +@enum OpCode::UInt8 CONTINUATION=0x00 TEXT=0x01 BINARY=0x02 CLOSE=0x08 PING=0x09 PONG=0x0A + +mutable struct WebSocket + id::String + host::String + path::String + not::Future{Nothing} + readchannel::Channel{Union{String, Vector{UInt8}}} + writebuffer::Vector{UInt8} + writepos::Int + writeclosed::Bool + closelock::ReentrantLock + handshake_request::Request + options::aws_websocket_client_connection_options + websocket_pointer::Ptr{aws_websocket} + handshake_response::Response + websocket_send_frame_options::aws_websocket_send_frame_options + + WebSocket(host::AbstractString, path::AbstractString) = new(string(rand(UInt32); base=58), String(host), String(path), Future{Nothing}(), Channel{Union{String, Vector{UInt8}}}(Inf), UInt8[], 0, false, ReentrantLock()) +end + +getresponse(ws::WebSocket) = ws.handshake_response + +const on_connection_setup = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_connection_setup(connection_setup_data::Ptr{aws_websocket_on_connection_setup_data}, ws_ptr) + ws = unsafe_pointer_to_objref(ws_ptr) + data = unsafe_load(connection_setup_data) + try + if data.error_code != 0 + notify(ws.not, CapturedException(aws_error(data.error_code), Base.backtrace())) + else + ws.websocket_pointer = data.websocket + ws.handshake_response.status = unsafe_load(data.handshake_response_status) + addheaders(ws.handshake_response.headers, data.handshake_response_header_array, data.num_handshake_response_headers) + if data.handshake_response_body != C_NULL + handshake_response_body = unsafe_load(data.handshake_response_body) + response_body = str(handshake_response_body) + else + response_body = nothing + end + setinputstream!(ws.handshake_response, response_body) + notify(ws.not, nothing) + end + catch e + notify(ws.not, CapturedException(e, Base.backtrace())) + end + return +end + +const on_connection_shutdown = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_connection_shutdown(websocket::Ptr{aws_websocket}, error_code::Cint, ws_ptr) + ws = unsafe_pointer_to_objref(ws_ptr) + if error_code != 0 + @error "$(ws.id): connection shutdown error" exception=(aws_error(error_code), Base.backtrace()) + end + close(ws) + return +end + +const on_incoming_frame_begin = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_incoming_frame_begin(websocket::Ptr{aws_websocket}, frame::Ptr{aws_websocket_incoming_frame}, ws_ptr) + # ws = unsafe_pointer_to_objref(ws_ptr) + # fr = unsafe_load(frame) + return true +end + +const on_incoming_frame_payload = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_incoming_frame_payload(websocket::Ptr{aws_websocket}, frame::Ptr{aws_websocket_incoming_frame}, data::aws_byte_cursor, ws_ptr) + ws = unsafe_pointer_to_objref(ws_ptr) + fr = unsafe_load(frame) + try + if fr.opcode == UInt8(TEXT) + put!(ws.readchannel, unsafe_string(data.ptr, data.len)) + else + rec = Vector{UInt8}(undef, data.len) + Base.unsafe_copyto!(pointer(rec), data.ptr, data.len) + put!(ws.readchannel, rec) + end + catch e + @error "$(ws.id): incoming frame payload error" exception=(e, catch_backtrace()) + end + return true +end + +const on_incoming_frame_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_incoming_frame_complete(websocket::Ptr{aws_websocket}, frame::Ptr{aws_websocket_incoming_frame}, error_code::Cint, ws_ptr) + ws = unsafe_pointer_to_objref(ws_ptr) + fr = unsafe_load(frame) + if error_code != 0 + @error "$(ws.id): incoming frame complete error" exception=(aws_error(error_code), Base.backtrace()) + end + return true +end + +function open(f::Function, url; + headers=[], + allocator::Ptr{aws_allocator}=default_aws_allocator(), + username=nothing, + password=nothing, + bearer=nothing, + query=nothing, + client::Union{Nothing, Client}=nothing, + # redirect options + redirect=true, + redirect_limit=3, + redirect_method=nothing, + forwardheaders=true, + # cookie options + cookies=true, + cookiejar::CookieJar=COOKIEJAR, + modifier=nothing, + verbose=0, + # client keywords + kw... + ) + key = base64encode(rand(Random.RandomDevice(), UInt8, 16)) + uri = parseuri(url, query, allocator) + # add required websocket headers + append!(headers, [ + "upgrade" => "websocket", + "connection" => "upgrade", + "sec-websocket-key" => key, + "sec-websocket-version" => "13" + ]) + ws = with_redirect(allocator, "GET", uri, headers, nothing, redirect, redirect_limit, redirect_method, forwardheaders) do method, uri, headers, body + reqclient = @something(client, getclient(ClientSettings(scheme(uri), host(uri), getport(uri); allocator=allocator, ssl_alpn_list="http/1.1", kw...)))::Client + path = resource(uri) + with_request(reqclient, method, path, headers, body, nothing, false, (username !== nothing && password !== nothing) ? "$username:$password" : userinfo(uri), bearer, modifier, false, cookies, cookiejar, verbose) do req + host = str(uri.host_name) + ws = WebSocket(host, path) + ws.handshake_request = req + ws.handshake_response = Response(0, nothing, nothing, false, allocator) + ws.options = aws_websocket_client_connection_options( + allocator, + reqclient.settings.bootstrap, + pointer(FieldRef(reqclient, :socket_options)), + reqclient.tls_options === nothing ? C_NULL : pointer(FieldRef(reqclient, :tls_options)), + reqclient.proxy_options === nothing ? C_NULL : pointer(FieldRef(reqclient, :proxy_options)), + uri.host_name, + uri.port, + ws.handshake_request.ptr, + 0, # initial_window_size + Ptr{Cvoid}(pointer_from_objref(ws)), # user_data + on_connection_setup[], + on_connection_shutdown[], + on_incoming_frame_begin[], + on_incoming_frame_payload[], + on_incoming_frame_complete[], + false, # manual_window_management + C_NULL, # requested_event_loop + C_NULL, # host_resolution_config + ) + if aws_websocket_client_connect(FieldRef(ws, :options)) != 0 + aws_throw_error() + end + # wait until connected + wait(ws.not) + return ws + end + end + verbose > 0 && @info "$(ws.id): WebSocket opened" + try + f(ws) + catch e + # if !isok(e) + # suppress_close_error || @error "$(ws.id): error" (e, catch_backtrace()) + # end + # if !isclosed(ws) + # if e isa WebSocketError && e.message isa CloseFrameBody + # close(ws, e.message) + # else + # close(ws, CloseFrameBody(1008, "Unexpected client websocket error")) + # end + # end + # if !isok(e) + rethrow() + # end + finally + # if !isclosed(ws) + close(ws) + # end + end +end + +function Base.close(ws::WebSocket) + @lock ws.closelock begin + if ws.websocket_pointer != C_NULL + aws_websocket_close(ws.websocket_pointer, false) + ws.websocket_pointer = C_NULL + ws.writeclosed = true + end + end + return +end + +""" + WebSockets.isclosed(ws) -> Bool + +Check whether a `WebSocket` has sent and received CLOSE frames +""" +isclosed(ws::WebSocket) = !isopen(ws.readchannel) && ws.writeclosed + +isbinary(x) = x isa AbstractVector{UInt8} +istext(x) = x isa AbstractString +opcode(x) = isbinary(x) ? BINARY : TEXT + +function payload(ws, x) + pload = isbinary(x) ? x : codeunits(string(x)) + len = length(pload) + resize!(ws.writebuffer, len) + copyto!(ws.writebuffer, pload) + ws.writepos = 1 + return ws.writebuffer +end + +const stream_outgoing_payload = Ref{Ptr{Cvoid}}(C_NULL) + +function c_stream_outgoing_payload(websocket::Ptr{aws_websocket}, out_buf::Ptr{aws_byte_buf}, ws_ptr::Ptr{Cvoid}) + ws = unsafe_pointer_to_objref(ws_ptr) + out = unsafe_load(out_buf) + try + space_available = out.capacity - out.len + amount_to_send = min(space_available, sizeof(ws.writebuffer) - ws.writepos + 1) + cursor = aws_byte_cursor(amount_to_send, pointer(ws.writebuffer, ws.writepos)) + @assert aws_byte_buf_write_from_whole_cursor(out_buf, cursor) + ws.writepos += amount_to_send + catch e + @error "$(ws.id): error" (e, catch_backtrace()) + return false + end + return true +end + +const on_complete = Ref{Ptr{Cvoid}}(C_NULL) + +function c_on_complete(websocket::Ptr{aws_websocket}, error_code::Cint, ws_ptr::Ptr{Cvoid}) + ws = unsafe_pointer_to_objref(ws_ptr) + if error_code != 0 + notify(ws.not, CapturedException(aws_error(error_code), Base.backtrace())) + end + notify(ws.not, nothing) + return +end + +function writeframe(ws::WebSocket, fin::Bool, opcode::OpCode, payload) + n = sizeof(payload) + ws.websocket_send_frame_options = aws_websocket_send_frame_options( + n % UInt64, + Ptr{Cvoid}(pointer_from_objref(ws)), # user_data + stream_outgoing_payload[], + on_complete[], + UInt8(opcode), + fin + ) + opts = pointer(FieldRef(ws, :websocket_send_frame_options)) + if aws_websocket_send_frame(ws.websocket_pointer, opts) != 0 + aws_throw_error() + end + # wait until frame sent + wait(ws.not) + return n +end + +""" + send(ws::WebSocket, msg) + +Send a message on a websocket connection. If `msg` is an `AbstractString`, +a TEXT websocket message will be sent; if `msg` is an `AbstractVector{UInt8}`, +a BINARY websocket message will be sent. Otherwise, `msg` should be an iterable +of either `AbstractString` or `AbstractVector{UInt8}`, and a fragmented message +will be sent, one frame for each iterated element. + +Control frames can be sent by calling `ping(ws[, data])`, `pong(ws[, data])`, +or `close(ws[, body::WebSockets.CloseFrameBody])`. Calling `close` will initiate +the close sequence and close the underlying connection. +""" +function send(ws::WebSocket, x) + @assert !ws.writeclosed "WebSocket is closed" + if !isbinary(x) && !istext(x) + # if x is not single binary or text, then assume it's an iterable of binary or text + # and we'll send fragmented message + first = true + n = 0 + state = iterate(x) + if state === nothing + # x was not binary or text, but is an empty iterable, send single empty frame + x = "" + @goto write_single_frame + end + @debug "$(ws.id): Writing fragmented message" + item, st = state + # we prefetch next state so we know if we're on the last item or not + # so we can appropriately set the FIN bit for the last fragmented frame + nextstate = iterate(x, st) + while true + n += writeframe(ws, nextstate === nothing, first ? opcode(item) : CONTINUATION, payload(ws, item)) + first = false + nextstate === nothing && break + item, st = nextstate + nextstate = iterate(x, st) + end + else + # single binary or text frame for message +@label write_single_frame + return writeframe(ws, true, opcode(x), payload(ws, x)) + end +end + +# control frames +""" + ping(ws, data=[]) + +Send a PING control frame on a websocket connection. `data` is an optional +body to send with the message. PONG messages are automatically responded +to when a PING message is received by a websocket connection. +""" +function ping(ws::WebSocket, data=UInt8[]) + @assert !ws.writeclosed "WebSocket is closed" + return writeframe(ws.io, true, PING, payload(ws, data)) +end + +""" + pong(ws, data=[]) + +Send a PONG control frame on a websocket connection. `data` is an optional +body to send with the message. Note that PING messages are automatically +responded to internally by the websocket connection with a corresponding +PONG message, but in certain cases, a unidirectional PONG message can be +used as a one-way heartbeat. +""" +function pong(ws::WebSocket, data=UInt8[]) + @assert !ws.writeclosed "WebSocket is closed" + return writeframe(ws.io, true, PONG, payload(ws, data)) +end + +""" + receive(ws::WebSocket) -> Union{String, Vector{UInt8}} + +Receive a message from a websocket connection. Returns a `String` if +the message was TEXT, or a `Vector{UInt8}` if the message was BINARY. +If control frames (ping or pong) are received, they are handled +automatically and a non-control message is waited for. If a CLOSE +message is received, it is responded to and a `WebSocketError` is thrown +with the `WebSockets.CloseFrameBody` as the error value. This error can +be checked with `WebSockets.isok(err)` to see if the closing was "normal" +or if an actual error occurred. For fragmented messages, the incoming +frames will continue to be read until the final fragment is received. +The bodies of each fragment are concatenated into the final message +returned by `receive`. Note that `WebSocket` objects can be iterated, +where each iteration yields a message until the connection is closed. +""" +function receive(ws::WebSocket) + @assert isopen(ws.readchannel) "WebSocket is closed" + return take!(ws.readchannel) +end + +""" + iterate(ws) + +Continuously call `receive(ws)` on a `WebSocket` connection, with +each iteration yielding a message until the connection is closed. +E.g. +```julia +for msg in ws + # do something with msg +end +``` +""" +function Base.iterate(ws::WebSocket, st=nothing) + isclosed(ws) && return nothing + try + return receive(ws), nothing + catch e + isok(e) && return nothing + rethrow(e) + end +end + +# given a WebSocket request, return the 101 response +function websocket_upgrade_handler(req::Request) + key = getheader(req.headers, "sec-websocket-key") + resp_ptr = aws_http_message_new_websocket_handshake_response(req.allocator, aws_byte_cursor_from_c_str(key)) + resp_ptr == C_NULL && aws_throw_error() + resp = Response() + resp.allocator = req.allocator + resp.ptr = resp_ptr + resp.request = req + return resp +end + +function websocket_upgrade_function(f) + #TODO: return WebSocketUpgradeArgs + # then schedule a task to do the actual upgrade + function websocket_upgrade(stream::Stream) + #TODO: get host/path from stream? + ws = WebSocket("", "") + stream.websocket_options = aws_websocket_server_upgrade_options( + 0, + Ptr{Cvoid}(pointer_from_objref(ws)), + on_incoming_frame_begin[], + on_incoming_frame_payload[], + on_incoming_frame_complete[], + false # manual_window_management + ) + ws_ptr = aws_websocket_upgrade(stream.allocator, stream.ptr, FieldRef(stream, :websocket_options)) + ws_ptr == C_NULL && aws_throw_error() + ws.websocket_pointer = ws_ptr + errormonitor(Threads.@spawn begin + try + f(ws) + finally + aws_websocket_release(ws_ptr) + end + end) + return + end +end + +serve!(f, host="127.0.0.1", port=8080; kw...) = + HTTP.serve!(websocket_upgrade_handler, host, port; on_stream_complete=websocket_upgrade_function(f), kw...) + +function __init__() + on_connection_setup[] = @cfunction(c_on_connection_setup, Cvoid, (Ptr{aws_websocket_on_connection_setup_data}, Ptr{Cvoid})) + on_connection_shutdown[] = @cfunction(c_on_connection_shutdown, Cvoid, (Ptr{aws_websocket}, Cint, Ptr{Cvoid})) + on_incoming_frame_begin[] = @cfunction(c_on_incoming_frame_begin, Bool, (Ptr{aws_websocket}, Ptr{aws_websocket_incoming_frame}, Ptr{Cvoid})) + on_incoming_frame_payload[] = @cfunction(c_on_incoming_frame_payload, Bool, (Ptr{aws_websocket}, Ptr{aws_websocket_incoming_frame}, aws_byte_cursor, Ptr{Cvoid})) + on_incoming_frame_complete[] = @cfunction(c_on_incoming_frame_complete, Bool, (Ptr{aws_websocket}, Ptr{aws_websocket_incoming_frame}, Cint, Ptr{Cvoid})) + stream_outgoing_payload[] = @cfunction(c_stream_outgoing_payload, Bool, (Ptr{aws_websocket}, Ptr{aws_byte_buf}, Ptr{Cvoid})) + on_complete[] = @cfunction(c_on_complete, Cvoid, (Ptr{aws_websocket}, Cint, Ptr{Cvoid})) + return +end + +end # module \ No newline at end of file diff --git a/test/ascii.jl b/test/ascii.jl deleted file mode 100644 index d4e6107fb..000000000 --- a/test/ascii.jl +++ /dev/null @@ -1,38 +0,0 @@ -using HTTP.Strings - -@testset "ascii.jl" begin - lc = HTTP.Strings.ascii_lc - lceq = HTTP.Strings.ascii_lc_isequal - - @testset "UInt8" begin - for c in UInt8(1):UInt8(127) - @test lc(c) == UInt8(lowercase(Char(c))) - @test lceq(c, c) - - @test !lceq(c, UInt8(c+1)) - @test !lceq(c, UInt8(0)) - @test !lceq(c, UInt8(128)) - end - end - - @testset "Strings" begin - @test lceq("", "") - - @test lceq("123!", "123!") - @test lceq("Foo", "Foo") - @test lceq("Foo", "foo") - @test lceq("foo", "Foo") - @test lceq("foo", "FOO") - - @test !lceq("", "FOO") - @test !lceq("FOO", "") - @test !lceq("Foo", "Fo") - @test !lceq("Foo", "Foox") - @test !lceq("123", "123!") - end - - @testset "Emojis" begin - @test lceq("NotAscii: 😬", "NotAscii: 😬") - @test lceq("notascii: 😬", "NotAscii: 😬") - end -end diff --git a/test/async.jl b/test/async.jl deleted file mode 100644 index c29109eef..000000000 --- a/test/async.jl +++ /dev/null @@ -1,133 +0,0 @@ -module test_async - -import ..httpbin -using Test, HTTP, JSON - -@time @testset "ASync" begin - configs = [ - Pair{Symbol, Any}[:verbose => 0], - Pair{Symbol, Any}[:verbose => 0, :reuse_limit => 200], - Pair{Symbol, Any}[:verbose => 0, :reuse_limit => 50] - ] - - dump_async_exception(e, st) = @error "async exception: " exception=(e, st) - - @testset "HTTP.request - Headers - $config - https" for config in configs - result = [] - - @sync begin - for i = 1:100 - @async try - response = HTTP.request("GET", "https://$httpbin/headers", ["i" => i]; config...) - response = JSON.parse(String(response.body)) - push!(result, response["headers"]["I"] => [string(i)]) - catch e - dump_async_exception(e, stacktrace(catch_backtrace())) - rethrow(e) - end - end - end - - for(a, b) in result - @test a == b - end - - HTTP.Connections.closeall() - end - - @testset "HTTP.request - Body - $config - https" for config in configs - result = [] - - @sync begin - for i=1:100 - @async try - response = HTTP.request("GET", "https://$httpbin/stream/$i"; config...) - response = String(response.body) - response = split(strip(response), "\n") - push!(result, length(response) => i) - catch e - dump_async_exception(e, stacktrace(catch_backtrace())) - rethrow(e) - end - end - end - - for (a, b) in result - @test a == b - end - - HTTP.Connections.closeall() - end - - @testset "HTTP.open - $config - https" for config in configs - result = [] - - @sync begin - for i=1:100 - @async try - open_response = nothing - url = "https://$httpbin/stream/$i" - - response = HTTP.open("GET", url; config...) do http - open_response = String(read(http)) - end - - open_response = split(strip(open_response), "\n") - push!(result, length(open_response) => i) - catch e - dump_async_exception(e, stacktrace(catch_backtrace())) - rethrow(e) - end - end - end - - for (a, b) in result - @test a == b - end - - HTTP.Connections.closeall() - end - - @testset "HTTP.request - Response Stream - $config - https" for config in configs - result = [] - - @sync begin - for i=1:100 - @async try - stream_response = nothing - - # Note: url for $i will give back $i responses split on "\n" - url = "https://$httpbin/stream/$i" - - try - stream = Base.BufferStream() - response = HTTP.request("GET", url; response_stream=stream, config...) - close(stream) - - stream_response = String(read(stream)) - stream_response = split(strip(stream_response), "\n") - if length(stream_response) != i - @show join(stream_response, "\n") - end - push!(result, length(stream_response) => i) - catch e - if !HTTP.RetryRequest.isrecoverable(e) - rethrow(e) - end - end - catch e - dump_async_exception(e, stacktrace(catch_backtrace())) - rethrow(e) - end - end - end - - for (a, b) in result - @test a == b - end - - HTTP.Connections.closeall() - end -end - -end # module diff --git a/test/benchmark.jl b/test/benchmark.jl deleted file mode 100644 index 05978c9b1..000000000 --- a/test/benchmark.jl +++ /dev/null @@ -1,292 +0,0 @@ -using Unitful - -include("http_parser_benchmark.jl") - -using HTTP -using HTTP.IOExtras -using HTTP.Connections -using HTTP.Streams -using HTTP.Messages - -responses = [ -"github" => """ -HTTP/1.1 200 OK\r -Date: Mon, 15 Jan 2018 00:57:33 GMT\r -Content-Type: text/html; charset=utf-8\r -Transfer-Encoding: chunked\r -Server: GitHub.com\r -Status: 200 OK\r -Cache-Control: no-cache\r -Vary: X-PJAX\r -X-UA-Compatible: IE=Edge,chrome=1\r -Set-Cookie: logged_in=no; domain=.github.com; path=/; expires=Fri, 15 Jan 2038 00:57:33 -0000; secure; HttpOnly\r -Set-Cookie: _gh_sess=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX; path=/; secure; HttpOnly\r -X-Request-Id: fa15328ce7eb90bfe0274b6c97ed4c59\r -X-Runtime: 0.323267\r -Expect-CT: max-age=2592000, report-uri="https://api.github.com/_private/browser/errors"\r -Content-Security-Policy: default-src 'none'; base-uri 'self'; block-all-mixed-content; child-src render.githubusercontent.com; connect-src 'self' uploads.github.com status.github.com collector.githubapp.com api.github.com www.google-analytics.com github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com wss://live.github.com; font-src assets-cdn.github.com; form-action 'self' github.com gist.github.com; frame-ancestors 'none'; img-src 'self' data: assets-cdn.github.com identicons.github.com collector.githubapp.com github-cloud.s3.amazonaws.com *.githubusercontent.com; media-src 'none'; script-src assets-cdn.github.com; style-src 'unsafe-inline' assets-cdn.github.com\r -Strict-Transport-Security: max-age=31536000; includeSubdomains; preload\r -X-Content-Type-Options: nosniff\r -X-Frame-Options: deny\r -X-XSS-Protection: 1; mode=block\r -X-Runtime-rack: 0.331298\r -Vary: Accept-Encoding\r -X-GitHub-Request-Id: EB38:2597E:1126615:197B74F:5A5BFC7B\r -\r -""" -, -"wikipedia" => """ -HTTP/1.1 200 OK\r -Date: Mon, 15 Jan 2018 01:05:05 GMT\r -Content-Type: text/html\r -Content-Length: 74711\r -Connection: keep-alive\r -Server: mw1327.eqiad.wmnet\r -Cache-Control: s-maxage=86400, must-revalidate, max-age=3600\r -ETag: W/"123d7-56241bf703864"\r -Last-Modified: Mon, 08 Jan 2018 11:03:27 GMT\r -Backend-Timing: D=257 t=1515662958810464\r -Vary: Accept-Encoding\r -X-Varnish: 581763780 554073291, 1859700 2056157, 154619657 149630877, 484912801 88058768\r -Via: 1.1 varnish-v4, 1.1 varnish-v4, 1.1 varnish-v4, 1.1 varnish-v4\r -Age: 56146\r -X-Cache: cp1054 hit/10, cp2016 hit/1, cp4029 hit/2, cp4031 hit/345299\r -X-Cache-Status: hit-front\r -Strict-Transport-Security: max-age=106384710; includeSubDomains; preload\r -Set-Cookie: WMF-Last-Access=15-Jan-2018;Path=/;HttpOnly;secure;Expires=Fri, 16 Feb 2018 00:00:00 GMT\r -Set-Cookie: WMF-Last-Access-Global=15-Jan-2018;Path=/;Domain=.wikipedia.org;HttpOnly;secure;Expires=Fri, 16 Feb 2018 00:00:00 GMT\r -X-Analytics: https=1;nocookies=1\r -X-Client-IP: 14.201.217.194\r -Set-Cookie: GeoIP=AU:VIC:Frankston:-38.14:145.12:v4; Path=/; secure; Domain=.wikipedia.org\r -Accept-Ranges: bytes\r -\r -""" -, -"netflix" => """ -HTTP/1.1 200 OK\r -Cache-Control: no-cache, no-store, must-revalidate\r -Content-Type: text/html; charset=utf-8\r -Date: Mon, 15 Jan 2018 01:06:12 GMT\r -Expires: 0\r -Pragma: no-cache\r -req_id: 5ee36317-8bbc-448d-88e8-d19a4fb20331\r -Server: shakti-prod i-0a9188462a5e21164\r -Set-Cookie: SecureNetflixId=v%3D2%26mac%3DAQEAEQABABTLdDaklPBpPR3Gwmj9zmjDfOlSLqOftDk.%26dt%3D1515978372532; Domain=.netflix.com; Path=/; Expires=Fri, 01 Jan 2038 00:00:00 GMT; HttpOnly; Secure\r -Set-Cookie: NetflixId=v%3D2%26ct%3DBQAOAAEBEMuUr7yphqTQpaSgxbmFKYeBoN-TtAcb2E5hDcV73R-Qu8VVqjODTPk7URvqrvzZ1bgpPpiS1DlPOn0ePBInOys9GExY-DUnxkMb99H9ogWoz_Ibi3zWJm7tBsizk3NRHdl4dLZJzrrUGy2HC1JJf3k52FBRx-gFf7UyhseihLocdFS579_IQA1xGugB0pAEadI14eeQkjiZadQ0DwHiZJqjxK8QsUX3_MqMFL9O0q2r5oVMO3sq9VLuxSZz3c_XCFPKTvYSCR_sZPCSPp4kQyIkPUzpQLDTC_dEFapTWGKRNnJBtcH6MJXICcB19SXi83lV26gxFRVyd7MVwZUtGvX_jRGkWgZeRXQuS1YHx0GefQF64y7uEwEgGL49fyKLafF9cHH0tGewQm0iPU3NJktKegMzOx1o4j0-HnWRQzDwgiqQqCLy5sqDCGJyxjagvGdQfc0TiHIVvAeuztEP9XPT1IMvydE8F9C7qzVgpwIaVjkrEzSEG4sazqy7xj5y_dTfOeoHPsQEO2IazjiM1SSMPlDK9SEpiQl18B0sR-tNbYGZgRw6KvSJNBSd9KcsfXsf%26bt%3Ddbl%26ch%3DAQEAEAABABTN0reTusSIN2zATnuy-b6hYllVzhgAaq0.%26mac%3DAQEAEAABABRKcEvZN9gx2qYCWw6GyrTFp5r9efkDmFg.; Domain=.netflix.com; Path=/; Expires=Tue, 15 Jan 2019 06:54:58 GMT; HttpOnly\r -Set-Cookie: clSharedContext=f6f2fc4b-427d-4ac1-b21a-45994d6aa3be; Domain=.netflix.com; Path=/\r -Set-Cookie: memclid=73a77399-8bb5-4a12-bdd8-e07f194d6e3d; Max-Age=31536000; Expires=Tue, 15 Jan 2019 01:06:12 GMT; Path=/; Domain=.netflix.com\r -Set-Cookie: nfvdid=BQFmAAEBEFBHN%2BNNVgwjmGFtH09kBA9AsMDpgxuORCUvQF8ZamZmnBCC1dU4zuJausLoGBA%2FqhGfeoY%2FN0%2F6KN9VlAaHaTU%2BLjtMOr8WsF5dSfyQVjJbNg%3D%3D; Max-Age=26265600; Expires=Thu, 15 Nov 2018 01:06:12 GMT; Path=/; Domain=.netflix.com\r -Strict-Transport-Security: max-age=31536000\r -Vary: Accept-Encoding\r -Via: 1.1 i-0842dec9615b5afe9 (us-west-2)\r -X-Content-Type-Options: nosniff\r -X-Frame-Options: DENY\r -X-Netflix-From-Zuul: true\r -X-Netflix.nfstatus: 1_1\r -X-Netflix.proxy.execution-time: 240\r -X-Originating-URL: https://www.netflix.com/au/\r -X-Xss-Protection: 1; mode=block; report=https://ichnaea.netflix.com/log/freeform/xssreport\r -transfer-encoding: chunked\r -Connection: keep-alive\r -\r -""" -, -"twitter" => """ -HTTP/1.1 200 OK\r -cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0\r -content-length: 324899\r -content-type: text/html;charset=utf-8\r -date: Mon, 15 Jan 2018 01:08:31 GMT\r -expires: Tue, 31 Mar 1981 05:00:00 GMT\r -last-modified: Mon, 15 Jan 2018 01:08:31 GMT\r -pragma: no-cache\r -server: tsa_l\r -set-cookie: fm=0; Expires=Mon, 15 Jan 2018 01:08:22 UTC; Path=/; Domain=.twitter.com; Secure; HTTPOnly, _twitter_sess=BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCDxUXPdgAToMY3NyZl9p%250AZCIlOGNhY2U5NjdjOWUyYzkwZWIxMGRiZTgyYjI5NDRkNjY6B2lkIiU4NDdl%250AOTIwNGZhN2JhZjg2NTE5ZjY4ZWJhZTEyOGNjNQ%253D%253D--09d63c3cb46541e33a903d83241480ceab65b33e; Path=/; Domain=.twitter.com; Secure; HTTPOnly, personalization_id="v1_7Q083SuV/G6hK5GjmjV/JQ=="; Expires=Wed, 15 Jan 2020 01:08:31 UTC; Path=/; Domain=.twitter.com, guest_id=v1%3A151597851141844671; Expires=Wed, 15 Jan 2020 01:08:31 UTC; Path=/; Domain=.twitter.com, ct0=d81bcd3211730706c80f4b6cb7b793e8; Expires=Mon, 15 Jan 2018 07:08:31 UTC; Path=/; Domain=.twitter.com; Secure\r -status: 200 OK\r -strict-transport-security: max-age=631138519\r -x-connection-hash: 9cc6e603545c8ee9e1895a9c2acddb89\r -x-content-type-options: nosniff\r -x-frame-options: SAMEORIGIN\r -x-response-time: 531\r -x-transaction: 0019e08a0043f702\r -x-twitter-response-tags: BouncerCompliant\r -x-ua-compatible: IE=edge,chrome=1\r -x-xss-protection: 1; mode=block; report=https://twitter.com/i/xss_report\r -\r -""" -, -"akamai" => """ -HTTP/1.1 403 Forbidden\r -Server: AkamaiGHost\r -Mime-Version: 1.0\r -Content-Type: text/html\r -Content-Length: 270\r -Expires: Mon, 15 Jan 2018 01:11:29 GMT\r -Date: Mon, 15 Jan 2018 01:11:29 GMT\r -Connection: close\r -Vary: User-Agent\r -\r -""" -, -"nytimes" => """ -HTTP/1.1 200 OK\r -Server: Apache\r -Cache-Control: no-cache\r -X-ESI: 1\r -X-App-Response-Time: 0.62\r -Content-Type: text/html; charset=utf-8\r -X-PageType: homepage\r -X-Age: 69\r -X-Origin-Time: 2018-01-14 20:47:16 EDT\r -Content-Length: 213038\r -Accept-Ranges: bytes\r -Date: Mon, 15 Jan 2018 01:13:41 GMT\r -Age: 1585\r -X-Frame-Options: DENY\r -Set-Cookie: vi_www_hp=z00; path=/; domain=.nytimes.com; expires=Wed, 01 Jan 2020 00:00:00 GMT\r -Set-Cookie: vistory=z00; path=/; domain=.nytimes.com; expires=Wed, 01 Jan 2020 00:00:00 GMT\r -Set-Cookie: nyt-a=4HpgpyMVIOWB1wkVLcX98Q; Expires=Tue, 15 Jan 2019 01:13:41 GMT; Path=/; Domain=.nytimes.com\r -Connection: close\r -X-API-Version: F-5-5\r -Content-Security-Policy: default-src data: 'unsafe-inline' 'unsafe-eval' https:; script-src data: 'unsafe-inline' 'unsafe-eval' https: blob:; style-src data: 'unsafe-inline' https:; img-src data: https: blob:; font-src data: https:; connect-src https: wss:; media-src https: blob:; object-src https:; child-src https: data: blob:; form-action https:; block-all-mixed-content;\r -X-Served-By: cache-mel6524-MEL\r -X-Cache: HIT\r -X-Cache-Hits: 2\r -X-Timer: S1515978821.355774,VS0,VE0\r -Vary: Accept-Encoding, Fastly-SSL\r -\r -""" -] - -pipeline_limit = 100 - -randbody(l) = Vector{UInt8}(rand('A':'Z', l)) - -const tunit = u"μs" -const tmul = Int(1u"s"/1tunit) -delta_t(a, b) = round((b - a) * tmul, digits=0)tunit - -fields = [:setup, :head, :body, :close, :joyent_head] - -function go(count::Int) - - times = Dict() - for (name, bytes) in responses - times[name] = Dict() - for f in fields - times[name][f] = [] - end - end - - t_init_start = time() - io = Base.BufferStream() - c = Connection("", "", pipeline_limit, 0, true, io) - - t_init_done = time() - for rep in 1:count - gc() - for (name, bytes) in shuffle(responses) - - - write(io, bytes) - - r = Request().response - readheaders(IOBuffer(bytes), r) - l = bodylength(r) - if l == unknown_length - for i = 1:100 - l = 10000 - chunk = randbody(l) - write(io, hex(l), "\r\n", chunk, "\r\n") - end - write(io, "0\r\n\r\n") - else - write(io, randbody(l)) - end - - t_start = time() - t = Transaction(c) - r = Request() - s = Stream(r.response, t) - #startread(s) - #function IOExtras.startread(http::Stream) - - http = s - startread(http.stream) - - # Ensure that the header and body are buffered in the Connection - # object. Otherwise, the time spent in readheaders below is - # dominated by readavailable() copying huge body data from the - # Base.BufferStream. We want to measure the parsing performance. - unread!(http.stream, readavailable(http.stream)) - - t_setup = time() - HTTP.Streams.readheaders(http.stream, http.message) - t_headers_done = time() - HTTP.Streams.handle_continue(http) - - http.readchunked = HTTP.Messages.ischunked(http.message) - http.ntoread = HTTP.Messages.bodylength(http.message) - - # return http.message - #end - - - while !eof(s) - readavailable(s) - end - t_body_done = time() - closeread(s) - t_done = time() - push!(times[name][:setup], delta_t(t_start, t_setup)) - push!(times[name][:head], delta_t(t_setup, t_headers_done)) - push!(times[name][:body], delta_t(t_headers_done, t_body_done)) - push!(times[name][:close], delta_t(t_body_done, t_done)) - - t_start = time() - r = HttpParserTest.parse(bytes) - t_done = time() - push!(times[name][:joyent_head], delta_t(t_start, t_done)) - end - end - - if count <= 10 - return - end - - w = 12 - print("| ") - print(lpad("Response",w)) - for f in fields - print(" | ") - print(lpad(f,w)) - end - print(" | ") - print(lpad("jl faster",w)) - println(" |") - - print("| ") - print(repeat("-",w)) - for f in [fields..., ""] - print(":| ") - print(repeat("-",w)) - end - println(":|") - - for (name, bytes) in responses - print("| ") - print(lpad("$name: ",w)) - for f in fields - print(" | ") - print(lpad(mean(times[name][f]), w)) - end - faster = mean(times[name][:joyent_head]) / mean(times[name][:head]) - print(" | ") - print(lpad("x $(round(faster, digits=1))", w)) - println(" |") - end -end - -for r in [10, 100] - go(r) -end diff --git a/test/chunking.jl b/test/chunking.jl deleted file mode 100644 index 7fd1c2117..000000000 --- a/test/chunking.jl +++ /dev/null @@ -1,65 +0,0 @@ -module TestChunking - -using Test, Sockets -using HTTP, HTTP.IOExtras -using BufferedStreams - -# For more information see: https://github.com/JuliaWeb/HTTP.jl/pull/288 -@testset "Chunking" begin - sz = 90 - port = 8095 - hex(n) = string(n, base=16) - encoded_data = "$(hex(sz + 9))\r\n" * "data: 1$(repeat("x", sz))\n\n" * "\r\n" * - "$(hex(sz + 9))\r\n" * "data: 2$(repeat("x", sz))\n\n" * "\r\n" * - "$(hex(sz + 9))\r\n" * "data: 3$(repeat("x", sz))\n\n" * "\r\n" - decoded_data = "data: 1$(repeat("x", sz))\n\n" * - "data: 2$(repeat("x", sz))\n\n" * - "data: 3$(repeat("x", sz))\n\n" - split1 = 106 - split2 = 300 - server = HTTP.listen!(port) do http - startwrite(http) - tcp = http.stream.io - - write(tcp, encoded_data[1:split1]) - flush(tcp) - sleep(1) - - write(tcp, encoded_data[split1+1:split2]) - flush(tcp) - sleep(1) - - write(tcp, encoded_data[split2+1:end]) - flush(tcp) - end - - try - r = HTTP.get("http://127.0.0.1:$port") - @test String(r.body) == decoded_data - - for wrap in (identity, BufferedInputStream) - r = "" - - # Ignore byte-by-byte read warning - CL = Base.CoreLogging - CL.with_logger(CL.SimpleLogger(stderr, CL.Error)) do - HTTP.open("GET", "http://127.0.0.1:$port") do io - io = wrap(io) - x = split(decoded_data, "\n") - - for i in 1:6 - l = readline(io) - @test l == x[i] - r *= l * "\n" - end - end - end - - @test r == decoded_data - end - finally - close(server) - end -end - -end # module diff --git a/test/client.jl b/test/client.jl index ad76ad9f6..d230212e8 100644 --- a/test/client.jl +++ b/test/client.jl @@ -1,117 +1,29 @@ -module TestClient - -using HTTP, HTTP.Exceptions, MbedTLS, OpenSSL -using HTTP: IOExtras -include(joinpath(dirname(pathof(HTTP)), "../test/resources/TestRequest.jl")) -import ..isok, ..httpbin -using .TestRequest -using .TestRequest2 -using Sockets -using JSON -using Test -using URIs -using InteractiveUtils: @which -using ConcurrentUtilities - -# ConcurrentUtilities changed a fieldname from max to limit in 2.3.0 -const max_or_limit = :max in fieldnames(ConcurrentUtilities.Pool) ? (:max) : (:limit) - -# test we can adjust default_connection_limit -for x in (10, 12) - HTTP.set_default_connection_limit!(x) - @test getfield(HTTP.Connections.TCP_POOL[], max_or_limit) == x - @test getfield(HTTP.Connections.MBEDTLS_POOL[], max_or_limit) == x - @test getfield(HTTP.Connections.OPENSSL_POOL[], max_or_limit) == x -end - -@testset "sslconfig without explicit socket_type_tls #1104" begin - # this was supported before 8f35185 - @test isok(HTTP.get("https://$httpbin/ip", sslconfig=MbedTLS.SSLConfig(false))) - # The OpenSSL package doesn't have enough docs, but this is a valid way to initialise an SSLContext. - @test isok(HTTP.get("https://$httpbin/ip", sslconfig=OpenSSL.SSLContext(OpenSSL.TLSClientMethod()))) - # Incompatible socket_type_tls and sslconfig should throw an error. - @test_throws ArgumentError HTTP.get("https://$httpbin/ip", sslconfig=MbedTLS.SSLConfig(false), socket_type_tls=OpenSSL.SSLStream) -end - -@testset "issue 1172" begin - # Connections through a proxy need to choose an IOType twice rather than - # just once. - # https://github.com/JuliaWeb/HTTP.jl/issues/1172 - - # This proxy accepts two requests, ignoring the content of the request and - # returning 200 each time. - proxy = listen(IPv4(0), 8082) - try - @async begin - sock = accept(proxy) - while isopen(sock) - line = readline(sock) - @show 1, line - isempty(line) && break - end - write(sock, "HTTP/1.1 200\r\n\r\n") - # Test that we receive something that looks like a client hello - # (indicating that we tried to upgrade the connection to TLS) - line = readline(sock) - @test startswith(line, "\x16") - end - - @test_throws HTTP.RequestError HTTP.head("https://$httpbin.com"; proxy="http://localhost:8082", readtimeout=1, retry=false) - finally - close(proxy) - HTTP.Connections.closeall() - end -end - -@testset "@client macro" begin - @eval module MyClient - using HTTP - HTTP.@client () () - end - # Test the `@client` sets the location info to the definition site, i.e. this file. - meth = @which MyClient.get() - file = String(meth.file) - @test file == @__FILE__ -end - -@testset "Custom HTTP Stack" begin - @testset "Low-level Request" begin - wasincluded = Ref(false) - result = TestRequest.get("https://$httpbin/ip"; httptestlayer=wasincluded) - @test isok(result) - @test wasincluded[] - end - @testset "Low-level Request" begin - TestRequest2.get("https://$httpbin/ip") - # tests included in the layers themselves - end -end - -@testset "Client.jl" for tls in [MbedTLS.SSLContext, OpenSSL.SSLStream] - @testset "GET, HEAD, POST, PUT, DELETE, PATCH" begin - @test isok(HTTP.get("https://$httpbin/ip", socket_type_tls=tls)) - @test isok(HTTP.head("https://$httpbin/ip", socket_type_tls=tls)) - @test HTTP.post("https://$httpbin/patch"; status_exception=false, socket_type_tls=tls).status == 405 - @test isok(HTTP.post("https://$httpbin/post", socket_type_tls=tls)) - @test isok(HTTP.put("https://$httpbin/put", socket_type_tls=tls)) - @test isok(HTTP.delete("https://$httpbin/delete", socket_type_tls=tls)) - @test isok(HTTP.patch("https://$httpbin/patch", socket_type_tls=tls)) +@testset "Client.jl" begin + @testset "GET, HEAD, POST, PUT, DELETE, PATCH: $scheme" for scheme in ["http", "https"] + @test isok(HTTP.get("$scheme://$httpbin/ip")) + @test isok(HTTP.head("$scheme://$httpbin/ip")) + @test HTTP.post("$scheme://$httpbin/patch"; status_exception=false).status == 405 + @test isok(HTTP.post("$scheme://$httpbin/post"; redirect_method=:same)) + @test isok(HTTP.put("$scheme://$httpbin/put"; redirect_method=:same)) + @test isok(HTTP.delete("$scheme://$httpbin/delete"; redirect_method=:same)) + @test isok(HTTP.patch("$scheme://$httpbin/patch"; redirect_method=:same)) end @testset "decompress" begin - r = HTTP.get("https://$httpbin/gzip", socket_type_tls=tls) + r = HTTP.get("https://$httpbin/gzip") @test isok(r) @test isascii(String(r.body)) - r = HTTP.get("https://$httpbin/gzip"; decompress=false, socket_type_tls=tls) + r = HTTP.get("https://$httpbin/gzip"; decompress=false) @test isok(r) @test !isascii(String(r.body)) - r = HTTP.get("https://$httpbin/gzip"; decompress=false, socket_type_tls=tls) - @test isascii(String(HTTP.decode(r, "gzip"))) + r = HTTP.get("https://$httpbin/gzip"; decompress=true) + @test isok(r) + @test isascii(String(r.body)) end @testset "ASync Client Requests" begin - @test isok(fetch(@async HTTP.get("https://$httpbin/ip", socket_type_tls=tls))) - @test isok(HTTP.get("https://$httpbin/encoding/utf8", socket_type_tls=tls)) + @test isok(fetch(Threads.@spawn HTTP.get("https://$httpbin/ip"))) + @test isok(HTTP.get("https://$httpbin/encoding/utf8")) end @testset "Query to URI" begin @@ -123,202 +35,150 @@ end @testset "Cookie Requests" begin empty!(HTTP.COOKIEJAR) url = "https://$httpbin/cookies" - r = HTTP.get(url, cookies=true, socket_type_tls=tls) + r = HTTP.get(url, cookies=true) @test String(r.body) == "{}" - cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, URI(url)) + cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, "https", httpbin, "/cookies") @test isempty(cookies) url = "https://$httpbin/cookies/set?hey=sailor&foo=bar" - r = HTTP.get(url, cookies=true, socket_type_tls=tls) + r = HTTP.get(url, cookies=true) @test isok(r) - cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, URI(url)) + @test String(r.body) == "{\"foo\":\"bar\",\"hey\":\"sailor\"}" + cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, "https", httpbin, "/cookies") @test length(cookies) == 2 url = "https://$httpbin/cookies/delete?hey" - r = HTTP.get(url, socket_type_tls=tls) - cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, URI(url)) + r = HTTP.get(url) + @test isok(r) + @test String(r.body) == "{\"foo\":\"bar\"}" + cookies = HTTP.Cookies.getcookies!(HTTP.COOKIEJAR, "https", httpbin, "/cookies") @test length(cookies) == 1 end @testset "Client Streaming Test" begin - r = HTTP.post("https://$httpbin/post"; body="hey", socket_type_tls=tls) - @test isok(r) - - # stream, but body is too small to actually stream - r = HTTP.post("https://$httpbin/post"; body="hey", stream=true, socket_type_tls=tls) + r = HTTP.post("https://$httpbin/anything"; body="hey") @test isok(r) + @test contains(String(r.body), "\"data\":\"hey\"") - r = HTTP.get("https://$httpbin/stream/100", socket_type_tls=tls) + r = HTTP.get("https://$httpbin/stream/100") @test isok(r) - - bytes = r.body - a = [JSON.parse(l) for l in split(chomp(String(bytes)), "\n")] - totallen = length(bytes) # number of bytes to expect + x = map(String, split(String(r.body), '\n'; keepempty=false)) + @test length(x) == 100 io = IOBuffer() - r = HTTP.get("https://$httpbin/stream/100"; response_stream=io, socket_type_tls=tls) - seekstart(io) + r = HTTP.get("https://$httpbin/stream/100"; response_body=io) @test isok(r) - - b = [JSON.parse(l) for l in eachline(io)] - @test all(zip(a, b)) do (x, y) - x["args"] == y["args"] && - x["id"] == y["id"] && - x["url"] == y["url"] && - x["origin"] == y["origin"] && - x["headers"]["Content-Length"] == y["headers"]["Content-Length"] && - x["headers"]["Host"] == y["headers"]["Host"] && - x["headers"]["User-Agent"] == y["headers"]["User-Agent"] - end + x2 = map(String, split(String(take!(io)), '\n'; keepempty=false)) + @test x == x2 # pass pre-allocated buffer body = zeros(UInt8, 100) - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=body, socket_type_tls=tls) - @test body === r.body - - # wrapping pre-allocated buffer in IOBuffer will write to buffer directly - io = IOBuffer(body; write=true) - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - @test Base.mightalias(body, r.body.data) + r = HTTP.get("https://$httpbin/bytes/100"; response_body=body) + @test length(body) == 100 + @test any(x -> x != 0, body) # if provided buffer is too small, we won't grow it for user body = zeros(UInt8, 10) - @test_throws HTTP.RequestError HTTP.get("https://$httpbin/bytes/100"; response_stream=body, socket_type_tls=tls, retry=false) + @test_throws BoundsError HTTP.get("https://$httpbin/bytes/100"; response_body=body) # also won't shrink it if buffer provided is larger than response body body = zeros(UInt8, 10) - r = HTTP.get("https://$httpbin/bytes/5"; response_stream=body, socket_type_tls=tls) - @test body === r.body + r = HTTP.get("https://$httpbin/bytes/5"; response_body=body) @test length(body) == 10 - @test HTTP.header(r, "Content-Length") == "5" + @test all(x -> x == 0, body[6:end]) # but if you wrap it in a writable IOBuffer, we will grow it io = IOBuffer(body; write=true) - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - # might be a new Array, resized larger - body = take!(io) - @test length(body) == 100 + r = HTTP.get("https://$httpbin/bytes/100"; response_body=io) + @test length(take!(io)) == 100 # and you can reuse it seekstart(io) - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - # `take!` should have given it a new Array - @test !Base.mightalias(body, r.body.data) - body = take!(io) - @test length(body) == 100 + r = HTTP.get("https://$httpbin/bytes/100"; response_body=io) + @test length(take!(io)) == 100 # we respect ptr and size body = zeros(UInt8, 100) io = IOBuffer(body; write=true, append=true) # size=100, ptr=1 - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - body = take!(io) - @test length(body) == 200 + r = HTTP.get("https://$httpbin/bytes/100"; response_body=io) + @test length(take!(io)) == 200 body = zeros(UInt8, 100) io = IOBuffer(body, write=true, append=false) write(io, body) # size=100, ptr=101 - r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - body = take!(io) - @test length(body) == 200 + r = HTTP.get("https://$httpbin/bytes/100"; response_body=io) + @test length(take!(io)) == 200 + # status error response body handling + r = HTTP.post("https://$httpbin/status/404"; status_exception=false) end @testset "Client Body Posting - Vector{UTF8}, String, IOStream, IOBuffer, BufferStream, Dict, NamedTuple" begin - @test isok(HTTP.post("https://$httpbin/post"; body="hey", socket_type_tls=tls)) - @test isok(HTTP.post("https://$httpbin/post"; body=UInt8['h','e','y'], socket_type_tls=tls)) + @test isok(HTTP.post("https://$httpbin/post"; body="hey")) + @test isok(HTTP.post("https://$httpbin/post"; body=UInt8['h','e','y'])) io = IOBuffer("hey"); seekstart(io) - @test isok(HTTP.post("https://$httpbin/post"; body=io, socket_type_tls=tls)) - tmp = tempname() - open(f->write(f, "hey"), tmp, "w") - io = open(tmp) - @test isok(HTTP.post("https://$httpbin/post"; body=io, enablechunked=false, socket_type_tls=tls)) - close(io); rm(tmp) + @test isok(HTTP.post("https://$httpbin/post"; body=io)) + mktemp() do path, io + write(io, "hey"); seekstart(io) + @test isok(HTTP.post("https://$httpbin/post"; body=io)) + end f = Base.BufferStream() write(f, "hey") close(f) - @test isok(HTTP.post("https://$httpbin/post"; body=f, enablechunked=false, socket_type_tls=tls)) - resp = HTTP.post("https://$httpbin/post"; body=Dict("name" => "value"), socket_type_tls=tls) + @test isok(HTTP.post("https://$httpbin/post"; body=f)) + resp = HTTP.post("https://$httpbin/post"; body=Dict("name" => "value")) @test isok(resp) - x = JSON.parse(IOBuffer(resp.body)) - @test x["form"] == Dict("name" => ["value"]) - resp = HTTP.post("https://$httpbin/post"; body=(name="value with spaces",), socket_type_tls=tls) + # x = JSONBase.materialize(resp.body) + # @test x["form"] == Dict("name" => ["value"]) + resp = HTTP.post("https://$httpbin/post"; body=(name="value with spaces",)) @test isok(resp) - x = JSON.parse(IOBuffer(resp.body)) - @test x["form"] == Dict("name" => ["value with spaces"]) - end - - @testset "Chunksize" begin - # https://github.com/JuliaWeb/HTTP.jl/issues/60 - # Currently $httpbin responds with 411 status and “Length Required” - # message to any POST/PUT requests that are sent using chunked encoding - # See https://github.com/kennethreitz/httpbin/issues/340#issuecomment-330176449 - @test isok(HTTP.post("https://$httpbin/post"; body="hey", socket_type_tls=tls, #=chunksize=2=#)) - @test isok(HTTP.post("https://$httpbin/post"; body=UInt8['h','e','y'], socket_type_tls=tls, #=chunksize=2=#)) - io = IOBuffer("hey"); seekstart(io) - @test isok(HTTP.post("https://$httpbin/post"; body=io, socket_type_tls=tls, #=chunksize=2=#)) - tmp = tempname() - open(f->write(f, "hey"), tmp, "w") - io = open(tmp) - @test isok(HTTP.post("https://$httpbin/post"; body=io, socket_type_tls=tls, #=chunksize=2=#)) - close(io); rm(tmp) - f = Base.BufferStream() - write(f, "hey") - close(f) - @test isok(HTTP.post("https://$httpbin/post"; body=f, socket_type_tls=tls, #=chunksize=2=#)) + # x = JSONBase.materialize(resp.body) + # @test x["form"] == Dict("name" => ["value with spaces"]) end @testset "ASync Client Request Body" begin f = Base.BufferStream() write(f, "hey") - t = @async HTTP.post("https://$httpbin/post"; body=f, enablechunked=false, socket_type_tls=tls) + t = Threads.@spawn HTTP.post("https://$httpbin/post"; body=f) #fetch(f) # fetch for the async call to write it's first data write(f, " there ") # as we write to f, it triggers another chunk to be sent in our async request write(f, "sailor") close(f) # setting eof on f causes the async request to send a final chunk and return the response - @test isok(fetch(t)) + r = fetch(t) + @test isok(r) + @test contains(String(r.body), "\"data\":\"hey there sailor\"") end @testset "Client Redirect Following - $read_method" for read_method in ["GET", "HEAD"] - @test isok(HTTP.request(read_method, "https://$httpbin/redirect/1", socket_type_tls=tls)) - @test HTTP.request(read_method, "https://$httpbin/redirect/1", redirect=false, socket_type_tls=tls).status == 302 - @test HTTP.request(read_method, "https://$httpbin/redirect/6", socket_type_tls=tls).status == 302 #over max number of redirects - @test isok(HTTP.request(read_method, "https://$httpbin/relative-redirect/1", socket_type_tls=tls)) - @test isok(HTTP.request(read_method, "https://$httpbin/absolute-redirect/1", socket_type_tls=tls)) - @test isok(HTTP.request(read_method, "https://$httpbin/redirect-to?url=http%3A%2F%2Fgoogle.com", socket_type_tls=tls)) + @test isok(HTTP.request(read_method, "https://$httpbin/redirect/1")) + @test HTTP.request(read_method, "https://$httpbin/redirect/1", redirect=false, status_exception=false).status == 302 + @test HTTP.request(read_method, "https://$httpbin/redirect/6", status_exception=false).status == 302 #over max number of redirects + @test isok(HTTP.request(read_method, "https://$httpbin/relative-redirect/1")) + @test isok(HTTP.request(read_method, "https://$httpbin/absolute-redirect/1")) + @test isok(HTTP.request(read_method, "https://$httpbin/redirect-to?url=http%3A%2F%2Fgoogle.com")) end @testset "Client Basic Auth" begin - @test isok(HTTP.get("https://user:pwd@$httpbin/basic-auth/user/pwd", socket_type_tls=tls)) - @test isok(HTTP.get("https://user:pwd@$httpbin/hidden-basic-auth/user/pwd", socket_type_tls=tls)) - @test isok(HTTP.get("https://test:%40test@$httpbin/basic-auth/test/%40test", socket_type_tls=tls)) + @test isok(HTTP.get("https://user:pwd@$httpbin/basic-auth/user/pwd")) + @test isok(HTTP.get("https://user:pwd@$httpbin/hidden-basic-auth/user/pwd")) + @test isok(HTTP.get("https://test:%40test@$httpbin/basic-auth/test/%40test")) + @test isok(HTTP.get("https://$httpbin/basic-auth/user/pwd"; username="user", password="pwd")) end @testset "Misc" begin - @test isok(HTTP.post("https://$httpbin/post"; body="√", socket_type_tls=tls)) - r = HTTP.request("GET", "https://$httpbin/ip", socket_type_tls=tls) + @test isok(HTTP.post("https://$httpbin/post"; body="√")) + r = HTTP.request("GET", "https://$httpbin/ip") @test isok(r) uri = HTTP.URI("https://$httpbin/ip") - r = HTTP.request("GET", uri, socket_type_tls=tls) + r = HTTP.request("GET", uri) @test isok(r) r = HTTP.get(uri) @test isok(r) - r = HTTP.request("GET", "https://$httpbin/ip", socket_type_tls=tls) - @test isok(r) - - uri = HTTP.URI("https://$httpbin/ip") - r = HTTP.request("GET", uri, socket_type_tls=tls) - @test isok(r) - - r = HTTP.get("https://$httpbin/image/png", socket_type_tls=tls) - @test isok(r) - # ensure we can use AbstractString for requests - r = HTTP.get(SubString("https://$httpbin/ip",1), socket_type_tls=tls) - - # canonicalizeheaders - @test isok(HTTP.get("https://$httpbin/ip"; canonicalizeheaders=false, socket_type_tls=tls)) + r = HTTP.get(SubString("https://$httpbin/ip",1)) + @test isok(r) # Ensure HEAD requests stay the same through redirects by default r = HTTP.head("https://$httpbin/redirect/1") @@ -329,494 +189,41 @@ end @test r.request.method == "GET" @test length(r.body) > 0 end -end - -@testset "Incomplete response with known content length" begin - server = nothing - try - server = HTTP.listen!("0.0.0.0", 8080) do http - HTTP.setstatus(http, 200) - HTTP.setheader(http, "Content-Length" => "64") # Promise 64 bytes... - HTTP.startwrite(http) - HTTP.write(http, rand(UInt8, 63)) # ...but only send 63 bytes. - # Close the stream so that eof(stream) is true and the client isn't - # waiting forever for the last byte. - HTTP.close(http.stream) - end - - err = try - HTTP.get("http://localhost:8080"; retry=false) - catch err - err - end - @test err isa HTTP.RequestError - @test err.error isa EOFError - finally - # Shutdown - @try Base.IOError close(server) - HTTP.Connections.closeall() - end -end - -@testset "HTTP.open accepts method::Symbol" begin - @test isok(HTTP.open(x -> x, :GET, "http://$httpbin/ip")) -end - -@testset "readtimeout" begin - @test_throws HTTP.TimeoutError begin - HTTP.get("http://$httpbin/delay/5"; readtimeout=1, retry=false) - end - HTTP.get("http://$httpbin/delay/1"; readtimeout=2, retry=false) -end - -@testset "connect_timeout does not include the time needed to acquire a connection from the pool" begin - connection_limit = getfield(HTTP.Connections.TCP_POOL[], max_or_limit) - try - dummy_conn = HTTP.Connection(Sockets.TCPSocket()) - HTTP.set_default_connection_limit!(1) - @assert getfield(HTTP.Connections.TCP_POOL[], max_or_limit) == 1 - # drain the pool - acquire(()->dummy_conn, HTTP.Connections.TCP_POOL[], HTTP.Connections.connectionkey(dummy_conn)) - # Put it back in 10 seconds - Timer(t->HTTP.Connections.releaseconnection(dummy_conn, false), 10; interval=0) - # If we count the time it takes to acquire the connection from the pool, we'll get a timeout error. - HTTP.get("https://$httpbin/get"; connect_timeout=5, retry=false, socket_type_tls=Sockets.TCPSocket) - @test true # if we get here, we didn't timeout - finally - HTTP.set_default_connection_limit!(connection_limit) - end -end - -@testset "Retry all resolved IP addresses" begin - # See issue https://github.com/JuliaWeb/HTTP.jl/issues/672 - # Bit tricky to test, but can at least be tested if localhost - # resolves to both IPv4 and IPv6 by listening to the respective - # interface - alladdrs = getalladdrinfo("localhost") - if ip"127.0.0.1" in alladdrs && ip"::1" in alladdrs - for interface in (IPv4(0), IPv6(0)) - server = nothing - try - server = HTTP.listen!(string(interface), 8080) do http - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - HTTP.write(http, "hello, world") - end - resp = HTTP.get("http://localhost:8080") - @test isok(resp) - @test String(resp.body) == "hello, world" - finally - @try Base.IOError close(server) - HTTP.Connections.closeall() - end - end - end -end - -@testset "Sockets.get(sock|peer)name(::HTTP.Stream)" begin - server = nothing - try - server = HTTP.listen!("0.0.0.0", 8080) do http - sock = Sockets.getsockname(http) - peer = Sockets.getpeername(http) - str = sprint() do io - print(io, sock[1], ":", sock[2], " - ", peer[1], ":", peer[2]) - end - HTTP.setstatus(http, 200) - HTTP.setheader(http, "Content-Length" => string(sizeof(str))) - HTTP.startwrite(http) - HTTP.write(http, str) - end - - # Tests for Stream{TCPSocket} - HTTP.open("GET", "http://localhost:8080") do http - # Test server peer/sock - reg = r"^127\.0\.0\.1:8080 - 127\.0\.0\.1:(\d+)$" - m = match(reg, read(http, String)) - @test m !== nothing - server_peerport = parse(Int, m[1]) - # Test client peer/sock - sock = Sockets.getsockname(http) - @test sock[1] == ip"127.0.0.1" - @test sock[2] == server_peerport - peer = Sockets.getpeername(http) - @test peer[1] == ip"127.0.0.1" - @test peer[2] == 8080 - end - finally - @try Base.IOError close(server) - HTTP.Connections.closeall() - end - - # Tests for Stream{SSLContext} - HTTP.open("GET", "https://julialang.org") do http - sock = Sockets.getsockname(http) - if VERSION >= v"1.2.0" - @test sock[1] in Sockets.getipaddrs() - end - peer = Sockets.getpeername(http) - @test peer[1] in Sockets.getalladdrinfo("julialang.org") - @test peer[2] == 443 - end -end - -@testset "input verification of bad URLs" begin - # HTTP.jl#527, HTTP.jl#545 - url = "julialang.org" - @test_throws ArgumentError("missing or unsupported scheme in URL (expected http(s) or ws(s)): $(url)") HTTP.get(url) - url = "ptth://julialang.org" - @test_throws ArgumentError("missing or unsupported scheme in URL (expected http(s) or ws(s)): $(url)") HTTP.get(url) - url = "http:julialang.org" - @test_throws ArgumentError("missing host in URL: $(url)") HTTP.get(url) -end - -@testset "Implicit request headers" begin - server = nothing - try - server = HTTP.listen!("0.0.0.0", 8080) do http - data = Dict{String,String}(http.message.headers) - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - HTTP.write(http, sprint(JSON.print, data)) - end - old_user_agent = HTTP.HeadersRequest.USER_AGENT[] - default_user_agent = "HTTP.jl/$VERSION" - # Default values - HTTP.setuseragent!(default_user_agent) - d = JSON.parse(IOBuffer(HTTP.get("http://localhost:8080").body)) - @test d["Host"] == "localhost:8080" - @test d["Accept"] == "*/*" - @test d["User-Agent"] == default_user_agent - # Overwriting behavior - headers = ["Host" => "http.jl", "Accept" => "application/json"] - HTTP.setuseragent!("HTTP.jl test") - d = JSON.parse(IOBuffer(HTTP.get("http://localhost:8080", headers).body)) - @test d["Host"] == "http.jl" - @test d["Accept"] == "application/json" - @test d["User-Agent"] == "HTTP.jl test" - # No User-Agent - HTTP.setuseragent!(nothing) - d = JSON.parse(IOBuffer(HTTP.get("http://localhost:8080").body)) - @test !haskey(d, "User-Agent") - - HTTP.setuseragent!(old_user_agent) - finally - @try Base.IOError close(server) - HTTP.Connections.closeall() - end -end - -import NetworkOptions, MbedTLS -@testset "NetworkOptions for host verification" begin - # Set up server with self-signed cert - server = nothing - try - cert, key = joinpath.(dirname(pathof(HTTP)), "../test", "resources", ("cert.pem", "key.pem")) - sslconfig = MbedTLS.SSLConfig(cert, key) - server = HTTP.listen!("0.0.0.0", 8443; sslconfig=sslconfig) do http - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - HTTP.write(http, "hello, world") - end - url = "https://localhost:8443" - env = ["JULIA_NO_VERIFY_HOSTS" => nothing, "JULIA_SSL_NO_VERIFY_HOSTS" => nothing, "JULIA_ALWAYS_VERIFY_HOSTS" => nothing] - withenv(env...) do - @test NetworkOptions.verify_host(url) - @test NetworkOptions.verify_host(url, "SSL") - @test_throws HTTP.ConnectError HTTP.get(url; retries=1) - @test_throws HTTP.ConnectError HTTP.get(url; require_ssl_verification=true, retries=1) - @test isok(HTTP.get(url; require_ssl_verification=false)) - end - withenv(env..., "JULIA_NO_VERIFY_HOSTS" => "localhost") do - @test !NetworkOptions.verify_host(url) - @test !NetworkOptions.verify_host(url, "SSL") - @test isok(HTTP.get(url)) - @test_throws HTTP.ConnectError HTTP.get(url; require_ssl_verification=true, retries=1) - @test isok(HTTP.get(url; require_ssl_verification=false)) - end - withenv(env..., "JULIA_SSL_NO_VERIFY_HOSTS" => "localhost") do - @test NetworkOptions.verify_host(url) - @test !NetworkOptions.verify_host(url, "SSL") - @test isok(HTTP.get(url)) - @test_throws HTTP.ConnectError HTTP.get(url; require_ssl_verification=true, retries=1) - @test isok(HTTP.get(url; require_ssl_verification=false)) - end - finally - @try Base.IOError close(server) - HTTP.Connections.closeall() - end -end - -@testset "Public entry point of HTTP.request and friends (e.g. issue #463)" begin - headers = Dict("User-Agent" => "HTTP.jl") - query = Dict("hello" => "world") - body = UInt8[1, 2, 3] - stack = HTTP.stack() - function test(r, m) - @test isok(r) - d = JSON.parse(IOBuffer(HTTP.payload(r))) - @test d["headers"]["User-Agent"] == ["HTTP.jl"] - @test d["data"] == "\x01\x02\x03" - @test endswith(d["url"], "?hello=world") - end - for uri in ("https://$httpbin/anything", HTTP.URI("https://$httpbin/anything")) - # HTTP.request - test(HTTP.request("GET", uri; headers=headers, body=body, query=query), "GET") - test(HTTP.request("GET", uri, headers; body=body, query=query), "GET") - test(HTTP.request("GET", uri, headers, body; query=query), "GET") - !isa(uri, HTTP.URI) && test(HTTP.request(stack, "GET", uri; headers=headers, body=body, query=query), "GET") - test(HTTP.request(stack, "GET", uri, headers; body=body, query=query), "GET") - test(HTTP.request(stack, "GET", uri, headers, body; query=query), "GET") - # HTTP.get - test(HTTP.get(uri; headers=headers, body=body, query=query), "GET") - test(HTTP.get(uri, headers; body=body, query=query), "GET") - test(HTTP.get(uri, headers, body; query=query), "GET") - # HTTP.put - test(HTTP.put(uri; headers=headers, body=body, query=query), "PUT") - test(HTTP.put(uri, headers; body=body, query=query), "PUT") - test(HTTP.put(uri, headers, body; query=query), "PUT") - # HTTP.post - test(HTTP.post(uri; headers=headers, body=body, query=query), "POST") - test(HTTP.post(uri, headers; body=body, query=query), "POST") - test(HTTP.post(uri, headers, body; query=query), "POST") - # HTTP.patch - test(HTTP.patch(uri; headers=headers, body=body, query=query), "PATCH") - test(HTTP.patch(uri, headers; body=body, query=query), "PATCH") - test(HTTP.patch(uri, headers, body; query=query), "PATCH") - # HTTP.delete - test(HTTP.delete(uri; headers=headers, body=body, query=query), "DELETE") - test(HTTP.delete(uri, headers; body=body, query=query), "DELETE") - test(HTTP.delete(uri, headers, body; query=query), "DELETE") - end -end - -@testset "HTTP CONNECT Proxy" begin - @testset "Host header" begin - # Stores the http request passed by the client - req = String[] - - # Trivial implementation of a proxy server - # We are only interested in the request passed in by the client - # Returns 400 after reading the http request into req - proxy = listen(IPv4(0), 8082) - try - @async begin - sock = accept(proxy) - while isopen(sock) - line = readline(sock) - isempty(line) && break - - push!(req, line) - end - write(sock, "HTTP/1.1 400 Bad Request\r\n\r\n") - end - - # Make the HTTP request - HTTP.get("https://example.com"; proxy="http://localhost:8082", retry=false, status_exception=false) - - # Test if the host header exist in the request - @test "Host: example.com:443" in req - finally - close(proxy) - HTTP.Connections.closeall() - end - end -end - -@testset "Retry with request/response body streams" begin - shouldfail = Ref(true) - status = Ref(200) - server = HTTP.listen!(8080) do http - @assert !eof(http) - msg = String(read(http)) - if shouldfail[] - shouldfail[] = false - error("500 unexpected error") - end - HTTP.setstatus(http, status[]) - if status[] != 200 - HTTP.startwrite(http) - HTTP.write(http, "$(status[]) unexpected error") - status[] = 200 - end - HTTP.startwrite(http) - HTTP.write(http, msg) - end - try - req_body = IOBuffer("hey there sailor") - seekstart(req_body) - res_body = IOBuffer() - resp = HTTP.get("http://localhost:8080/retry"; body=req_body, response_stream=res_body) - @test isok(resp) - @test String(take!(res_body)) == "hey there sailor" - # ensure if retry=false, that we write the response body immediately - shouldfail[] = true - seekstart(req_body) - resp = HTTP.get("http://localhost:8080/retry"; body=req_body, response_stream=res_body, retry=false, status_exception=false) - @test resp.status == 500 - # even if StatusError, we should still get the right response body - shouldfail[] = true - seekstart(req_body) - try - resp = HTTP.get("http://localhost:8080/retry"; body=req_body, response_stream=res_body, retry=false, forcenew=true) - catch e - @test e isa HTTP.StatusError - @test e.status == 500 - end - # don't throw a 500, but set status to status we don't retry by default - shouldfail[] = false - status[] = 404 - seekstart(req_body) - checked = Ref(0) - @test !HTTP.retryable(404) - check = (s, ex, req, resp, resp_body) -> begin - checked[] += 1 - str = String(resp_body) - if str != "404 unexpected error" || resp.status != 404 - @error "unexpected response body" str - return false - end - @test str == "404 unexpected error" - return resp.status == 404 - end - resp = HTTP.get("http://localhost:8080/retry"; body=req_body, response_stream=res_body, retry_check=check) - @test isok(resp) - @test String(take!(res_body)) == "hey there sailor" - @test checked[] >= 1 - finally - close(server) - HTTP.Connections.closeall() - end -end - -@testset "Don't retry on internal exceptions" begin - kws = (retry_delays = [10, 20, 30], retries=3) # ~ 60 secs - max_wait = 30 - - function test_finish_within(f, secs) - timedout = Ref(false) - t = Timer((t)->(timedout[] = true), secs) - try - f() - finally - close(t) - end - @test !timedout[] - end - - expected = ErrorException("request") - test_finish_within(max_wait) do - @test_throws expected ErrorRequest.get("https://$httpbin/ip"; request_exception=expected, kws...) - end - expected = ArgumentError("request") - test_finish_within(max_wait) do - @test_throws expected ErrorRequest.get("https://$httpbin/ip"; request_exception=expected, kws...) - end - - test_finish_within(max_wait) do - expected = ErrorException("stream") - e = try - ErrorRequest.get("https://$httpbin/ip"; stream_exception=expected, kws...) - catch e - e - end - @assert e isa HTTP.RequestError - @test e.error == expected - end - - test_finish_within(max_wait) do - expected = ArgumentError("stream") - e = try - ErrorRequest.get("https://$httpbin/ip"; stream_exception=expected, kws...) - catch e - e - end - @assert e isa HTTP.RequestError - @test e.error == expected - end -end - -@testset "Retry with ConnectError" begin - mktemp() do path, io - redirect_stdout(io) do - redirect_stderr(io) do - try - HTTP.request("GET", "http://n0nexist3nthost/"; verbose=true) - catch - # ignore - end - end - end - close(io) - logs = read(path, String) - - if occursin("EAI_NODATA", logs) - @test occursin("No Retry: GET / HTTP/1.1", logs) - elseif occursin("EAI_EAGAIN", logs) - # interestingly some environments also result in EAGAIN in these cases - @test occursin("Retry HTTP.Exceptions.ConnectError", logs) - end - end - - # isrecoverable tests - @test !HTTP.RetryRequest.isrecoverable(nothing) - - @test !HTTP.RetryRequest.isrecoverable(ErrorException("")) - @test !HTTP.RetryRequest.isrecoverable(ArgumentError("yikes")) - @test HTTP.RetryRequest.isrecoverable(ArgumentError("stream is closed or unusable")) - - @test HTTP.RetryRequest.isrecoverable(HTTP.RequestError(nothing, ArgumentError("stream is closed or unusable"))) - @test !HTTP.RetryRequest.isrecoverable(HTTP.RequestError(nothing, ArgumentError("yikes"))) - - @test HTTP.RetryRequest.isrecoverable(CapturedException(ArgumentError("stream is closed or unusable"), Any[])) - - recoverable_dns_error = Sockets.DNSError("localhost", Base.UV_EAI_AGAIN) - unrecoverable_dns_error = Sockets.DNSError("localhost", Base.UV_EAI_NONAME) - @test HTTP.RetryRequest.isrecoverable(recoverable_dns_error) - @test !HTTP.RetryRequest.isrecoverable(unrecoverable_dns_error) - @test HTTP.RetryRequest.isrecoverable(HTTP.Exceptions.ConnectError("http://localhost", recoverable_dns_error)) - @test !HTTP.RetryRequest.isrecoverable(HTTP.Exceptions.ConnectError("http://localhost", unrecoverable_dns_error)) - @test HTTP.RetryRequest.isrecoverable(CompositeException([ - recoverable_dns_error, - HTTP.Exceptions.ConnectError("http://localhost", recoverable_dns_error), - ])) - @test HTTP.RetryRequest.isrecoverable(CompositeException([ - recoverable_dns_error, - HTTP.Exceptions.ConnectError("http://localhost", recoverable_dns_error), - CompositeException([recoverable_dns_error]) - ])) - @test !HTTP.RetryRequest.isrecoverable(CompositeException([ - recoverable_dns_error, - HTTP.Exceptions.ConnectError("http://localhost", unrecoverable_dns_error), - ])) - @test !HTTP.RetryRequest.isrecoverable(CompositeException([ - recoverable_dns_error, - HTTP.Exceptions.ConnectError("http://localhost", recoverable_dns_error), - CompositeException([unrecoverable_dns_error]) - ])) -end - -findnewline(bytes) = something(findfirst(==(UInt8('\n')), bytes), 0) - -@testset "IOExtras.readuntil on Stream" begin - HTTP.open(:GET, "https://$httpbin/stream/5") do io - while !eof(io) - bytes = IOExtras.readuntil(io, findnewline) - isempty(bytes) && break - x = JSON.parse(IOBuffer(bytes)) - end - end -end - -@testset "CA_BUNDEL env" begin - resp = withenv("HTTP_CA_BUNDLE" => HTTP.MbedTLS.MozillaCACerts_jll.cacert) do - HTTP.get("https://$httpbin/ip"; socket_type_tls=SSLStream) - end - @test isok(resp) - resp = withenv("HTTP_CA_BUNDLE" => HTTP.MbedTLS.MozillaCACerts_jll.cacert) do - HTTP.get("https://$httpbin/ip") - end - @test isok(resp) -end -end # module + @testset "readtimeout" begin + @test_throws CapturedException HTTP.get("http://$httpbin/delay/5"; readtimeout=1, max_retries=0) + @test isok(HTTP.get("http://$httpbin/delay/1"; readtimeout=2, max_retries=0)) + end + + @testset "Public entry point of HTTP.request and friends (e.g. issue #463)" begin + headers = Dict("User-Agent" => "HTTP.jl") + query = Dict("hello" => "world") + body = UInt8[1, 2, 3] + for uri in ("https://$httpbin/anything", HTTP.URI("https://$httpbin/anything")) + # HTTP.request + @test isok(HTTP.request("GET", uri; headers=headers, body=body, query=query)) + @test isok(HTTP.request("GET", uri, headers; body=body, query=query)) + @test isok(HTTP.request("GET", uri, headers, body; query=query)) + # HTTP.get + @test isok(HTTP.get(uri; headers=headers, body=body, query=query)) + @test isok(HTTP.get(uri, headers; body=body, query=query)) + @test isok(HTTP.get(uri, headers, body; query=query)) + # HTTP.put + @test isok(HTTP.put(uri; headers=headers, body=body, query=query)) + @test isok(HTTP.put(uri, headers; body=body, query=query)) + @test isok(HTTP.put(uri, headers, body; query=query)) + # HTTP.post + @test isok(HTTP.post(uri; headers=headers, body=body, query=query)) + @test isok(HTTP.post(uri, headers; body=body, query=query)) + @test isok(HTTP.post(uri, headers, body; query=query)) + # HTTP.patch + @test isok(HTTP.patch(uri; headers=headers, body=body, query=query)) + @test isok(HTTP.patch(uri, headers; body=body, query=query)) + @test isok(HTTP.patch(uri, headers, body; query=query)) + # HTTP.delete + @test isok(HTTP.delete(uri; headers=headers, body=body, query=query)) + @test isok(HTTP.delete(uri, headers; body=body, query=query)) + @test isok(HTTP.delete(uri, headers, body; query=query)) + end + end +end \ No newline at end of file diff --git a/test/cookies.jl b/test/cookies.jl deleted file mode 100644 index 24b7916a4..000000000 --- a/test/cookies.jl +++ /dev/null @@ -1,287 +0,0 @@ -module TestCookies - -using HTTP -using Sockets, Test - -@testset "Cookies" begin - c = HTTP.Cookies.Cookie() - @test c.name == "" - @test HTTP.Cookies.domainmatch(c, "") - - c.path = "/any" - @test HTTP.Cookies.pathmatch(c, "/any/path") - @test !HTTP.Cookies.pathmatch(c, "/nottherightpath") - - writesetcookietests = [ - (HTTP.Cookie("cookie-1", "v\$1"), "cookie-1=v\$1"), - (HTTP.Cookie("cookie-2", "two", maxage=3600), "cookie-2=two; Max-Age=3600"), - (HTTP.Cookie("cookie-3", "three", domain=".example.com"), "cookie-3=three; Domain=example.com"), - (HTTP.Cookie("cookie-4", "four", path="/restricted/"), "cookie-4=four; Path=/restricted/"), - (HTTP.Cookie("cookie-5", "five", domain="wrong;bad.abc"), "cookie-5=five"), - (HTTP.Cookie("cookie-6", "six", domain="bad-.abc"), "cookie-6=six"), - (HTTP.Cookie("cookie-7", "seven", domain="127.0.0.1"), "cookie-7=seven; Domain=127.0.0.1"), - (HTTP.Cookie("cookie-8", "eight", domain="::1"), "cookie-8=eight"), - (HTTP.Cookie("cookie-9", "expiring", expires=HTTP.Dates.unix2datetime(1257894000)), "cookie-9=expiring; Expires=Tue, 10 Nov 2009 23:00:00 GMT"), - - # According to IETF 6265 Section 5.1.1.5, the year cannot be less than 1601 - (HTTP.Cookie("cookie-10", "expiring-1601", expires=HTTP.Dates.DateTime(1601, 1, 1, 1, 1, 1, 1)), "cookie-10=expiring-1601; Expires=Mon, 01 Jan 1601 01:01:01 GMT"), - (HTTP.Cookie("cookie-11", "invalid-expiry", expires=HTTP.Dates.DateTime(1600, 1, 1, 1, 1, 1, 1)), "cookie-11=invalid-expiry"), - - # The "special" cookies have values containing commas or spaces which - # are disallowed by RFC 6265 but are common in the wild. - (HTTP.Cookie("special-1", "a z"), "special-1=\"a z\""), - (HTTP.Cookie("special-2", " z"), "special-2=\" z\""), - (HTTP.Cookie("special-3", "a "), "special-3=\"a \""), - (HTTP.Cookie("special-4", " "), "special-4=\" \""), - (HTTP.Cookie("special-5", "a,z"), "special-5=\"a,z\""), - (HTTP.Cookie("special-6", ",z"), "special-6=\",z\""), - (HTTP.Cookie("special-7", "a,"), "special-7=\"a,\""), - (HTTP.Cookie("special-8", ","), "special-8=\",\""), - (HTTP.Cookie("empty-value", ""), "empty-value="), - (HTTP.Cookie("", ""), ""), - (HTTP.Cookie("\t", ""), ""), - ] - - @testset "stringify(::Cookie)" begin - for (cookie, expected) in writesetcookietests - @test HTTP.stringify(cookie, false) == expected - end - - cookies = [HTTP.Cookie("cookie-1", "v\$1"), - HTTP.Cookie("cookie-2", "v\$2"), - HTTP.Cookie("cookie-3", "v\$3"), - ] - expected = "cookie-1=v\$1; cookie-2=v\$2; cookie-3=v\$3" - @test HTTP.stringify("", cookies) == expected - - @testset "combine cookies with existing header" begin - @test HTTP.stringify("cookie-0", cookies) == "cookie-0; $expected" - @test HTTP.stringify("cookie-0=", cookies) == "cookie-0=; $expected" - @test HTTP.stringify("cookie-0=0", cookies) == "cookie-0=0; $expected" - @test HTTP.stringify("cookie-0=0 ", cookies) == "cookie-0=0 ; $expected" - @test HTTP.stringify("cookie-0=0 ", cookies) == "cookie-0=0 ; $expected" - @test HTTP.stringify("cookie-0=0;", cookies) == "cookie-0=0; $expected" - @test HTTP.stringify("cookie-0=0; ", cookies) == "cookie-0=0; $expected" - @test HTTP.stringify("cookie-0=0; ", cookies) == "cookie-0=0; $expected" - end - end - - @testset "readsetcookies" begin - cookietests = [ - (["Set-Cookie" => "Cookie-1=v\$1"], [HTTP.Cookie("Cookie-1", "v\$1")]), - (["Set-Cookie" => "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"], - [HTTP.Cookie("NID", "99=YsDT5i3E-CXax-"; path="/", domain=".google.ch", httponly=true, expires=HTTP.Dates.DateTime(2011, 11, 23, 1, 5, 3, 0))]), - (["Set-Cookie" => "NID=99=YsDT5i3E-CXax-; expires=Wed, 23 Nov 2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"], - [HTTP.Cookie("NID", "99=YsDT5i3E-CXax-"; path="/", domain=".google.ch", httponly=true, expires=HTTP.Dates.DateTime(2011, 11, 23, 1, 5, 3, 0))]), - (["Set-Cookie" => ".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly"], - [HTTP.Cookie(".ASPXAUTH", "7E3AA"; path="/", expires=HTTP.Dates.DateTime(2012, 3, 7, 14, 25, 6, 0), httponly=true)]), - (["Set-Cookie" => ".ASPXAUTH=7E3AA; expires=Wed, 07 Mar 2012 14:25:06 GMT; path=/; HttpOnly"], - [HTTP.Cookie(".ASPXAUTH", "7E3AA"; path="/", expires=HTTP.Dates.DateTime(2012, 3, 7, 14, 25, 6, 0), httponly=true)]), - (["Set-Cookie" => "ASP.NET_SessionId=foo; path=/; HttpOnly"], - [HTTP.Cookie("ASP.NET_SessionId", "foo"; path="/", httponly=true)]), - (["Set-Cookie" => "samesitedefault=foo; SameSite"], [HTTP.Cookie("samesitedefault", "foo"; samesite=HTTP.Cookies.SameSiteDefaultMode)]), - (["Set-Cookie" => "samesiteinvalidisdefault=foo; SameSite=invalid"], [HTTP.Cookie("samesiteinvalidisdefault", "foo"; samesite=HTTP.Cookies.SameSiteDefaultMode)]), - (["Set-Cookie" => "samesitelax=foo; SameSite=Lax"], [HTTP.Cookie("samesitelax", "foo"; samesite=HTTP.Cookies.SameSiteLaxMode)]), - (["Set-Cookie" => "samesitestrict=foo; SameSite=Strict"], [HTTP.Cookie("samesitestrict", "foo"; samesite=HTTP.Cookies.SameSiteStrictMode)]), - (["Set-Cookie" => "samesitenone=foo; SameSite=None"], [HTTP.Cookie("samesitenone", "foo"; samesite=HTTP.Cookies.SameSiteNoneMode)]), - (["Set-Cookie" => "special-1=a z"], [HTTP.Cookie("special-1", "a z")]), - (["Set-Cookie" => "special-2=\" z\""], [HTTP.Cookie("special-2", " z")]), - (["Set-Cookie" => "special-3=\"a \""], [HTTP.Cookie("special-3", "a ")]), - (["Set-Cookie" => "special-4=\" \""], [HTTP.Cookie("special-4", " ")]), - (["Set-Cookie" => "special-5=a,z"], [HTTP.Cookie("special-5", "a,z")]), - (["Set-Cookie" => "special-6=\",z\""], [HTTP.Cookie("special-6", ",z")]), - (["Set-Cookie" => "special-7=a,"], [HTTP.Cookie("special-7", "a,")]), - (["Set-Cookie" => "special-8=\",\""], [HTTP.Cookie("special-8", ",")]), - ] - - for (h, c) in cookietests - @test HTTP.Cookies.readsetcookies(h) == c - end - end - - @testset "SetCookieDoubleQuotes" begin - cookiestrings = [ - ["Set-Cookie" => "quoted0=none; max-age=30"], - ["Set-Cookie" => "quoted1=\"cookieValue\"; max-age=31"], - ["Set-Cookie" => "quoted2=cookieAV; max-age=\"32\""], - ["Set-Cookie" => "quoted3=\"both\"; max-age=\"33\""], - ] - - want = [ - [HTTP.Cookie("quoted0", "none", maxage=30)], - [HTTP.Cookie("quoted1", "cookieValue", maxage=31)], - [HTTP.Cookie("quoted2", "cookieAV")], - [HTTP.Cookie("quoted3", "both")], - ] - @test all(HTTP.Cookies.readsetcookies.(cookiestrings) .== want) - end - - @testset "Cookie sanitize value" begin - values = Dict( - "foo" => "foo", - "foo;bar" => "foobar", - "foo\\bar" => "foobar", - "foo\"bar" => "foobar", - String(UInt8[0x00, 0x7e, 0x7f, 0x80]) => String(UInt8[0x7e]), - "\"withquotes\"" => "withquotes", - "a z" => "\"a z\"", - " z" => "\" z\"", - "a " => "\"a \"", - "a,z" => "\"a,z\"", - ",z" => "\",z\"", - "a," => "\"a,\"", - ) - - for (k, v) in values - @test HTTP.Cookies.sanitizeCookieValue(k) == v - end - end - - @testset "Cookie sanitize path" begin - paths = Dict( - "/path" => "/path", - "/path with space/" => "/path with space/", - "/just;no;semicolon\0orstuff/" => "/justnosemicolonorstuff/", - ) - - for (k, v) in paths - @test HTTP.Cookies.sanitizeCookiePath(k) == v - end - end - - @testset "HTTP.readcookies" begin - testcookies = [ - (Dict("Cookie" => "Cookie-1=v\$1; c2=v2"), "", [HTTP.Cookie("Cookie-1", "v\$1"), HTTP.Cookie("c2", "v2")]), - (Dict("Cookie" => "Cookie-1=v\$1; c2=v2"), "c2", [HTTP.Cookie("c2", "v2")]), - (Dict("Cookie" => "Cookie-1=v\$1; c2=v2"), "", [HTTP.Cookie("Cookie-1", "v\$1"), HTTP.Cookie("c2", "v2")]), - (Dict("Cookie" => "Cookie-1=v\$1; c2=v2"), "c2", [HTTP.Cookie("c2", "v2")]), - (Dict("Cookie" => "Cookie-1=\"v\$1\"; c2=\"v2\""), "", [HTTP.Cookie("Cookie-1", "v\$1"), HTTP.Cookie("c2", "v2")]), - ] - - for (h, filter, cookies) in testcookies - @test HTTP.Cookies.readcookies(h, filter) == cookies - end - end - - @testset "Set-Cookie casing" begin - server = HTTP.listen!(8080) do http - t = http.message.target - HTTP.setstatus(http, 200) - if t == "/set-cookie" - HTTP.setheader(http, "set-cookie" => "cookie=lc_cookie") - elseif t == "/Set-Cookie" - HTTP.setheader(http, "Set-Cookie" => "cookie=cc_cookie") - elseif t == "/SET-COOKIE" - HTTP.setheader(http, "SET-COOKIE" => "cookie=uc_cookie") - elseif t == "/SeT-CooKiE" - HTTP.setheader(http, "SeT-CooKiE" => "cookie=spongebob_cookie") - elseif t =="/cookie" - HTTP.setheader(http, "X-Cookie" => HTTP.header(http, "Cookie")) - end - HTTP.startwrite(http) - end - - cookiejar = HTTP.Cookies.CookieJar() - HTTP.get("http://localhost:8080/set-cookie"; cookies=true, cookiejar=cookiejar) - r = HTTP.get("http://localhost:8080/cookie"; cookies=true, cookiejar=cookiejar) - @test HTTP.header(r, "X-Cookie") == "cookie=lc_cookie" - empty!(cookiejar) - HTTP.get("http://localhost:8080/Set-Cookie"; cookies=true, cookiejar=cookiejar) - r = HTTP.get("http://localhost:8080/cookie"; cookies=true, cookiejar=cookiejar) - @test HTTP.header(r, "X-Cookie") == "cookie=cc_cookie" - empty!(cookiejar) - HTTP.get("http://localhost:8080/SET-COOKIE"; cookies=true, cookiejar=cookiejar) - r = HTTP.get("http://localhost:8080/cookie"; cookies=true, cookiejar=cookiejar) - @test HTTP.header(r, "X-Cookie") == "cookie=uc_cookie" - empty!(cookiejar) - HTTP.get("http://localhost:8080/SeT-CooKiE"; cookies=true, cookiejar=cookiejar) - r = HTTP.get("http://localhost:8080/cookie"; cookies=true, cookiejar=cookiejar) - @test HTTP.header(r, "X-Cookie") == "cookie=spongebob_cookie" - close(server) - end - - @testset "splithostport" begin - testcases = [ - # Host name - ("localhost:http", "localhost", "http"), - ("localhost:80", "localhost", "80"), - - # Go-specific host name with zone identifier - ("localhost%lo0:http", "localhost%lo0", "http"), - ("localhost%lo0:80", "localhost%lo0", "80"), - ("[localhost%lo0]:http", "localhost%lo0", "http"), # Go 1 behavior - ("[localhost%lo0]:80", "localhost%lo0", "80"), # Go 1 behavior - - # IP literal - ("127.0.0.1:http", "127.0.0.1", "http"), - ("127.0.0.1:80", "127.0.0.1", "80"), - ("[::1]:http", "::1", "http"), - ("[::1]:80", "::1", "80"), - - # IP literal with zone identifier - ("[::1%lo0]:http", "::1%lo0", "http"), - ("[::1%lo0]:80", "::1%lo0", "80"), - - # Go-specific wildcard for host name - (":http", "", "http"), # Go 1 behavior - (":80", "", "80"), # Go 1 behavior - - # Go-specific wildcard for service name or transport port number - ("golang.org:", "golang.org", ""), # Go 1 behavior - ("127.0.0.1:", "127.0.0.1", ""), # Go 1 behavior - ("[::1]:", "::1", ""), # Go 1 behavior - - # Opaque service name - ("golang.org:https%foo", "golang.org", "https%foo"), # Go 1 behavior - ] - for (hostport, host, port) in testcases - @test HTTP.Cookies.splithostport(hostport) == (host, port, false) - end - errorcases = [ - ("golang.org", "missing port in address"), - ("127.0.0.1", "missing port in address"), - ("[::1]", "missing port in address"), - ("[fe80::1%lo0]", "missing port in address"), - ("[localhost%lo0]", "missing port in address"), - ("localhost%lo0", "missing port in address"), - - ("::1", "too many colons in address"), - ("fe80::1%lo0", "too many colons in address"), - ("fe80::1%lo0:80", "too many colons in address"), - - # Test cases that didn't fail in Go 1 - ("[foo:bar]", "missing port in address"), - ("[foo:bar]baz", "missing port in address"), - ("[foo]bar:baz", "missing port in address"), - - ("[foo]:[bar]:baz", "too many colons in address"), - - ("[foo]:[bar]baz", "unexpected '[' in address"), - ("foo[bar]:baz", "unexpected '[' in address"), - - ("foo]bar:baz", "unexpected ']' in address"), - ] - for (hostport, err) in errorcases - @test HTTP.Cookies.splithostport(hostport) == ("", "", true) - end - end - - @testset "addcookie!" begin - r = HTTP.Request("GET", "/") - c = HTTP.Cookie("NID", "99=YsDT5i3E-CXax-"; path="/", domain=".google.ch", httponly=true, expires=HTTP.Dates.DateTime(2011, 11, 23, 1, 5, 3, 0)) - c_parsed = HTTP.Cookie("NID", "99=YsDT5i3E-CXax-"; path="/", domain="google.ch", httponly=true, expires=HTTP.Dates.DateTime(2011, 11, 23, 1, 5, 3, 0)) - HTTP.addcookie!(r, c) - @test HTTP.header(r, "Cookie") == "NID=99=YsDT5i3E-CXax-" - HTTP.addcookie!(r, c) - @test HTTP.header(r, "Cookie") == "NID=99=YsDT5i3E-CXax-; NID=99=YsDT5i3E-CXax-" - r = HTTP.Response(200) - HTTP.addcookie!(r, c) - @test HTTP.header(r, "Set-Cookie") == "NID=99=YsDT5i3E-CXax-; Path=/; Domain=google.ch; Expires=Wed, 23 Nov 2011 01:05:03 GMT; HttpOnly" - @test [c_parsed] == HTTP.Cookies.readsetcookies(["Set-Cookie" => HTTP.header(r, "Set-Cookie")]) - HTTP.addcookie!(r, c) - @test HTTP.headers(r, "Set-Cookie") == ["NID=99=YsDT5i3E-CXax-; Path=/; Domain=google.ch; Expires=Wed, 23 Nov 2011 01:05:03 GMT; HttpOnly", "NID=99=YsDT5i3E-CXax-; Path=/; Domain=google.ch; Expires=Wed, 23 Nov 2011 01:05:03 GMT; HttpOnly"] - @test [c_parsed, c_parsed] == HTTP.Cookies.readsetcookies(["Set-Cookie"] .=> HTTP.headers(r, "Set-Cookie")) - end -end - -end # module diff --git a/test/coverage/Project.toml b/test/coverage/Project.toml deleted file mode 100644 index 4fbdc4772..000000000 --- a/test/coverage/Project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[deps] -Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037" diff --git a/test/download.jl b/test/download.jl deleted file mode 100644 index 06c49ade4..000000000 --- a/test/download.jl +++ /dev/null @@ -1,92 +0,0 @@ -using HTTP - -import ..httpbin - -@testset "HTTP.download" begin - @testset "Update Period" begin - @test_logs (:info, "Downloading") HTTP.download( - "http://test.greenbytes.de/tech/tc2231/inlwithasciifilenamepdf.asis";) - @test_logs (:info, "Downloading") HTTP.download( - "http://test.greenbytes.de/tech/tc2231/inlwithasciifilenamepdf.asis"; - update_period=0.5) - @test_logs HTTP.download( - "http://test.greenbytes.de/tech/tc2231/inlwithasciifilenamepdf.asis"; - update_period=Inf) - end - - @testset "filename from remote path" begin - - file = HTTP.download("https://httpbingo.julialang.org/html") - @test basename(file) == "html" - file = HTTP.download("https://httpbingo.julialang.org/redirect/2") - @test basename(file) == "2" - # HTTP.jl#696 - file = HTTP.download("https://httpbingo.julialang.org/html?a=b") - @test basename(file) == "html" - # HTTP.jl#896 - file = HTTP.download("https://www.cryst.ehu.es/") - @test isfile(file) # just ensure it downloads and doesn't stack overflow - end - - @testset "Content-Disposition" begin - invalid_content_disposition_fn = HTTP.download( - "http://test.greenbytes.de/tech/tc2231/attonlyquoted.asis") - @test isfile(invalid_content_disposition_fn) - @test basename(invalid_content_disposition_fn) == "attonlyquoted.asis" - - content_disposition_fn = HTTP.download( - "http://test.greenbytes.de/tech/tc2231/inlwithasciifilenamepdf.asis") - @test isfile(content_disposition_fn) - @test basename(content_disposition_fn) == "foo.pdf" - - if Sys.isunix() # Don't try this on windows, quotes are not allowed in windows filenames. - escaped_content_disposition_fn = HTTP.download( - "http://test.greenbytes.de/tech/tc2231/attwithasciifnescapedquote.asis") - @test isfile(escaped_content_disposition_fn) - @test basename(escaped_content_disposition_fn) == "\"quoting\" tested.html" - end - - # HTTP#760 - # This has a redirect, where the original Response has the Content-Disposition - # but the redirected one doesn't - # Neither https://httpbingo.julialang.org/ nor http://test.greenbytes.de/tech/tc2231/ - # has a test-case for this. See: https://github.com/postmanlabs/httpbin/issues/652 - # This test might stop validating the code-path if FigShare changes how they do - # redirects (which they have done before, causing issue HTTP#760, in the first place) - redirected_content_disposition_fn = HTTP.download( - "https://ndownloader.figshare.com/files/6294558") - @test isfile(redirected_content_disposition_fn) - @test basename(redirected_content_disposition_fn) == "rsta20150293_si_001.xlsx" - end - - @testset "Provided Filename" begin - provided_filename = tempname() - returned_filename = HTTP.download( - "http://test.greenbytes.de/tech/tc2231/inlwithasciifilenamepdf.asis", - provided_filename - ) - @test provided_filename == returned_filename - @test isfile(provided_filename) - end - - @testset "Content-Encoding" begin - # Add gz extension if we are determining the filename - gzip_content_encoding_fn = HTTP.download("https://$httpbin/gzip") - @test isfile(gzip_content_encoding_fn) - - # Check content auto decoding - open(gzip_content_encoding_fn, "r") do f - @test HTTP.sniff(read(f, String)) == "application/json; charset=utf-8" - end - - # But not if the local name is fully given. HTTP#573 - mktempdir() do dir - name = joinpath(dir, "foo") - downloaded_name = HTTP.download( - "https://pkg.julialang.org/registry/23338594-aafe-5451-b93e-139f81909106/7858451b7a520344eb60354f69809d30a44e7dae", - name, - ) - @test name == downloaded_name - end - end -end diff --git a/test/handlers.jl b/test/handlers.jl index 0602e272b..c73cf1ca0 100644 --- a/test/handlers.jl +++ b/test/handlers.jl @@ -46,10 +46,10 @@ using HTTP, Test HTTP.register!(r, "/api/widgets/{id}", req -> HTTP.getparam(req, "id")) @test r(HTTP.Request("GET", "/api/widgets/11")) == "11" - HTTP.register!(r, "/api/widgets/{name}", req -> (req.context[:params]["name"], HTTP.getroute(req))) + HTTP.register!(r, "/api/widgets/{name}", req -> (HTTP.getparam(req, "name"), HTTP.getroute(req))) @test r(HTTP.Request("GET", "/api/widgets/11")) == ("11", "/api/widgets/{name}") - HTTP.register!(r, "/api/widgets/acme/{id:[a-z]+}", req -> req.context[:params]["id"]) + HTTP.register!(r, "/api/widgets/acme/{id:[a-z]+}", req -> HTTP.getparam(req, "id")) called[] = false @test r(HTTP.Request("GET", "/api/widgets/acme/11")) == 0 @test !called[] @@ -77,8 +77,8 @@ using HTTP, Test @test r(HTTP.Request("GET", "/api/widgets/abc/subwidget")) == 15 @test r(HTTP.Request("GET", "/api/widgets/abc/subwidgetname")) == 16 - cookie = HTTP.Cookie("abc", "def") - req = HTTP.Request("GET", "/") - req.context[:cookies] = [cookie] - @test HTTP.getcookies(req)[1] == cookie + # cookie = HTTP.Cookie("abc", "def") + # req = HTTP.Request("GET", "/") + # req.context[:cookies] = [cookie] + # @test HTTP.getcookies(req)[1] == cookie end diff --git a/test/http_parser_benchmark.jl b/test/http_parser_benchmark.jl deleted file mode 100644 index 75b2616a3..000000000 --- a/test/http_parser_benchmark.jl +++ /dev/null @@ -1,108 +0,0 @@ -# Based on HttpParser.jl/test/runtests.jl - -module HttpParserTest - -using HttpParser -using Test -using Compat -import Compat: String - -headers = Dict() - -function on_message_begin(parser) - return 0 -end - -function on_url(parser, at, len) - return 0 -end - -function on_status_complete(parser) - return 0 -end - -function on_header_field(parser, at, len) - header = unsafe_string(convert(Ptr{UInt8}, at), Int(len)) - # set the current header - headers["current_header"] = header - return 0 -end - -function on_header_value(parser, at, len) - s = unsafe_string(convert(Ptr{UInt8}, at), Int(len)) - # once we know we have the header value, that will be the value for current header - headers[headers["current_header"]] = s - # reset current_header - headers["current_header"] = "" - return 0 -end - -function on_headers_complete(parser) - p = unsafe_load(parser) - # get first two bits of p.type_and_flags - - # The parser type are the bottom two bits - # 0x03 = 00000011 - ptype = p.type_and_flags & 0x03 - # flags = p.type_and_flags >>> 3 - if ptype == 0 - method = http_method_str(convert(Int, p.method)) - end - if ptype == 1 - headers["status_code"] = string(convert(Int, p.status_code)) - end - headers["http_major"] = string(convert(Int, p.http_major)) - headers["http_minor"] = string(convert(Int, p.http_minor)) - headers["Keep-Alive"] = string(http_should_keep_alive(parser)) - return 0 -end - -function on_body(parser, at, len) - return 0 -end - -function on_message_complete(parser) - return 0 -end - -function on_chunk_header(parser) - return 0 -end - -function on_chunk_complete(parser) - return 0 -end - -c_message_begin_cb = cfunction(on_message_begin, HttpParser.HTTP_CB...) -c_url_cb = cfunction(on_url, HttpParser.HTTP_DATA_CB...) -c_status_complete_cb = cfunction(on_status_complete, HttpParser.HTTP_CB...) -c_header_field_cb = cfunction(on_header_field, HttpParser.HTTP_DATA_CB...) -c_header_value_cb = cfunction(on_header_value, HttpParser.HTTP_DATA_CB...) -c_headers_complete_cb = cfunction(on_headers_complete, HttpParser.HTTP_CB...) -c_body_cb = cfunction(on_body, HttpParser.HTTP_DATA_CB...) -c_message_complete_cb = cfunction(on_message_complete, HttpParser.HTTP_CB...) -c_body_cb = cfunction(on_body, HttpParser.HTTP_DATA_CB...) -c_message_complete_cb = cfunction(on_message_complete, HttpParser.HTTP_CB...) -c_chunk_header_cb = cfunction(on_chunk_header, HttpParser.HTTP_CB...) -c_chunk_complete_cb = cfunction(on_chunk_complete, HttpParser.HTTP_CB...) - -function parse(bytes) - # reset request - # Moved this up for testing purposes - global headers = Dict() - parser = Parser() - http_parser_init(parser, false) - settings = ParserSettings(c_message_begin_cb, c_url_cb, - c_status_complete_cb, c_header_field_cb, - c_header_value_cb, c_headers_complete_cb, - c_body_cb, c_message_complete_cb, - c_chunk_header_cb, c_chunk_complete_cb) - - size = http_parser_execute(parser, settings, bytes) - - return headers -end - - - -end diff --git a/test/loopback.jl b/test/loopback.jl deleted file mode 100644 index d8681078b..000000000 --- a/test/loopback.jl +++ /dev/null @@ -1,358 +0,0 @@ -module TestLoopback - -using Test -using HTTP -using HTTP.IOExtras -using HTTP.Parsers -using HTTP.Messages -using HTTP.Sockets -import ..httpbin - -mutable struct FunctionIO <: IO - f::Function - buf::IOBuffer - done::Bool -end -FunctionIO(f::Function) = FunctionIO(f, IOBuffer(), false) - -mutable struct Loopback <: IO - got_headers::Bool - buf::IOBuffer - io::Base.BufferStream -end -Loopback() = Loopback(false, IOBuffer(), Base.BufferStream()) - -pool = HTTP.Pool(1) - -config = [ - :socket_type => Loopback, - :retry => false, - :pool => pool, -] - -server_events = [] - -call(fio::FunctionIO) = !fio.done && (fio.buf = IOBuffer(fio.f()) ; fio.done = true) - -Base.bytesavailable(fio::FunctionIO) = (call(fio); bytesavailable(fio.buf)) -Base.bytesavailable(lb::Loopback) = bytesavailable(lb.io) -Base.close(lb::Loopback) = (close(lb.io); close(lb.buf)) -Base.eof(fio::FunctionIO) = (call(fio); eof(fio.buf)) -Base.eof(lb::Loopback) = eof(lb.io) -Base.isopen(lb::Loopback) = isopen(lb.io) -Base.read(fio::FunctionIO, a...) = (call(fio); read(fio.buf, a...)) -Base.readavailable(fio::FunctionIO) = (call(fio); readavailable(fio.buf)) -Base.readavailable(lb::Loopback) = readavailable(lb.io) -Base.unsafe_read(lb::Loopback, p::Ptr{UInt8}, n::UInt) = unsafe_read(lb.io, p, n) - -HTTP.IOExtras.tcpsocket(::Loopback) = TCPSocket() - -lbreq(req, headers, body; method="GET", kw...) = - HTTP.request(method, "http://test/$req", headers, body; config..., kw...) - -lbopen(f, req, headers) = - HTTP.open(f, "PUT", "http://test/$req", headers; config...) - -function reset(lb::Loopback) - truncate(lb.buf, 0) - lb.got_headers = false -end - -""" - escapelines(string) - -Escape `string` and insert '\n' after escaped newline characters. -""" -function escapelines(s::String) - s = Base.escape_string(s) - s = replace(s, "\\n" => "\\n\n ") - return string(" ", strip(s)) -end - -function on_headers(f::Function, lb::Loopback) - if lb.got_headers - return - end - - buf = copy(lb.buf) - seek(buf, 0) - req = Request() - - try - readheaders(buf, req) - lb.got_headers = true - catch e - if !(e isa EOFError || e isa HTTP.ParseError) - rethrow(e) - end - end - - if lb.got_headers - f(req) - end -end - -function on_body(f::Function, lb::Loopback) - s = String(take!(copy(lb.buf))) - req = nothing - - try - req = parse(HTTP.Request, s) - catch e - if !(e isa EOFError || e isa HTTP.ParseError) - rethrow(e) - end - end - - if req !== nothing - reset(lb) - @async try - f(req) - catch e - println("⚠️ on_body exception: $(sprint(showerror, e))\n$(stacktrace(catch_backtrace()))") - end - end -end - -function Base.unsafe_write(lb::Loopback, p::Ptr{UInt8}, n::UInt) - global server_events - - if !isopen(lb.buf) - throw(ArgumentError("stream is closed or unusable")) - end - - n = unsafe_write(lb.buf, p, n) - - on_headers(lb) do req - println("📡 $(HTTP.sprintcompact(req))") - push!(server_events, "Request: $(HTTP.sprintcompact(req))") - - if req.target == "/abort" - reset(lb) - response = HTTP.Response(403, ["Connection" => "close", - "Content-Length" => 0]; request=req) - push!(server_events, "Response: $(HTTP.sprintcompact(response))") - write(lb.io, response) - end - end - - on_body(lb) do req - l = length(req.body) - response = HTTP.Response(200, ["Content-Length" => l], - body = req.body; request=req) - if req.target == "/echo" - push!(server_events, "Response: $(HTTP.sprintcompact(response))") - write(lb.io, response) - elseif (m = match(r"^/delay([0-9]*)$", req.target)) !== nothing - t = parse(Int, first(m.captures)) - sleep(t/10) - push!(server_events, "Response: $(HTTP.sprintcompact(response))") - write(lb.io, response) - else - response = HTTP.Response(403, - ["Connection" => "close", - "Content-Length" => 0]; request=req) - push!(server_events, "Response: $(HTTP.sprintcompact(response))") - write(lb.io, response) - end - end - - return n -end - -function HTTP.Connections.getconnection(::Type{Loopback}, - host::AbstractString, - port::AbstractString; - kw...)::Loopback - return Loopback() -end - -function async_test(m=["GET","GET","GET","GET","GET"];kw...) - r1 = r2 = r3 = r4 = r5 = nothing - t1 = time() - - @sync begin - @async r1 = lbreq("delay1", [], FunctionIO(()->(sleep(0.00); "Hello World! 1")); - method=m[1], kw...) - @async r2 = lbreq("delay2", [], - FunctionIO(()->(sleep(0.01); "Hello World! 2")); - method=m[2], kw...) - @async r3 = lbreq("delay3", [], - FunctionIO(()->(sleep(0.02); "Hello World! 3")); - method=m[3], kw...) - @async r4 = lbreq("delay4", [], - FunctionIO(()->(sleep(0.03); "Hello World! 4")); - method=m[4], kw...) - @async r5 = lbreq("delay5", [], - FunctionIO(()->(sleep(0.04); "Hello World! 5")); - method=m[5], kw...) - end - t2 = time() - - @test String(r1.body) == "Hello World! 1" - @test String(r2.body) == "Hello World! 2" - @test String(r3.body) == "Hello World! 3" - @test String(r4.body) == "Hello World! 4" - @test String(r5.body) == "Hello World! 5" - - return t2 - t1 -end - -@testset "loopback" begin - global server_events - - @testset "FunctionIO" begin - io = FunctionIO(()->"Hello World!") - @test String(read(io)) == "Hello World!" - end - - @testset "lbreq - IOBuffer" begin - r = lbreq("echo", [], ["Hello", IOBuffer(" "), "World!"]); - @test String(r.body) == "Hello World!" - end - - @testset "lbreq - FunctionIO" begin - r = lbreq("echo", [], FunctionIO(()->"Hello World!")) - @test String(r.body) == "Hello World!" - end - - @testset "lbreq - Array of Strings" begin - r = lbreq("echo", [], ["Hello", " ", "World!"]); - @test String(r.body) == "Hello World!" - end - - @testset "lbreq - Array of Bytes - Echo" begin - r = lbreq("echo", [], [HTTP.bytes("Hello"), - HTTP.bytes(" "), - HTTP.bytes("World!")]); - @test String(r.body) == "Hello World!" - end - - @testset "lbreq - Array of Bytes - Delay" begin - r = lbreq("delay10", [], [HTTP.bytes("Hello"), - HTTP.bytes(" "), - HTTP.bytes("World!")]); - @test String(r.body) == "Hello World!" - end - - @testset "lbopen - Body - Delay" begin - body = Ref{Any}(nothing) - body_sent = Ref(false) - r = lbopen("delay10", []) do http - @sync begin - @async begin - write(http, "Hello World!") - closewrite(http) - body_sent[] = true - end - startread(http) - body[] = read(http) - closeread(http) - end - end - @test String(body[]) == "Hello World!" - end - - # "If [the response] indicates the server does not wish to receive the - # message body and is closing the connection, the client SHOULD - # immediately cease transmitting the body and close the connection." - # https://tools.ietf.org/html/rfc7230#section-6.5 - @testset "lbopen - Body - Abort" begin - body = nothing - body_aborted = false - body_sent = false - @test_throws HTTP.StatusError begin - r = lbopen("abort", []) do http - @sync begin - event = Base.Event() - @async try - wait(event) - write(http, "Hello World!") - closewrite(http) - body_sent = true - catch e - if e isa ArgumentError && - e.msg == "stream is closed or unusable" - body_aborted = true - else - rethrow(e) - end - end - startread(http) - body = read(http) - closeread(http) - notify(event) - end - end - end - @test body_aborted == true - @test body_sent == false - end - - @testset "libreq - Sleep" begin - r = lbreq("echo", [], [ - FunctionIO(()->(sleep(0.1); "Hello")), - FunctionIO(()->(sleep(0.1); " World!"))]) - @test String(r.body) == "Hello World!" - - hello_sent = Ref(false) - world_sent = Ref(false) - @test_throws HTTP.RequestError begin - r = lbreq("abort", [], [ - FunctionIO(()->(hello_sent[] = true; sleep(1.0); "Hello")), - FunctionIO(()->(world_sent[] = true; " World!"))]) - end - @test hello_sent[] - @test !world_sent[] - end - - @testset "ASync - Pipeline limit = 0" begin - server_events = [] - t = async_test(;pipeline_limit=0) - if haskey(ENV, "HTTP_JL_TEST_TIMING_SENSITIVE") - @test server_events == [ - "Request: GET /delay1 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay1 HTTP/1.1)", - "Request: GET /delay2 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay2 HTTP/1.1)", - "Request: GET /delay3 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay3 HTTP/1.1)", - "Request: GET /delay4 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay4 HTTP/1.1)", - "Request: GET /delay5 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay5 HTTP/1.1)"] - end - end - - @testset "ASync - " begin - server_events = [] - t = async_test() - if haskey(ENV, "HTTP_JL_TEST_TIMING_SENSITIVE") - @test server_events == [ - "Request: GET /delay1 HTTP/1.1", - "Request: GET /delay2 HTTP/1.1", - "Request: GET /delay3 HTTP/1.1", - "Request: GET /delay4 HTTP/1.1", - "Request: GET /delay5 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (GET /delay1 HTTP/1.1)", - "Response: HTTP/1.1 200 OK <= (GET /delay2 HTTP/1.1)", - "Response: HTTP/1.1 200 OK <= (GET /delay3 HTTP/1.1)", - "Response: HTTP/1.1 200 OK <= (GET /delay4 HTTP/1.1)", - "Response: HTTP/1.1 200 OK <= (GET /delay5 HTTP/1.1)"] - end - end - - # "A user agent SHOULD NOT pipeline requests after a - # non-idempotent method, until the final response - # status code for that method has been received" - # https://tools.ietf.org/html/rfc7230#section-6.3.2 - @testset "ASync - " begin - server_events = [] - t = async_test(["POST","GET","GET","GET","GET"]) - @test server_events[1:2] == [ - "Request: POST /delay1 HTTP/1.1", - "Response: HTTP/1.1 200 OK <= (POST /delay1 HTTP/1.1)"] - end -end - -end # module diff --git a/test/messages.jl b/test/messages.jl deleted file mode 100644 index 29c39fc3f..000000000 --- a/test/messages.jl +++ /dev/null @@ -1,226 +0,0 @@ -using Test - -using HTTP.Messages -import ..isok, ..httpbin -import HTTP.Messages: appendheader, mkheaders -import HTTP.URI -import HTTP.request -import HTTP: bytes, nbytes -using HTTP: StatusError - -using JSON - -@testset "HTTP.Messages" begin - req = Request("GET", "/foo", ["Foo" => "Bar"]) - res = Response(200, ["Content-Length" => "5"]; body="Hello", request=req) - - http_reads = ["GET", "HEAD", "OPTIONS"] - http_writes = ["POST", "PUT", "DELETE", "PATCH"] - - @testset "Invalid headers" begin - @test_throws ArgumentError mkheaders(["hello"]) - @test_throws ArgumentError mkheaders([(1,2,3,4,5)]) - @test_throws ArgumentError mkheaders([1, 2]) - @test_throws ArgumentError mkheaders(["hello", "world"]) - end - - @testset "Body Length" begin - @test nbytes(7) === nothing - @test nbytes(UInt8[1,2,3]) == 3 - @test nbytes(view(UInt8[1,2,3], 1:2)) == 2 - @test nbytes("Hello") == 5 - @test nbytes(SubString("World!",1,5)) == 5 - @test nbytes(["Hello", " ", "World!"]) == 12 - @test nbytes(["Hello", " ", SubString("World!",1,5)]) == 11 - @test nbytes([SubString("Hello", 1,5), " ", SubString("World!",1,5)]) == 11 - @test nbytes([UInt8[1,2,3], UInt8[4,5,6]]) == 6 - @test nbytes([UInt8[1,2,3], view(UInt8[4,5,6],1:2)]) == 5 - @test nbytes([view(UInt8[1,2,3],1:2), view(UInt8[4,5,6],1:2)]) == 4 - @test nbytes(IOBuffer("foo")) == 3 - @test nbytes([IOBuffer("foo"), IOBuffer("bar")]) == 6 - end - - @testset "Body Bytes" begin - @test bytes(7) == 7 - @test bytes(UInt8[1,2,3]) == UInt8[1,2,3] - @test bytes(view(UInt8[1,2,3], 1:2)) == UInt8[1,2] - @test bytes("Hello") == codeunits("Hello") - @test bytes(SubString("World!",1,5)) == codeunits("World") - @test bytes(["Hello", " ", "World!"]) == ["Hello", " ", "World!"] - @test bytes([UInt8[1,2,3], UInt8[4,5,6]]) == [UInt8[1,2,3], UInt8[4,5,6]] - end - - @testset "Request" begin - @test req.method == "GET" - @test res.request.method == "GET" - - @test String(req) == "GET /foo HTTP/1.1\r\nFoo: Bar\r\n\r\n" - @test String(res) == "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nHello" - - @test header(req, "Foo") == "Bar" - @test header(res, "Content-Length") == "5" - - setheader(req, "X" => "Y") - @test header(req, "X") == "Y" - removeheader(req, "X") - @test header(req, "X") == "" - setheader(req, "X" => "Y") - end - - @testset "Response" begin - @test HTTP.Response(HTTP.Response(200).status).status == 200 - end - - @testset "Header Append" begin - append_header(m, h) = appendheader(m, SubString(h[1]) => SubString(h[2])) - - append_header(req, "X" => "Z") - @test header(req, "X") == "Y, Z" - @test hasheader(req, "X", "Y, Z") - @test headercontains(req, "X", "Y") - @test headercontains(req, "X", "Z") - @test !headercontains(req, "X", "more") - - append_header(req, "X" => "more") - @test header(req, "X") == "Y, Z, more" - @test hasheader(req, "X", "Y, Z, more") - @test headercontains(req, "X", "Y") - @test headercontains(req, "X", "Z") - @test headercontains(req, "X", "more") - - append_header(req, "Set-Cookie" => "A") - append_header(req, "Set-Cookie" => "B") - @test filter(x->first(x) == "Set-Cookie", req.headers) == ["Set-Cookie" => "A", "Set-Cookie" => "B"] - end - - @testset "Header default" begin - @test !hasheader(req, "Null") - @test header(req, "Null") == "" - @test header(req, "Null", nothing) === nothing - end - - @testset "HTTP message parsing" begin - raw = String(req) - req = parse(Request,raw) - @test String(req) == raw - - req = parse(Request, raw * "xxx") - @test String(req) == raw - - raw = String(res) - res = parse(Response,raw) - @test String(res) == raw - - res = parse(Response,raw * "xxx") - @test String(res) == raw - end - - @testset "Read methods" for method in http_reads - @test isok(request(method, "https://$httpbin/ip", verbose=1)) - end - - @testset "Body - Response Stream" for method in http_writes - uri = "https://$httpbin/$(lowercase(method))" - r = request(method, uri, verbose=1) - @test isok(r) - r1 = JSON.parse(String(r.body)) - io = IOBuffer() - r = request(method, uri, response_stream=io, verbose=1) - seekstart(io) - @test isok(r) - r2 = JSON.parse(io) - for (k, v) in r1 - if k == "headers" - for (k2, v2) in r1[k] - if k2 != "X-Amzn-Trace-Id" - @test r1[k][k2] == r2[k][k2] - end - end - else - @test r1[k] == r2[k] - end - end - end - - @testset "Body - JSON Parse" for method in http_writes - uri = "https://$httpbin/$(lowercase(method))" - io = IOBuffer() - r = request(method, uri, response_stream=io, verbose=1) - @test isok(r) - - r = request("POST", - "https://$httpbin/post", - ["Expect" => "100-continue"], - "Hello", - verbose=1) - - @test isok(r) - r = JSON.parse(String(r.body)) - @test r["data"] == "Hello" - end - - @testset "Write to file" begin - cd(mktempdir()) do - - line_count = 0 - num_lines = 50 - open("result_file", "w") do io - r = request("GET", - "https://$httpbin/stream/$num_lines", - response_stream=io, - verbose=1) - @test isok(r) - end - - for line in readlines("result_file") - line_parsed = JSON.parse(line) - @test line_count == line_parsed["id"] - line_count += 1 - end - - @test line_count == num_lines - end - end - - @testset "Display" begin - @test repr(Response(200, []; body="Hello world.")) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\"\"\"" - - # truncation of long bodies - for body_show_max in (Messages.BODY_SHOW_MAX[], 100) - Messages.set_show_max(body_show_max) - @test repr(Response(200, []; body="Hello world.\n"*'x'^10000)) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\nHello world.\n"*'x'^(body_show_max-13)*"\n⋮\n10013-byte body\n\"\"\"" - end - - # don't display raw binary (non-Unicode) data: - @test repr(Response(200, []; body=String([0xde,0xad,0xc1,0x71,0x1c]))) == "Response:\n\"\"\"\nHTTP/1.1 200 OK\r\n\r\n\n⋮\n5-byte body\n\"\"\"" - - # https://github.com/JuliaWeb/HTTP.jl/issues/828 - # don't include empty headers in request when writing - @test repr(Request("GET", "/", ["Accept" => ""])) == "Request:\n\"\"\"\nGET / HTTP/1.1\r\n\r\n\"\"\"" - - # Test that sensitive header values are masked when `show`ing HTTP.Request and HTTP.Response - for H in ["Authorization", "Proxy-Authorization", "Cookie", "Set-Cookie"], h in (lowercase(H), H) - req = HTTP.Request("GET", "https://xyz.com", [h => "secret", "User-Agent" => "HTTP.jl"]) - req_str = sprint(show, req) - @test !occursin("secret", req_str) - @test occursin("$h: ******", req_str) - @test occursin("HTTP.jl", req_str) - resp = HTTP.Response(200, [h => "secret", "Server" => "HTTP.jl"]) - resp_str = sprint(show, resp) - @test !occursin("secret", resp_str) - @test occursin("$h: ******", req_str) - @test occursin("HTTP.jl", resp_str) - end - end - - @testset "queryparams" begin - no_params = Request("GET", "http://google.com") - with_params = Request("GET", "http://google.com?q=123&l=345") - - @test HTTP.queryparams(no_params) == Dict{String, String}() - @test HTTP.queryparams(with_params) == Dict("q" => "123", "l" => "345") - - @test HTTP.queryparams(Response(200; body="", request=with_params)) == Dict("q" => "123", "l" => "345") - @test isnothing(HTTP.queryparams(Response(200; body=""))) - end -end diff --git a/test/multipart.jl b/test/multipart.jl index 5d4f0e744..1defa2c09 100644 --- a/test/multipart.jl +++ b/test/multipart.jl @@ -1,6 +1,8 @@ +import HTTP.Forms: find_multipart_boundary, find_multipart_boundaries, find_header_boundary, parse_multipart_chunk, parse_multipart_body, parse_multipart_form + function test_multipart(r, body) @test isok(r) - json = JSON.parse(IOBuffer(HTTP.payload(r))) + json = JSON.parse(IOBuffer(r.body)) @test startswith(json["headers"]["Content-Type"][1], "multipart/form-data; boundary=") reset(body); mark(body) end @@ -30,8 +32,216 @@ end @test HTTP.Form(Dict(); boundary="a") isa HTTP.Form @test HTTP.Form(Dict(); boundary=" Aa1'()+,-.:=?") isa HTTP.Form @test HTTP.Form(Dict(); boundary='a'^70) isa HTTP.Form - @test_throws ArgumentError HTTP.Form(Dict(); boundary="") - @test_throws ArgumentError HTTP.Form(Dict(); boundary='a'^71) - @test_throws ArgumentError HTTP.Form(Dict(); boundary="a ") + @test_throws AssertionError HTTP.Form(Dict(); boundary="") + @test_throws AssertionError HTTP.Form(Dict(); boundary='a'^71) + @test_throws AssertionError HTTP.Form(Dict(); boundary="a ") + end +end + +function generate_test_body() + Vector{UInt8}(join([ + "----------------------------918073721150061572809433", + "Content-Disposition: form-data; name=\"namevalue\"; filename=\"multipart.txt\"", + "Content-Type: text/plain", + "", + "not much to say\n", + "----------------------------918073721150061572809433", + "Content-Disposition: form-data; name=\"key1\"", + "", + "1", + "----------------------------918073721150061572809433", + "Content-Disposition: form-data; name=\"key2\"", + "", + "key the second", + "----------------------------918073721150061572809433", + "Content-Disposition: form-data; name=\"namevalue2\"; filename=\"multipart-leading-newline.txt\"", + "Content-Type: text/plain", + "", + "\nfile with leading newline\n", + "----------------------------918073721150061572809433", + "Content-Disposition: form-data; name=\"json_file1\"; filename=\"my-json-file-1.json\"", + "Content-Type: application/json", + "", + "{\"data\": [\"this is json data\"]}", + "----------------------------918073721150061572809433", + "content-type: text/plain", + "content-disposition: form-data; name=\"key3\"", + "", + "This file has lower-cased content- keys, and disposition comes second.", + "----------------------------918073721150061572809433--", + "", + ], "\r\n")) +end + +function generate_test_request() + body = generate_test_body() + headers = [ + "User-Agent" => "PostmanRuntime/7.15.2", + "Accept" => "*/*", + "Cache-Control" => "no-cache", + "Postman-Token" => "288c2481-1837-4ba9-add3-f23d380fa440", + "Host" => "localhost:8888", + "Accept-Encoding" => "gzip, deflate", + "Accept-Encoding" => "gzip, deflate", + "Content-Type" => "multipart/form-data; boundary=--------------------------918073721150061572809433", + "Content-Length" => string(length(body)), + "Connection" => "keep-alive", + ] + r = HTTP.Request("POST", "/", headers) + r.body = body + r +end + +function generate_non_multi_test_request() + headers = [ + "User-Agent" => "PostmanRuntime/7.15.2", + "Accept" => "*/*", + "Cache-Control" => "no-cache", + "Postman-Token" => "288c2481-1837-4ba9-add3-f23d380fa440", + "Host" => "localhost:8888", + "Accept-Encoding" => "gzip, deflate", + "Accept-Encoding" => "gzip, deflate", + "Content-Length" => "0", + "Connection" => "keep-alive", + ] + HTTP.Request("POST", "/", headers, Vector{UInt8}()) +end + +function generate_test_response() + body = generate_test_body() + headers = [ + "Date" => "Fri, 25 Mar 2022 14:16:21 GMT", + "Transfer-Encoding" => "chunked", + "Content-Type" => "multipart/form-data; boundary=--------------------------918073721150061572809433", + "Content-Length" => string(length(body)), + "Connection" => "keep-alive", + ] + r = HTTP.Response(200) + r.headers = headers + r.body = body + r +end + +@testset "parse multipart form-data" begin + @testset "find_multipart_boundary" begin + request = generate_test_request() + + # NOTE: this is the start of a "boundary delimiter line" and has two leading + # '-' characters prepended to the boundary delimiter from Content-Type header + delimiter = Vector{UInt8}("--------------------------918073721150061572809433") + body = generate_test_body() + # length of the delimiter, CRLF, and -1 for the end index to be the LF character + endIndexOffset = length(delimiter) + 4 - 1 + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter) + @test !isTerminatingDelimiter + @test 1 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + # the remaining "boundary delimiter lines" will have a CRLF preceding them + endIndexOffset += 2 + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 1) + @test !isTerminatingDelimiter + @test 175 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 3) + @test !isTerminatingDelimiter + @test 279 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 3) + @test !isTerminatingDelimiter + @test 396 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 3) + @test !isTerminatingDelimiter + @test 600 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 3) + @test !isTerminatingDelimiter + @test 804 == startIndex + @test (startIndex + endIndexOffset) == endIndex + + (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, startIndex + 3) + @test isTerminatingDelimiter + @test 1003 == startIndex + # +2 because of the two additional '--' characters + @test (startIndex + endIndexOffset + 2) == endIndex + end + + @testset "parse_multipart_form request" begin + @test HTTP.parse_multipart_form(generate_non_multi_test_request()) === nothing + + multiparts = HTTP.parse_multipart_form(generate_test_request()) + @test 6 == length(multiparts) + + @test "multipart.txt" === multiparts[1].filename + @test "namevalue" === multiparts[1].name + @test "text/plain" === multiparts[1].contenttype + @test "not much to say\n" === String(read(multiparts[1].data)) + + @test multiparts[2].filename === nothing + @test "key1" === multiparts[2].name + @test "text/plain" === multiparts[2].contenttype + @test "1" === String(read(multiparts[2].data)) + + @test multiparts[3].filename === nothing + @test "key2" === multiparts[3].name + @test "text/plain" === multiparts[3].contenttype + @test "key the second" === String(read(multiparts[3].data)) + + @test "multipart-leading-newline.txt" === multiparts[4].filename + @test "namevalue2" === multiparts[4].name + @test "text/plain" === multiparts[4].contenttype + @test "\nfile with leading newline\n" === String(read(multiparts[4].data)) + + @test "my-json-file-1.json" === multiparts[5].filename + @test "json_file1" === multiparts[5].name + @test "application/json" === multiparts[5].contenttype + @test """{"data": ["this is json data"]}""" === String(read(multiparts[5].data)) + + @test multiparts[6].filename === nothing + @test "key3" === multiparts[6].name + @test "text/plain" === multiparts[6].contenttype + @test "This file has lower-cased content- keys, and disposition comes second." === String(read(multiparts[6].data)) + end + + @testset "parse_multipart_form response" begin + multiparts = HTTP.parse_multipart_form(generate_test_response()) + @test 6 == length(multiparts) + + @test "multipart.txt" === multiparts[1].filename + @test "namevalue" === multiparts[1].name + @test "text/plain" === multiparts[1].contenttype + @test "not much to say\n" === String(read(multiparts[1].data)) + + @test multiparts[2].filename === nothing + @test "key1" === multiparts[2].name + @test "text/plain" === multiparts[2].contenttype + @test "1" === String(read(multiparts[2].data)) + + @test multiparts[3].filename === nothing + @test "key2" === multiparts[3].name + @test "text/plain" === multiparts[3].contenttype + @test "key the second" === String(read(multiparts[3].data)) + + @test "multipart-leading-newline.txt" === multiparts[4].filename + @test "namevalue2" === multiparts[4].name + @test "text/plain" === multiparts[4].contenttype + @test "\nfile with leading newline\n" === String(read(multiparts[4].data)) + + @test "my-json-file-1.json" === multiparts[5].filename + @test "json_file1" === multiparts[5].name + @test "application/json" === multiparts[5].contenttype + @test """{"data": ["this is json data"]}""" === String(read(multiparts[5].data)) + + @test multiparts[6].filename === nothing + @test "key3" === multiparts[6].name + @test "text/plain" === multiparts[6].contenttype + @test "This file has lower-cased content- keys, and disposition comes second." === String(read(multiparts[6].data)) end end diff --git a/test/mwe.jl b/test/mwe.jl deleted file mode 100644 index 1a30dff6d..000000000 --- a/test/mwe.jl +++ /dev/null @@ -1,40 +0,0 @@ -using HTTP -using HTTP: hasheader -using MbedTLS - -@static if Sys.isapple() - launch(x) = run(`open $x`) -elseif Sys.islinux() - launch(x) = run(`xdg-open $x`) -elseif Sys.iswindows() - launch(x) = run(`cmd /C start $x`) -end - -@test_skip @testset "MWE" begin - @async begin - sleep(2) - launch("https://127.0.0.1:8000/examples/mwe") - end - dir = joinpath(dirname(pathof(HTTP)), "../test") - HTTP.listen("127.0.0.1", 8000; - sslconfig = MbedTLS.SSLConfig(joinpath(dir, "resources/cert.pem"), - joinpath(dir, "resources/key.pem"))) do http - - if HTTP.WebSockets.isupgrade(http.message) - HTTP.WebSockets.upgrade(http) do client - count = 1 - while !eof(client); - msg = String(readavailable(client)) - println(msg) - write(client, "Hello JavaScript! From Julia $count") - count += 1 - end - end - else - h = HTTP.Handlers.RequestHandlerFunction() do req::HTTP.Request - HTTP.Response(200,read(joinpath(dir, "resources/mwe.html"), String)) - end - HTTP.Handlers.handle(h, http) - end - end -end diff --git a/test/parsemultipart.jl b/test/parsemultipart.jl deleted file mode 100644 index 86c029da1..000000000 --- a/test/parsemultipart.jl +++ /dev/null @@ -1,213 +0,0 @@ -using Test -using HTTP -import HTTP.MultiPartParsing: find_multipart_boundary, find_multipart_boundaries, find_header_boundary, parse_multipart_chunk, parse_multipart_body, parse_multipart_form - -function generate_test_body() - Vector{UInt8}(join([ - "----------------------------918073721150061572809433", - "Content-Disposition: form-data; name=\"namevalue\"; filename=\"multipart.txt\"", - "Content-Type: text/plain", - "", - "not much to say\n", - "----------------------------918073721150061572809433", - "Content-Disposition: form-data; name=\"key1\"", - "", - "1", - "----------------------------918073721150061572809433", - "Content-Disposition: form-data; name=\"key2\"", - "", - "key the second", - "----------------------------918073721150061572809433", - "Content-Disposition: form-data; name=\"namevalue2\"; filename=\"multipart-leading-newline.txt\"", - "Content-Type: text/plain", - "", - "\nfile with leading newline\n", - "----------------------------918073721150061572809433", - "Content-Disposition: form-data; name=\"json_file1\"; filename=\"my-json-file-1.json\"", - "Content-Type: application/json", - "", - "{\"data\": [\"this is json data\"]}", - "----------------------------918073721150061572809433", - "content-type: text/plain", - "content-disposition: form-data; name=\"key3\"", - "", - "This file has lower-cased content- keys, and disposition comes second.", - "----------------------------918073721150061572809433--", - "", - ], "\r\n")) -end - -function generate_test_request() - body = generate_test_body() - - headers = [ - "User-Agent" => "PostmanRuntime/7.15.2", - "Accept" => "*/*", - "Cache-Control" => "no-cache", - "Postman-Token" => "288c2481-1837-4ba9-add3-f23d380fa440", - "Host" => "localhost:8888", - "Accept-Encoding" => "gzip, deflate", - "Accept-Encoding" => "gzip, deflate", - "Content-Type" => "multipart/form-data; boundary=--------------------------918073721150061572809433", - "Content-Length" => string(length(body)), - "Connection" => "keep-alive", - ] - - HTTP.Request("POST", "/", headers, body) -end - -function generate_non_multi_test_request() - headers = [ - "User-Agent" => "PostmanRuntime/7.15.2", - "Accept" => "*/*", - "Cache-Control" => "no-cache", - "Postman-Token" => "288c2481-1837-4ba9-add3-f23d380fa440", - "Host" => "localhost:8888", - "Accept-Encoding" => "gzip, deflate", - "Accept-Encoding" => "gzip, deflate", - "Content-Length" => "0", - "Connection" => "keep-alive", - ] - - HTTP.Request("POST", "/", headers, Vector{UInt8}()) -end - -function generate_test_response() - body = generate_test_body() - - headers = [ - "Date" => "Fri, 25 Mar 2022 14:16:21 GMT", - "Transfer-Encoding" => "chunked", - "Content-Type" => "multipart/form-data; boundary=--------------------------918073721150061572809433", - "Content-Length" => string(length(body)), - "Connection" => "keep-alive", - ] - - HTTP.Response(200, headers, body=body) -end - - -@testset "parse multipart form-data" begin - @testset "find_multipart_boundary" begin - request = generate_test_request() - - # NOTE: this is the start of a "boundary delimiter line" and has two leading - # '-' characters prepended to the boundary delimiter from Content-Type header - delimiter = Vector{UInt8}("--------------------------918073721150061572809433") - body = generate_test_body() - # length of the delimiter, CRLF, and -1 for the end index to be the LF character - endIndexOffset = length(delimiter) + 4 - 1 - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter) - @test !isTerminatingDelimiter - @test 1 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - # the remaining "boundary delimiter lines" will have a CRLF preceding them - endIndexOffset += 2 - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 1) - @test !isTerminatingDelimiter - @test 175 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 3) - @test !isTerminatingDelimiter - @test 279 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 3) - @test !isTerminatingDelimiter - @test 396 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 3) - @test !isTerminatingDelimiter - @test 600 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 3) - @test !isTerminatingDelimiter - @test 804 == startIndex - @test (startIndex + endIndexOffset) == endIndex - - (isTerminatingDelimiter, startIndex, endIndex) = find_multipart_boundary(body, delimiter, start = startIndex + 3) - @test isTerminatingDelimiter - @test 1003 == startIndex - # +2 because of the two additional '--' characters - @test (startIndex + endIndexOffset + 2) == endIndex - end - - - @testset "parse_multipart_form request" begin - @test HTTP.parse_multipart_form(generate_non_multi_test_request()) === nothing - - multiparts = HTTP.parse_multipart_form(generate_test_request()) - @test 6 == length(multiparts) - - @test "multipart.txt" === multiparts[1].filename - @test "namevalue" === multiparts[1].name - @test "text/plain" === multiparts[1].contenttype - @test "not much to say\n" === String(read(multiparts[1].data)) - - @test multiparts[2].filename === nothing - @test "key1" === multiparts[2].name - @test "text/plain" === multiparts[2].contenttype - @test "1" === String(read(multiparts[2].data)) - - @test multiparts[3].filename === nothing - @test "key2" === multiparts[3].name - @test "text/plain" === multiparts[3].contenttype - @test "key the second" === String(read(multiparts[3].data)) - - @test "multipart-leading-newline.txt" === multiparts[4].filename - @test "namevalue2" === multiparts[4].name - @test "text/plain" === multiparts[4].contenttype - @test "\nfile with leading newline\n" === String(read(multiparts[4].data)) - - @test "my-json-file-1.json" === multiparts[5].filename - @test "json_file1" === multiparts[5].name - @test "application/json" === multiparts[5].contenttype - @test """{"data": ["this is json data"]}""" === String(read(multiparts[5].data)) - - @test multiparts[6].filename === nothing - @test "key3" === multiparts[6].name - @test "text/plain" === multiparts[6].contenttype - @test "This file has lower-cased content- keys, and disposition comes second." === String(read(multiparts[6].data)) - end - - @testset "parse_multipart_form response" begin - multiparts = HTTP.parse_multipart_form(generate_test_response()) - @test 6 == length(multiparts) - - @test "multipart.txt" === multiparts[1].filename - @test "namevalue" === multiparts[1].name - @test "text/plain" === multiparts[1].contenttype - @test "not much to say\n" === String(read(multiparts[1].data)) - - @test multiparts[2].filename === nothing - @test "key1" === multiparts[2].name - @test "text/plain" === multiparts[2].contenttype - @test "1" === String(read(multiparts[2].data)) - - @test multiparts[3].filename === nothing - @test "key2" === multiparts[3].name - @test "text/plain" === multiparts[3].contenttype - @test "key the second" === String(read(multiparts[3].data)) - - @test "multipart-leading-newline.txt" === multiparts[4].filename - @test "namevalue2" === multiparts[4].name - @test "text/plain" === multiparts[4].contenttype - @test "\nfile with leading newline\n" === String(read(multiparts[4].data)) - - @test "my-json-file-1.json" === multiparts[5].filename - @test "json_file1" === multiparts[5].name - @test "application/json" === multiparts[5].contenttype - @test """{"data": ["this is json data"]}""" === String(read(multiparts[5].data)) - - @test multiparts[6].filename === nothing - @test "key3" === multiparts[6].name - @test "text/plain" === multiparts[6].contenttype - @test "This file has lower-cased content- keys, and disposition comes second." === String(read(multiparts[6].data)) - end -end diff --git a/test/parser.jl b/test/parser.jl deleted file mode 100644 index dbb466c60..000000000 --- a/test/parser.jl +++ /dev/null @@ -1,457 +0,0 @@ -using Test, HTTP, HTTP.Messages, HTTP.Parsers, HTTP.Strings -include(joinpath(dirname(pathof(HTTP)), "../test/resources/HTTPMessages.jl")) -using .HTTPMessages - -import Base.== - -const strict = false - -==(a::Request,b::Request) = (a.method == b.method) && - (a.version == b.version) && - (a.headers == b.headers) && - (a.body == b.body) - -macro errmsg(expr) - esc(quote - try - $expr - catch e - sprint(show, e) - end - end) -end - -@testset "HTTP.parser" begin - @testset "parse - Strings" begin - @testset "Requests - $request" for request in requests - r = parse(Request, request.raw) - - if r.method == "CONNECT" - host, port = split(r.target, ":") - @test host == request.host - @test port == request.port - else - if r.target == "*" - @test r.target == request.request_path - else - target = parse(HTTP.URI, r.target) - @test target.query == request.query_string - @test target.fragment == request.fragment - @test target.path == request.request_path - @test target.host == request.host - @test target.userinfo == request.userinfo - @test target.port in (request.port, "80", "443") - @test string(target) == request.request_url - end - end - - r_headers = [tocameldash(n) => String(v) for (n,v) in r.headers] - - @test r.version.major == request.http_major - @test r.version.minor == request.http_minor - @test r.method == string(request.method) - @test length(r.headers) == request.num_headers - @test r_headers == request.headers - @test String(r.body) == request.body - - @test_broken HTTP.http_should_keep_alive(HTTP.DEFAULT_PARSER) == request.should_keep_alive - @test_broken String(collect(upgrade[])) == request.upgrade - end - - @testset "Request - Headers" begin - reqstr = "GET http://www.techcrunch.com/ HTTP/1.1\r\n" * - "Host: www.techcrunch.com\r\n" * - "User-Agent: Fake\r\n" * - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" * - "Accept-Language: en-us,en;q=0.5\r\n" * - "Accept-Encoding: gzip,deflate\r\n" * - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" * - "Keep-Alive: 300\r\n" * - "Content-Length: 7\r\n" * - "Proxy-Connection: keep-alive\r\n\r\n1234567" - req = Request("GET", "http://www.techcrunch.com/", ["Host"=>"www.techcrunch.com","User-Agent"=>"Fake","Accept"=>"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Language"=>"en-us,en;q=0.5","Accept-Encoding"=>"gzip,deflate","Accept-Charset"=>"ISO-8859-1,utf-8;q=0.7,*;q=0.7","Keep-Alive"=>"300","Content-Length"=>"7","Proxy-Connection"=>"keep-alive"]) - req.body = HTTP.bytes("1234567") - @test parse(Request,reqstr).headers == req.headers - @test parse(Request,reqstr) == req - end - - @testset "Request - Hostname - URL" begin - reqstr = "GET / HTTP/1.1\r\n" * - "Host: foo.com\r\n\r\n" - req = Request("GET", "/", ["Host"=>"foo.com"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - Hostname - Path" begin - reqstr = "GET //user@host/is/actually/a/path/ HTTP/1.1\r\n" * - "Host: test\r\n\r\n" - req = Request("GET", "//user@host/is/actually/a/path/", - ["Host"=>"test"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - Hostname - Path - ParseError" begin - reqstr = "GET HTTP/1.1\r\n" * - "Host: test\r\n\r\n" - @test_throws HTTP.ParseError parse(Request, reqstr) - end - - @testset "Request - Hostname - URL - ParseError" begin - reqstr = "GET ../../../../etc/passwd HTTP/1.1\r\n" * - "Host: test\r\n\r\n" - @test_throws HTTP.ParseError HTTP.URI(parse(Request, reqstr).target) - end - - @testset "Request - HTTP - Bytes" begin - reqstr = "POST / HTTP/1.1\r\n" * - "Host: foo.com\r\n" * - "Transfer-Encoding: chunked\r\n\r\n" * - "3\r\nfoo\r\n" * - "3\r\nbar\r\n" * - "0\r\n" * - "Trailer-Key: Trailer-Value\r\n" * - "\r\n" - req = Request("POST", "/", - ["Host"=>"foo.com", "Transfer-Encoding"=>"chunked", "Trailer-Key"=>"Trailer-Value"]) - req.body = HTTP.bytes("foobar") - @test parse(Request, reqstr) == req - end - - @test_skip @testset "Request - HTTP - ParseError" begin - reqstr = "POST / HTTP/1.1\r\n" * - "Host: foo.com\r\n" * - "Transfer-Encoding: chunked\r\n" * - "Content-Length: 9999\r\n\r\n" * # to be removed. - "3\r\nfoo\r\n" * - "3\r\nbar\r\n" * - "0\r\n" * - "\r\n" - - @test_throws HTTP.ParseError parse(Request, reqstr) - end - - @testset "Request - URL" begin - reqstr = "CONNECT www.google.com:443 HTTP/1.1\r\n\r\n" - req = Request("CONNECT", "www.google.com:443") - @test parse(Request, reqstr) == req - end - - @testset "Request - Localhost" begin - reqstr = "CONNECT 127.0.0.1:6060 HTTP/1.1\r\n\r\n" - req = Request("CONNECT", "127.0.0.1:6060") - @test parse(Request, reqstr) == req - end - - @testset "Request - RPC" begin - reqstr = "CONNECT /_goRPC_ HTTP/1.1\r\n\r\n" - req = HTTP.Request("CONNECT", "/_goRPC_") - @test parse(Request, reqstr) == req - end - - @testset "Request - NOTIFY" begin - reqstr = "NOTIFY * HTTP/1.1\r\nServer: foo\r\n\r\n" - req = Request("NOTIFY", "*", ["Server"=>"foo"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - OPTIONS" begin - reqstr = "OPTIONS * HTTP/1.1\r\nServer: foo\r\n\r\n" - req = Request("OPTIONS", "*", ["Server"=>"foo"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - GET" begin - reqstr = "GET / HTTP/1.1\r\nHost: issue8261.com\r\nConnection: close\r\n\r\n" - req = Request("GET", "/", ["Host"=>"issue8261.com", "Connection"=>"close"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - HEAD" begin - reqstr = "HEAD / HTTP/1.1\r\nHost: issue8261.com\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" - req = Request("HEAD", "/", ["Host"=>"issue8261.com", "Connection"=>"close", "Content-Length"=>"0"]) - @test parse(Request, reqstr) == req - end - - @testset "Request - POST" begin - reqstr = "POST /cgi-bin/process.cgi HTTP/1.1\r\n" * - "User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r\n" * - "Host: www.tutorialspoint.com\r\n" * - "Content-Type: text/xml; charset=utf-8\r\n" * - "Content-Length: 19\r\n" * - "Accept-Language: en-us\r\n" * - "Accept-Encoding: gzip, deflate\r\n" * - "Connection: Keep-Alive\r\n\r\n" * - "first=Zara&last=Ali\r\n\r\n" - req = Request("POST", "/cgi-bin/process.cgi", - ["User-Agent"=>"Mozilla/4.0 (compatible; MSIE5.01; Windows NT)", - "Host"=>"www.tutorialspoint.com", - "Content-Type"=>"text/xml; charset=utf-8", - "Content-Length"=>"19", - "Accept-Language"=>"en-us", - "Accept-Encoding"=>"gzip, deflate", - "Connection"=>"Keep-Alive"]) - req.body = HTTP.bytes("first=Zara&last=Ali") - @test parse(Request, reqstr) == req - end - - @testset "Request - Pair{}" begin - r = parse(Request,"GET / HTTP/1.1\r\n" * "Test: Düsseldorf\r\n\r\n") - @test Pair{String,String}[r.headers...] == ["Test" => "Düsseldorf"] - end - - @testset "Request - Methods - 1" begin - for m in ["GET", "PUT", "M-SEARCH", "FOOMETHOD"] - r = parse(Request,"$m / HTTP/1.1\r\n\r\n") - @test r.method == string(m) - end - end - - @testset "Request - Methods - 2" begin - for m in ("ASDF","C******","COLA","GEM","GETA","M****","MKCOLA","PROPPATCHA","PUN","PX","SA") - @test parse(Request,"$m / HTTP/1.1\r\n\r\n").method == m - end - end - - @testset "Request - HTTPS" begin - reqstr = "GET / HTTP/1.1\r\n" * - "X-SSL-FoooBarr: -----BEGIN CERTIFICATE-----\r\n" * - "\tMIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n" * - "\tETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n" * - "\tAkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n" * - "\tdWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n" * - "\tSzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n" * - "\tBAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n" * - "\tBQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n" * - "\tW51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n" * - "\tgW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n" * - "\t0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n" * - "\tu2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n" * - "\twgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n" * - "\tA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n" * - "\tBglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n" * - "\tVR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n" * - "\tloCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n" * - "\taWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n" * - "\t9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n" * - "\tIjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n" * - "\tBgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n" * - "\tcHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4QgEDBDAWLmh0\r\n" * - "\tdHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC5jcmwwPwYD\r\n" * - "\tVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n" * - "\tY3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n" * - "\tXCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n" * - "\tUO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n" * - "\thTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n" * - "\twTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n" * - "\tYhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n" * - "\tRA==\r\n" * - "\t-----END CERTIFICATE-----\r\n" * - "\r\n" - - r = parse(Request, reqstr) - @test r.method == "GET" - @test "GET / HTTP/1.1X-SSL-FoooBarr: $(header(r, "X-SSL-FoooBarr"))" == replace(reqstr, "\r\n" => "") - @test_throws HTTP.HTTP.ParseError HTTP.parse(Request, "GET / HTTP/1.1\r\nHost: www.example.com\r\nConnection\r\033\065\325eep-Alive\r\nAccept-Encoding: gzip\r\n\r\n") - - r = parse(Request,"GET /bad_get_no_headers_no_body/world HTTP/1.1\r\nAccept: */*\r\n\r\nHELLO") - @test String(r.body) == "" - end - - @testset "Response - readheaders() - 1" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Content-Length: " * "1844674407370955160" * "\r\n\r\n" - r = Response() - readheaders(IOBuffer(respstr), r) - @test r.status == 200 - @test [r.headers...] == ["Content-Length"=>"1844674407370955160"] - end - - @testset "Response - readheaders() - 2" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Transfer-Encoding: chunked\r\n\r\n" * "FFFFFFFFFFFFFFE" * "\r\n..." - r = Response() - readheaders(IOBuffer(respstr), r) - @test r.status == 200 - @test [r.headers...] == ["Transfer-Encoding"=>"chunked"] - end - end - - @testset "parse - $response" for response in responses - r = parse(Response, response.raw) - r_headers = [tocameldash(n) => String(v) for (n,v) in r.headers] - - @test r.version.major == response.http_major - @test r.version.minor == response.http_minor - @test r.status == response.status_code - @test HTTP.StatusCodes.statustext(r.status) == response.response_status - @test length(r.headers) == response.num_headers - @test r_headers == response.headers - @test String(r.body) == response.body - - @test_skip @test HTTP.http_should_keep_alive(HTTP.DEFAULT_PARSER) == response.should_keep_alive - end - - @testset "Parse Errors" begin - @testset "Requests" begin - @testset "ArgumentError - Invalid Base 10 Digit" begin - reqstr = "GET / HTTP/1.1\r\n" * "Content-Length: 0\r\nContent-Length: 1\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, ArgumentError) - end - - @testset "EOFError - 1" begin - reqstr = "GET / HTTP/1.1\r\n" * "Transfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, EOFError) - end - - @testset "EOFError - 2" begin - reqstr = "GET / HTTP/1.1\r\nheader: value\nhdr: value\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, EOFError) - end - - @testset "Invalid Request Line" begin - reqstr = "GET / HTP/1.1\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_REQUEST_LINE - end - - @testset "Invalid Header Field - 1" begin - reqstr = "GET / HTTP/1.1\r\n" * "Fo@: Failure\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Header Field - 2" begin - reqstr = "GET / HTTP/1.1\r\n" * "Foo\01\test: Bar\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Header Field - 3" begin - reqstr = "GET / HTTP/1.1\r\n" * "Foo: 1\rBar: 1\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Header Field - 4" begin - reqstr = "GET / HTTP/1.1\r\n" * "name\r\n" * " : value\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Request Line" begin - for m in ("HTTP/1.1", "hello world") - reqstr = "$m / HTTP/1.1\r\n\r\n" - e = try parse(Request, reqstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_REQUEST_LINE - end - end - - @testset "Strict Headers - 1" begin - reqstr = "GET / HTTP/1.1\r\n" * "Foo: F\01ailure\r\n\r\n" - strict && @test_throws HTTP.ParseError parse(Request,reqstr) - - if !strict - r = HTTP.parse(HTTP.Messages.Request, reqstr) - @test r.method == "GET" - @test r.target == "/" - @test length(r.headers) == 1 - end - end - - @testset "Strict Headers - 2" begin - reqstr = "GET / HTTP/1.1\r\n" * "Foo: B\02ar\r\n\r\n" - strict && @test_throws HTTP.ParseError parse(Request, reqstr) - - if !strict - r = parse(HTTP.Messages.Request, reqstr) - @test r.method == "GET" - @test r.target == "/" - @test length(r.headers) == 1 - end - end - - # https://github.com/JuliaWeb/HTTP.jl/issues/796 - @testset "Latin-1 values in header" begin - reqstr = "GET / HTTP/1.1\r\n" * "link: ; rel=\"canonical\", ; version=\"vor\"; type=\"text/xml\"; rel=\"item\", ; version=\"vor\"; type=\"text/plain\"; rel=\"item\", ; version=\"tdm\"; rel=\"license\", ; title=\"Santiago Badia\"; rel=\"author\", ; title=\"Alberto F. Mart\xedn\"; rel=\"author\"\r\n\r\n" - r = parse(HTTP.Messages.Request, reqstr) - @test r.method == "GET" - @test r.target == "/" - @test length(r.headers) == 1 - @test r.headers[1][2] == "; rel=\"canonical\", ; version=\"vor\"; type=\"text/xml\"; rel=\"item\", ; version=\"vor\"; type=\"text/plain\"; rel=\"item\", ; version=\"tdm\"; rel=\"license\", ; title=\"Santiago Badia\"; rel=\"author\", ; title=\"Alberto F. Martín\"; rel=\"author\"" - end - end - - @testset "Responses" begin - @testset "ArgumentError - Invalid Base 10 Digit" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Content-Length: 0\r\nContent-Length: 1\r\n\r\n" - e = try parse(Response, respstr) catch e e end - @test isa(e, ArgumentError) - end - - @testset "Chunk Size Exceeds Limit" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Transfer-Encoding: chunked\r\n\r\n" * "FFFFFFFFFFFFFFF" * "\r\n..." - e = try parse(Response,respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code == :CHUNK_SIZE_EXCEEDS_LIMIT - end - - @testset "Chunk Size Exceeds Limit" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Transfer-Encoding: chunked\r\n\r\n" * "10000000000000000" * "\r\n..." - e = try parse(Response,respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code == :CHUNK_SIZE_EXCEEDS_LIMIT - end - - @testset "EOF Error" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Transfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n" - e = try parse(Response, respstr) catch e e end - @test isa(e, EOFError) - end - - @test_skip @testset "Invalid Content Length" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Content-Length: " * "18446744073709551615" * "\r\n\r\n" - e = try parse(Response,respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code == Parsers.HPE_INVALID_CONTENT_LENGTH - end - - @testset "Invalid Header Field - 1" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Fo@: Failure\r\n\r\n" - e = try parse(Response, respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Header Field - 2" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Foo\01\test: Bar\r\n\r\n" - e = try parse(Response, respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Invalid Header Field - 3" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Foo: 1\rBar: 1\r\n\r\n" - e = try parse(Response, respstr) catch e e end - @test isa(e, HTTP.ParseError) && e.code ==:INVALID_HEADER_FIELD - end - - @testset "Strict Headers - 1" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Foo: F\01ailure\r\n\r\n" - strict && @test_throws HTTP.ParseError parse(Response,respstr) - - if !strict - r = parse(HTTP.Messages.Response, respstr) - @test r.status == 200 - @test length(r.headers) == 1 - end - end - - @testset "Strict Headers - 2" begin - respstr = "HTTP/1.1 200 OK\r\n" * "Foo: B\02ar\r\n\r\n" - strict && @test_throws HTTP.ParseError parse(Response,respstr) - - if !strict - r = parse(HTTP.Messages.Response, respstr) - @test r.status == 200 - @test length(r.headers) == 1 - end - end - end - end -end diff --git a/test/resources/FooRouter.jl b/test/resources/FooRouter.jl deleted file mode 100644 index ce227f71f..000000000 --- a/test/resources/FooRouter.jl +++ /dev/null @@ -1,8 +0,0 @@ -module FooRouter - -using HTTP -const r = HTTP.Router() -f = HTTP.Handlers.RequestHandlerFunction((req) -> HTTP.Response(200)) -HTTP.@register(r, "/test", f) - -end # module diff --git a/test/resources/HTTPMessages.jl b/test/resources/HTTPMessages.jl deleted file mode 100644 index 635ffcbdf..000000000 --- a/test/resources/HTTPMessages.jl +++ /dev/null @@ -1,1361 +0,0 @@ -module HTTPMessages -export Message, requests, responses, Headers - -const Headers = Vector{Pair{String,String}} - -mutable struct Message - name::String - raw::String - method::String - status_code::Int - response_status::String - request_path::String - request_url::String - fragment::String - query_string::String - body::String - body_size::Int - host::String - userinfo::String - port::String - num_headers::Int - headers::Headers - should_keep_alive::Bool - upgrade::String - http_major::Int - http_minor::Int - - Message(name::String) = new(name, "", "GET", 200, "", "", "", "", "", "", 0, "", "", "", 0, Headers(), true, "", 1, 1) -end - -function Message(; name::String="", kwargs...) - m = Message(name) - for (k, v) in kwargs - try - setfield!(m, k, v) - catch e - error("error setting k=$k, v=$v") - end - end - return m -end - -#= * R E Q U E S T S * =# -const requests = Message[ -Message(name= "curl get" -,raw= "GET /test HTTP/1.1\r\n" * - "User-Agent: curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1\r\n" * - "Host:0.0.0.0=5000\r\n" * # missing space after colon - "Accept: */*\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/test" -,request_url= "/test" -,num_headers= 3 -,headers=[ - "User-Agent"=> "curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1" - , "Host"=> "0.0.0.0=5000" - , "Accept"=> "*/*" - ] -,body= "" -), Message(name= "firefox get" -,raw= "GET /favicon.ico HTTP/1.1\r\n" * - "Host: 0.0.0.0=5000\r\n" * - "User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0\r\n" * - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" * - "Accept-Language: en-us,en;q=0.5\r\n" * - "Accept-Encoding: gzip,deflate\r\n" * - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" * - "Keep-Alive: 300\r\n" * - "Connection: keep-alive\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/favicon.ico" -,request_url= "/favicon.ico" -,num_headers= 8 -,headers=[ - "Host"=> "0.0.0.0=5000" - , "User-Agent"=> "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0" - , "Accept"=> "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - , "Accept-Language"=> "en-us,en;q=0.5" - , "Accept-Encoding"=> "gzip,deflate" - , "Accept-Charset"=> "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - , "Keep-Alive"=> "300" - , "Connection"=> "keep-alive" -] -,body= "" -), Message(name= "abcdefgh" -,raw= "GET /abcdefgh HTTP/1.1\r\n" * - "aaaaaaaaaaaaa:++++++++++\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/abcdefgh" -,request_url= "/abcdefgh" -,num_headers= 1 -,headers=[ - "Aaaaaaaaaaaaa"=> "++++++++++" -] -,body= "" -), Message(name= "fragment in url" -,raw= "GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "page=1" -,fragment= "posts-17408" -,request_path= "/forums/1/topics/2375" -#= XXX request url does include fragment? =# -,request_url= "/forums/1/topics/2375?page=1#posts-17408" -,num_headers= 0 -,body= "" -), Message(name= "get no headers no body" -,raw= "GET /get_no_headers_no_body/world HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/get_no_headers_no_body/world" -,request_url= "/get_no_headers_no_body/world" -,num_headers= 0 -,body= "" -), Message(name= "get one header no body" -,raw= "GET /get_one_header_no_body HTTP/1.1\r\n" * - "Accept: */*\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/get_one_header_no_body" -,request_url= "/get_one_header_no_body" -,num_headers= 1 -,headers=[ - "Accept" => "*/*" -] -,body= "" -), Message(name= "get funky content length body hello" -,raw= "GET /get_funky_content_length_body_hello HTTP/1.0\r\n" * - "conTENT-Length: 5\r\n" * - "\r\n" * - "HELLO" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/get_funky_content_length_body_hello" -,request_url= "/get_funky_content_length_body_hello" -,num_headers= 1 -,headers=[ - "Content-Length" => "5" -] -,body= "HELLO" -), Message(name= "post identity body world" -,raw= "POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n" * - "Accept: */*\r\n" * - "Transfer-Encoding: identity\r\n" * - "Content-Length: 5\r\n" * - "\r\n" * - "World" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "q=search" -,fragment= "hey" -,request_path= "/post_identity_body_world" -,request_url= "/post_identity_body_world?q=search#hey" -,num_headers= 3 -,headers=[ - "Accept"=> "*/*" - , "Transfer-Encoding"=> "identity" - , "Content-Length"=> "5" -] -,body= "World" -), Message(name= "post - chunked body: all your base are belong to us" -,raw= "POST /post_chunked_all_your_base HTTP/1.1\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "1e\r\nall your base are belong to us\r\n" * - "0\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/post_chunked_all_your_base" -,request_url= "/post_chunked_all_your_base" -,num_headers= 1 -,headers=[ - "Transfer-Encoding" => "chunked" -] -,body= "all your base are belong to us" -), Message(name= "two chunks ; triple zero ending" -,raw= "POST /two_chunks_mult_zero_end HTTP/1.1\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "5\r\nhello\r\n" * - "6\r\n world\r\n" * - "000\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/two_chunks_mult_zero_end" -,request_url= "/two_chunks_mult_zero_end" -,num_headers= 1 -,headers=[ - "Transfer-Encoding"=> "chunked" -] -,body= "hello world" -), Message(name= "chunked with trailing headers. blech." -,raw= "POST /chunked_w_trailing_headers HTTP/1.1\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "5\r\nhello\r\n" * - "6\r\n world\r\n" * - "0\r\n" * - "Vary: *\r\n" * - "Content-Type: text/plain\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/chunked_w_trailing_headers" -,request_url= "/chunked_w_trailing_headers" -,num_headers= 3 -,headers=[ - "Transfer-Encoding"=> "chunked" - , "Vary"=> "*" - , "Content-Type"=> "text/plain" -] -,body= "hello world" -), Message(name= "with excessss after the length" -,raw= "POST /chunked_w_excessss_after_length HTTP/1.1\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "5; ihatew3;whattheheck=aretheseparametersfor\r\nhello\r\n" * - "6; blahblah; blah\r\n world\r\n" * - "0\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/chunked_w_excessss_after_length" -,request_url= "/chunked_w_excessss_after_length" -,num_headers= 1 -,headers=[ - "Transfer-Encoding"=> "chunked" -] -,body= "hello world" -), Message(name= "with quotes" -,raw= "GET /with_\"stupid\"_quotes?foo=\"bar\" HTTP/1.1\r\n\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "foo=\"bar\"" -,fragment= "" -,request_path= "/with_\"stupid\"_quotes" -,request_url= "/with_\"stupid\"_quotes?foo=\"bar\"" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name = "apachebench get" -,raw= "GET /test HTTP/1.0\r\n" * - "Host: 0.0.0.0:5000\r\n" * - "User-Agent: ApacheBench/2.3\r\n" * - "Accept: */*\r\n\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/test" -,request_url= "/test" -,num_headers= 3 -,headers=[ "Host"=> "0.0.0.0:5000" - , "User-Agent"=> "ApacheBench/2.3" - , "Accept"=> "*/*" - ] -,body= "" -), Message(name = "query url with question mark" -,raw= "GET /test.cgi?foo=bar?baz HTTP/1.1\r\n\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "foo=bar?baz" -,fragment= "" -,request_path= "/test.cgi" -,request_url= "/test.cgi?foo=bar?baz" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name = "newline prefix get" -,raw= "\r\nGET /test HTTP/1.1\r\n\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/test" -,request_url= "/test" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name = "upgrade request" -,raw= "GET /demo HTTP/1.1\r\n" * - "Host: example.com\r\n" * - "Connection: Upgrade\r\n" * - "Sec-WebSocket-Key2: 12998 5 Y3 1 .P00\r\n" * - "Sec-WebSocket-Protocol: sample\r\n" * - "Upgrade: WebSocket\r\n" * - "Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5\r\n" * - "Origin: http://example.com\r\n" * - "\r\n" * - "Hot diggity dogg" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/demo" -,request_url= "/demo" -,num_headers= 7 -,upgrade="Hot diggity dogg" -,headers=[ "Host"=> "example.com" - , "Connection"=> "Upgrade" - , "Sec-Websocket-Key2"=> "12998 5 Y3 1 .P00" - , "Sec-Websocket-Protocol"=> "sample" - , "Upgrade"=> "WebSocket" - , "Sec-Websocket-Key1"=> "4 @1 46546xW%0l 1 5" - , "Origin"=> "http://example.com" - ] -,body= "" -), Message(name = "connect request" -,raw= "CONNECT 0-home0.netscape.com:443 HTTP/1.0\r\n" * - "User-agent: Mozilla/1.1N\r\n" * - "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" * - "\r\n" * - "some data\r\n" * - "and yet even more data" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "CONNECT" -,query_string= "" -,fragment= "" -,request_path= "" -,host="0-home0.netscape.com" -,port="443" -,request_url= "0-home0.netscape.com:443" -,num_headers= 2 -,upgrade="some data\r\nand yet even more data" -,headers=[ "User-Agent"=> "Mozilla/1.1N" - , "Proxy-Authorization"=> "basic aGVsbG86d29ybGQ=" - ] -,body= "" -), Message(name= "report request" -,raw= "REPORT /test HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "REPORT" -,query_string= "" -,fragment= "" -,request_path= "/test" -,request_url= "/test" -,num_headers= 0 -,headers=Headers() -,body= "" -#= -), Message(name= "request with no http version" -,raw= "GET /\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 0 -,http_minor= 9 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 0 -,headers=Headers() -,body= "" -=# -), Message(name= "m-search request" -,raw= "M-SEARCH * HTTP/1.1\r\n" * - "HOST: 239.255.255.250:1900\r\n" * - "MAN: \"ssdp:discover\"\r\n" * - "ST: \"ssdp:all\"\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "M-SEARCH" -,query_string= "" -,fragment= "" -,request_path= "*" -,request_url= "*" -,num_headers= 3 -,headers=[ "Host"=> "239.255.255.250:1900" - , "Man"=> "\"ssdp:discover\"" - , "St"=> "\"ssdp:all\"" - ] -,body= "" -), Message(name= "host terminated by a query string" -,raw= "GET http://hypnotoad.org?hail=all HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "hail=all" -,fragment= "" -,request_path= "" -,request_url= "http://hypnotoad.org?hail=all" -,host= "hypnotoad.org" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name= "host:port terminated by a query string" -,raw= "GET http://hypnotoad.org:1234?hail=all HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "hail=all" -,fragment= "" -,request_path= "" -,request_url= "http://hypnotoad.org:1234?hail=all" -,host= "hypnotoad.org" -,port= "1234" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name= "host:port terminated by a space" -,raw= "GET http://hypnotoad.org:1234 HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "" -,request_url= "http://hypnotoad.org:1234" -,host= "hypnotoad.org" -,port= "1234" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name = "PATCH request" -,raw= "PATCH /file.txt HTTP/1.1\r\n" * - "Host: www.example.com\r\n" * - "Content-Type: application/example\r\n" * - "If-Match: \"e0023aa4e\"\r\n" * - "Content-Length: 10\r\n" * - "\r\n" * - "cccccccccc" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "PATCH" -,query_string= "" -,fragment= "" -,request_path= "/file.txt" -,request_url= "/file.txt" -,num_headers= 4 -,headers=[ "Host"=> "www.example.com" - , "Content-Type"=> "application/example" - , "If-Match"=> "\"e0023aa4e\"" - , "Content-Length"=> "10" - ] -,body= "cccccccccc" -), Message(name = "connect caps request" -,raw= "CONNECT HOME0.NETSCAPE.COM:443 HTTP/1.0\r\n" * - "User-agent: Mozilla/1.1N\r\n" * - "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "CONNECT" -,query_string= "" -,fragment= "" -,request_path= "" -,request_url= "HOME0.NETSCAPE.COM:443" -,host="HOME0.NETSCAPE.COM" -,port="443" -,num_headers= 2 -,upgrade="" -,headers=[ "User-Agent"=> "Mozilla/1.1N" - , "Proxy-Authorization"=> "basic aGVsbG86d29ybGQ=" - ] -,body= "" -), Message(name= "utf-8 path request" -,raw= "GET /δ¶/δt/pope?q=1#narf HTTP/1.1\r\n" * - "Host: github.com\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "q=1" -,fragment= "narf" -,request_path= "/δ¶/δt/pope" -,request_url= "/δ¶/δt/pope?q=1#narf" -,num_headers= 1 -,headers=["Host" => "github.com"] -,body= "" -), Message(name = "hostname underscore" -,raw= "CONNECT home_0.netscape.com:443 HTTP/1.0\r\n" * - "User-agent: Mozilla/1.1N\r\n" * - "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "CONNECT" -,query_string= "" -,fragment= "" -,request_path= "" -,request_url= "home_0.netscape.com:443" -,host="home_0.netscape.com" -,port="443" -,num_headers= 2 -,upgrade="" -,headers=[ "User-Agent"=> "Mozilla/1.1N" - , "Proxy-Authorization"=> "basic aGVsbG86d29ybGQ=" - ] -,body= "" -), Message(name = "eat CRLF between requests, no \"Connection: close\" header" -,raw= "POST / HTTP/1.1\r\n" * - "Host: www.example.com\r\n" * - "Content-Type: application/x-www-form-urlencoded\r\n" * - "Content-Length: 4\r\n" * - "\r\n" * - "q=42\r\n" #= note the trailing CRLF =# -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 3 -,upgrade= "" -,headers=[ "Host"=> "www.example.com" - , "Content-Type"=> "application/x-www-form-urlencoded" - , "Content-Length"=> "4" - ] -,body= "q=42" -), Message(name = "eat CRLF between requests even if \"Connection: close\" is set" -,raw= "POST / HTTP/1.1\r\n" * - "Host: www.example.com\r\n" * - "Content-Type: application/x-www-form-urlencoded\r\n" * - "Content-Length: 4\r\n" * - "Connection: close\r\n" * - "\r\n" * - "q=42\r\n" #= note the trailing CRLF =# -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,method= "POST" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 4 -,upgrade= "" -,headers=[ "Host"=> "www.example.com" - , "Content-Type"=> "application/x-www-form-urlencoded" - , "Content-Length"=> "4" - , "Connection"=> "close" - ] -,body= "q=42" -), Message(name = "PURGE request" -,raw= "PURGE /file.txt HTTP/1.1\r\n" * - "Host: www.example.com\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "PURGE" -,query_string= "" -,fragment= "" -,request_path= "/file.txt" -,request_url= "/file.txt" -,num_headers= 1 -,headers=[ "Host"=> "www.example.com" ] -,body= "" -), Message(name = "SEARCH request" -,raw= "SEARCH / HTTP/1.1\r\n" * - "Host: www.example.com\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "SEARCH" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 1 -,headers=[ "Host"=> "www.example.com"] -,body= "" -), Message(name= "host:port and basic_auth" -,raw= "GET http://a%12:b!&*\$@hypnotoad.org:1234/toto HTTP/1.1\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,fragment= "" -,request_path= "/toto" -,request_url= "http://a%12:b!&*\$@hypnotoad.org:1234/toto" -,host= "hypnotoad.org" -,userinfo= "a%12:b!&*\$" -,port= "1234" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name = "upgrade post request" -,raw= "POST /demo HTTP/1.1\r\n" * - "Host: example.com\r\n" * - "Connection: Upgrade\r\n" * - "Upgrade: HTTP/2.0\r\n" * - "Content-Length: 15\r\n" * - "\r\n" * - "sweet post body" * - "Hot diggity dogg" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "POST" -,request_path= "/demo" -,request_url= "/demo" -,num_headers= 4 -,upgrade="Hot diggity dogg" -,headers=[ "Host"=> "example.com" - , "Connection"=> "Upgrade" - , "Upgrade"=> "HTTP/2.0" - , "Content-Length"=> "15" - ] -,body= "sweet post body" -), Message(name = "connect with body request" -,raw= "CONNECT foo.bar.com:443 HTTP/1.0\r\n" * - "User-agent: Mozilla/1.1N\r\n" * - "Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n" * - "Content-Length: 10\r\n" * - "\r\n" * - "blarfcicle" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,method= "CONNECT" -,request_url= "foo.bar.com:443" -,host="foo.bar.com" -,port="443" -,num_headers= 3 -,upgrade="" -,headers=[ "User-Agent"=> "Mozilla/1.1N" - , "Proxy-Authorization"=> "basic aGVsbG86d29ybGQ=" - , "Content-Length"=> "10" - ] -,body= "blarfcicle" -), Message(name = "link request" -,raw= "LINK /images/my_dog.jpg HTTP/1.1\r\n" * - "Host: example.com\r\n" * - "Link: ; rel=\"tag\"\r\n" * - "Link: ; rel=\"tag\"\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "LINK" -,request_path= "/images/my_dog.jpg" -,request_url= "/images/my_dog.jpg" -,query_string= "" -,fragment= "" -,num_headers= 2 -,headers=[ "Host"=> "example.com" - , "Link"=> "; rel=\"tag\", ; rel=\"tag\"" - ] -,body= "" -), Message(name = "link request" -,raw= "UNLINK /images/my_dog.jpg HTTP/1.1\r\n" * - "Host: example.com\r\n" * - "Link: ; rel=\"tag\"\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "UNLINK" -,request_path= "/images/my_dog.jpg" -,request_url= "/images/my_dog.jpg" -,query_string= "" -,fragment= "" -,num_headers= 2 -,headers=[ "Host"=> "example.com" - , "Link"=> "; rel=\"tag\"" - ] -,body= "" -), Message(name = "multiple connection header values with folding" -,raw= "GET /demo HTTP/1.1\r\n" * - "Host: example.com\r\n" * - "Connection: Something,\r\n" * - " Upgrade, ,Keep-Alive\r\n" * - "Sec-WebSocket-Key2: 12998 5 Y3 1 .P00\r\n" * - "Sec-WebSocket-Protocol: sample\r\n" * - "Upgrade: WebSocket\r\n" * - "Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5\r\n" * - "Origin: http://example.com\r\n" * - "\r\n" * - "Hot diggity dogg" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/demo" -,request_url= "/demo" -,num_headers= 7 -,upgrade="Hot diggity dogg" -,headers=[ "Host"=> "example.com" - , "Connection"=> "Something, Upgrade, ,Keep-Alive" - , "Sec-Websocket-Key2"=> "12998 5 Y3 1 .P00" - , "Sec-Websocket-Protocol"=> "sample" - , "Upgrade"=> "WebSocket" - , "Sec-Websocket-Key1"=> "4 @1 46546xW%0l 1 5" - , "Origin"=> "http://example.com" - ] -,body= "" -), Message(name= "line folding in header value" -,raw= "GET / HTTP/1.1\r\n" * - "Line1: abc\r\n" * - "\tdef\r\n" * - " ghi\r\n" * - "\t\tjkl\r\n" * - " mno \r\n" * - "\t \tqrs\r\n" * - "Line2: \t line2\t\r\n" * - "Line3:\r\n" * - " line3\r\n" * - "Line4: \r\n" * - " \r\n" * - "Connection:\r\n" * - " close\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 5 -,headers=[ "Line1"=> "abc\tdef ghi\t\tjkl mno \t \tqrs" - , "Line2"=> "line2" - , "Line3"=> " line3" - , "Line4"=> " " - , "Connection"=> " close" - ] -,body= "" -), Message(name = "multiple connection header values with folding and lws" -,raw= "GET /demo HTTP/1.1\r\n" * - "Connection: keep-alive, upgrade\r\n" * - "Upgrade: WebSocket\r\n" * - "\r\n" * - "Hot diggity dogg" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/demo" -,request_url= "/demo" -,num_headers= 2 -,upgrade="Hot diggity dogg" -,headers=[ "Connection"=> "keep-alive, upgrade" - , "Upgrade"=> "WebSocket" - ] -,body= "" -), Message(name = "multiple connection header values with folding and lws" -,raw= "GET /demo HTTP/1.1\r\n" * - "Connection: keep-alive, \r\n upgrade\r\n" * - "Upgrade: WebSocket\r\n" * - "\r\n" * - "Hot diggity dogg" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/demo" -,request_url= "/demo" -,num_headers= 2 -,upgrade="Hot diggity dogg" -,headers=[ "Connection"=> "keep-alive, upgrade" - , "Upgrade"=> "WebSocket" - ] -,body= "" -), Message(name= "line folding in header value" -,raw= "GET / HTTP/1.1\n" * - "Line1: abc\n" * - "\tdef\n" * - " ghi\n" * - "\t\tjkl\n" * - " mno \n" * - "\t \tqrs\n" * - "Line2: \t line2\t\n" * - "Line3:\n" * - " line3\n" * - "Line4: \n" * - " \n" * - "Connection:\n" * - " close\n" * - "\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,method= "GET" -,query_string= "" -,fragment= "" -,request_path= "/" -,request_url= "/" -,num_headers= 5 -,headers=[ "Line1"=> "abc\tdef ghi\t\tjkl mno \t \tqrs" - , "Line2"=> "line2" - , "Line3"=> " line3" - , "Line4"=> " " - , "Connection"=> " close" - ] -,body= "" -) -] - -#= * R E S P O N S E S * =# -const responses = Message[ - Message(name= "google 301" -,raw= "HTTP/1.1 301 Moved Permanently\r\n" * - "Location: http://www.google.com/\r\n" * - "Content-Type: text/html; charset=UTF-8\r\n" * - "Date: Sun, 26 Apr 2009 11:11:49 GMT\r\n" * - "Expires: Tue, 26 May 2009 11:11:49 GMT\r\n" * - "X-\$PrototypeBI-Version: 1.6.0.3\r\n" * #= $ char in header field =# - "Cache-Control: public, max-age=2592000\r\n" * - "Server: gws\r\n" * - "Content-Length: 219 \r\n" * - "\r\n" * - "\n" * - "301 Moved\n" * - "

301 Moved

\n" * - "The document has moved\n" * - "here.\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 301 -,response_status= "Moved Permanently" -,num_headers= 8 -,headers=[ - "Location"=> "http://www.google.com/" - , "Content-Type"=> "text/html; charset=UTF-8" - , "Date"=> "Sun, 26 Apr 2009 11:11:49 GMT" - , "Expires"=> "Tue, 26 May 2009 11:11:49 GMT" - , "X-\$prototypebi-Version"=> "1.6.0.3" - , "Cache-Control"=> "public, max-age=2592000" - , "Server"=> "gws" - , "Content-Length"=> "219" -] -,body= "\n" * - "301 Moved\n" * - "

301 Moved

\n" * - "The document has moved\n" * - "here.\r\n" * - "\r\n" -), Message(name= "no content-length response" -,raw= "HTTP/1.1 200 OK\r\n" * - "Date: Tue, 04 Aug 2009 07:59:32 GMT\r\n" * - "Server: Apache\r\n" * - "X-Powered-By: Servlet/2.5 JSP/2.1\r\n" * - "Content-Type: text/xml; charset=utf-8\r\n" * - "Connection: close\r\n" * - "\r\n" * - "\n" * - "\n" * - " \n" * - " \n" * - " SOAP-ENV:Client\n" * - " Client Error\n" * - " \n" * - " \n" * - "" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 5 -,headers=[ - "Date"=> "Tue, 04 Aug 2009 07:59:32 GMT" - , "Server"=> "Apache" - , "X-Powered-By"=> "Servlet/2.5 JSP/2.1" - , "Content-Type"=> "text/xml; charset=utf-8" - , "Connection"=> "close" -] -,body= "\n" * - "\n" * - " \n" * - " \n" * - " SOAP-ENV:Client\n" * - " Client Error\n" * - " \n" * - " \n" * - "" -), Message(name= "404 no headers no body" -,raw= "HTTP/1.1 404 Not Found\r\n\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 404 -,response_status= "Not Found" -,num_headers= 0 -,headers=Headers() -,body_size= 0 -,body= "" -), Message(name= "301 no response phrase" -,raw= "HTTP/1.1 301\r\n\r\n" -,should_keep_alive = false -,http_major= 1 -,http_minor= 1 -,status_code= 301 -,response_status= "Moved Permanently" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name="200 trailing space on chunked body" -,raw= "HTTP/1.1 200 OK\r\n" * - "Content-Type: text/plain\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "25 \r\n" * - "This is the data in the first chunk\r\n" * - "\r\n" * - "1C\r\n" * - "and this is the second one\r\n" * - "\r\n" * - "0 \r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 2 -,headers=[ - "Content-Type"=> "text/plain" - , "Transfer-Encoding"=> "chunked" -] -,body_size = 37+28 -,body = - "This is the data in the first chunk\r\n" * - "and this is the second one\r\n" -), Message(name="no carriage ret" -,raw= "HTTP/1.1 200 OK\n" * - "Content-Type: text/html; charset=utf-8\n" * - "Connection: close\n" * - "\n" * - "these headers are from http://news.ycombinator.com/" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 2 -,headers=[ - "Content-Type"=> "text/html; charset=utf-8" - , "Connection"=> "close" -] -,body= "these headers are from http://news.ycombinator.com/" -), Message(name="proxy connection" -,raw= "HTTP/1.1 200 OK\r\n" * - "Content-Type: text/html; charset=UTF-8\r\n" * - "Content-Length: 11\r\n" * - "Proxy-Connection: close\r\n" * - "Date: Thu, 31 Dec 2009 20:55:48 +0000\r\n" * - "\r\n" * - "hello world" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 4 -,headers=[ - "Content-Type"=> "text/html; charset=UTF-8" - , "Content-Length"=> "11" - , "Proxy-Connection"=> "close" - , "Date"=> "Thu, 31 Dec 2009 20:55:48 +0000" -] -,body= "hello world" -), Message(name="underscore header key" -,raw= "HTTP/1.1 200 OK\r\n" * - "Server: DCLK-AdSvr\r\n" * - "Content-Type: text/xml\r\n" * - "Content-Length: 0\r\n" * - "DCLK_imp: v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o\r\n\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 4 -,headers=[ - "Server"=> "DCLK-AdSvr" - , "Content-Type"=> "text/xml" - , "Content-Length"=> "0" - , "Dclk_imp"=> "v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o" -] -,body= "" -), Message(name= "bonjourmadame.fr" -,raw= "HTTP/1.0 301 Moved Permanently\r\n" * - "Date: Thu, 03 Jun 2010 09:56:32 GMT\r\n" * - "Server: Apache/2.2.3 (Red Hat)\r\n" * - "Cache-Control: public\r\n" * - "Pragma: \r\n" * - "Location: http://www.bonjourmadame.fr/\r\n" * - "Vary: Accept-Encoding\r\n" * - "Content-Length: 0\r\n" * - "Content-Type: text/html; charset=UTF-8\r\n" * - "Connection: keep-alive\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 0 -,status_code= 301 -,response_status= "Moved Permanently" -,num_headers= 9 -,headers=[ - "Date"=> "Thu, 03 Jun 2010 09:56:32 GMT" - , "Server"=> "Apache/2.2.3 (Red Hat)" - , "Cache-Control"=> "public" - , "Pragma"=> "" - , "Location"=> "http://www.bonjourmadame.fr/" - , "Vary"=> "Accept-Encoding" - , "Content-Length"=> "0" - , "Content-Type"=> "text/html; charset=UTF-8" - , "Connection"=> "keep-alive" -] -,body= "" -), Message(name= "field underscore" -,raw= "HTTP/1.1 200 OK\r\n" * - "Date: Tue, 28 Sep 2010 01:14:13 GMT\r\n" * - "Server: Apache\r\n" * - "Cache-Control: no-cache, must-revalidate\r\n" * - "Expires: Mon, 26 Jul 1997 05:00:00 GMT\r\n" * - ".et-Cookie: PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com\r\n" * - "Vary: Accept-Encoding\r\n" * - "_eep-Alive: timeout=45\r\n" * #= semantic value ignored =# - "_onnection: Keep-Alive\r\n" * #= semantic value ignored =# - "Transfer-Encoding: chunked\r\n" * - "Content-Type: text/html\r\n" * - "Connection: close\r\n" * - "\r\n" * - "0\r\n\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 11 -,headers=[ - "Date"=> "Tue, 28 Sep 2010 01:14:13 GMT" - , "Server"=> "Apache" - , "Cache-Control"=> "no-cache, must-revalidate" - , "Expires"=> "Mon, 26 Jul 1997 05:00:00 GMT" - , ".et-Cookie"=> "PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com" - , "Vary"=> "Accept-Encoding" - , "_eep-Alive"=> "timeout=45" - , "_onnection"=> "Keep-Alive" - , "Transfer-Encoding"=> "chunked" - , "Content-Type"=> "text/html" - , "Connection"=> "close" -] -,body= "" -), Message(name= "non-ASCII in status line" -,raw= "HTTP/1.1 500 Oriëntatieprobleem\r\n" * - "Date: Fri, 5 Nov 2010 23:07:12 GMT+2\r\n" * - "Content-Length: 0\r\n" * - "Connection: close\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 500 -,response_status= "Internal Server Error" -,num_headers= 3 -,headers=[ - "Date"=> "Fri, 5 Nov 2010 23:07:12 GMT+2" - , "Content-Length"=> "0" - , "Connection"=> "close" -] -,body= "" -), Message(name= "http version 0.9" -,raw= "HTTP/0.9 200 OK\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 0 -,http_minor= 9 -,status_code= 200 -,response_status= "OK" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name= "neither content-length nor transfer-encoding response" -,raw= "HTTP/1.1 200 OK\r\n" * - "Content-Type: text/plain\r\n" * - "\r\n" * - "hello world" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 1 -,headers=[ - "Content-Type"=> "text/plain" -] -,body= "hello world" -), Message(name= "HTTP/1.0 with keep-alive and EOF-terminated 200 status" -,raw= "HTTP/1.0 200 OK\r\n" * - "Connection: keep-alive\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 0 -,status_code= 200 -,response_status= "OK" -,num_headers= 1 -,headers=[ - "Connection"=> "keep-alive" -] -,body_size= 0 -,body= "" -), Message(name= "HTTP/1.0 with keep-alive and a 204 status" -,raw= "HTTP/1.0 204 No content\r\n" * - "Connection: keep-alive\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 0 -,status_code= 204 -,response_status= "No Content" -,num_headers= 1 -,headers=[ - "Connection"=> "keep-alive" -] -,body_size= 0 -,body= "" -), Message(name= "HTTP/1.1 with an EOF-terminated 200 status" -,raw= "HTTP/1.1 200 OK\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 0 -,headers=Headers() -,body_size= 0 -,body= "" -), Message(name= "HTTP/1.1 with a 204 status" -,raw= "HTTP/1.1 204 No content\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 204 -,response_status= "No Content" -,num_headers= 0 -,headers=Headers() -,body_size= 0 -,body= "" -), Message(name= "HTTP/1.1 with a 204 status and keep-alive disabled" -,raw= "HTTP/1.1 204 No content\r\n" * - "Connection: close\r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 204 -,response_status= "No Content" -,num_headers= 1 -,headers=[ - "Connection"=> "close" -] -,body_size= 0 -,body= "" -), Message(name= "HTTP/1.1 with chunked endocing and a 200 response" -,raw= "HTTP/1.1 200 OK\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "0\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 1 -,headers=[ - "Transfer-Encoding"=> "chunked" -] -,body_size= 0 -,body= "" -#= -No reference to source of this was provided when requested here: -https://github.com/nodejs/http-parser/pull/64#issuecomment-2042429 -), Message(name= "field space" -,raw= "HTTP/1.1 200 OK\r\n" * - "Server: Microsoft-IIS/6.0\r\n" * - "X-Powered-By: ASP.NET\r\n" * - "en-US Content-Type: text/xml\r\n" * #= this is the problem =# - "Content-Type: text/xml\r\n" * - "Content-Length: 16\r\n" * - "Date: Fri, 23 Jul 2010 18:45:38 GMT\r\n" * - "Connection: keep-alive\r\n" * - "\r\n" * - "hello" #= fake body =# -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 7 -,headers=[ - "Server"=> "Microsoft-IIS/6.0" - , "X-Powered-By"=> "ASP.NET" - , "En-Us content-Type"=> "text/xml" - , "Content-Type"=> "text/xml" - , "Content-Length"=> "16" - , "Date"=> "Fri, 23 Jul 2010 18:45:38 GMT" - , "Connection"=> "keep-alive" -] -,body= "hello" -=# -), Message(name= "amazon.com" -,raw= "HTTP/1.1 301 MovedPermanently\r\n" * - "Date: Wed, 15 May 2013 17:06:33 GMT\r\n" * - "Server: Server\r\n" * - "x-amz-id-1: 0GPHKXSJQ826RK7GZEB2\r\n" * - "p3p: policyref=\"http://www.amazon.com/w3c/p3p.xml\",CP=\"CAO DSP LAW CUR ADM IVAo IVDo CONo OTPo OUR DELi PUBi OTRi BUS PHY ONL UNI PUR FIN COM NAV INT DEM CNT STA HEA PRE LOC GOV OTC \"\r\n" * - "x-amz-id-2: STN69VZxIFSz9YJLbz1GDbxpbjG6Qjmmq5E3DxRhOUw+Et0p4hr7c/Q8qNcx4oAD\r\n" * - "Location: http://www.amazon.com/Dan-Brown/e/B000AP9DSU/ref=s9_pop_gw_al1?_encoding=UTF8&refinementId=618073011&pf_rd_m=ATVPDKIKX0DER&pf_rd_s=center-2&pf_rd_r=0SHYY5BZXN3KR20BNFAY&pf_rd_t=101&pf_rd_p=1263340922&pf_rd_i=507846\r\n" * - "Vary: Accept-Encoding,User-Agent\r\n" * - "Content-Type: text/html; charset=ISO-8859-1\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "1\r\n" * - "\n\r\n" * - "0\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 301 -,response_status= "Moved Permanently" -,num_headers= 9 -,headers=[ "Date"=> "Wed, 15 May 2013 17:06:33 GMT" - , "Server"=> "Server" - , "X-Amz-Id-1"=> "0GPHKXSJQ826RK7GZEB2" - , "P3p"=> "policyref=\"http://www.amazon.com/w3c/p3p.xml\",CP=\"CAO DSP LAW CUR ADM IVAo IVDo CONo OTPo OUR DELi PUBi OTRi BUS PHY ONL UNI PUR FIN COM NAV INT DEM CNT STA HEA PRE LOC GOV OTC \"" - , "X-Amz-Id-2"=> "STN69VZxIFSz9YJLbz1GDbxpbjG6Qjmmq5E3DxRhOUw+Et0p4hr7c/Q8qNcx4oAD" - , "Location"=> "http://www.amazon.com/Dan-Brown/e/B000AP9DSU/ref=s9_pop_gw_al1?_encoding=UTF8&refinementId=618073011&pf_rd_m=ATVPDKIKX0DER&pf_rd_s=center-2&pf_rd_r=0SHYY5BZXN3KR20BNFAY&pf_rd_t=101&pf_rd_p=1263340922&pf_rd_i=507846" - , "Vary"=> "Accept-Encoding,User-Agent" - , "Content-Type"=> "text/html; charset=ISO-8859-1" - , "Transfer-Encoding"=> "chunked" - ] -,body= "\n" -), Message(name= "empty reason phrase after space" -,raw= "HTTP/1.1 200 \r\n" * - "\r\n" -,should_keep_alive= false -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 0 -,headers=Headers() -,body= "" -), Message(name= "Content-Length-X" -,raw= "HTTP/1.1 200 OK\r\n" * - "Content-Length-X: 0\r\n" * - "Transfer-Encoding: chunked\r\n" * - "\r\n" * - "2\r\n" * - "OK\r\n" * - "0\r\n" * - "\r\n" -,should_keep_alive= true -,http_major= 1 -,http_minor= 1 -,status_code= 200 -,response_status= "OK" -,num_headers= 2 -,headers=[ "Content-Length-X"=> "0" - , "Transfer-Encoding"=> "chunked" - ] -,body= "OK" -) -] - -end # module HTTPMessages \ No newline at end of file diff --git a/test/resources/TestRequest.jl b/test/resources/TestRequest.jl deleted file mode 100644 index 01caa71b8..000000000 --- a/test/resources/TestRequest.jl +++ /dev/null @@ -1,79 +0,0 @@ -module TestRequest - -using HTTP - -function testrequestlayer(handler) - return function(req; httptestlayer=Ref(false), kw...) - httptestlayer[] = true - return handler(req; kw...) - end -end - -function teststreamlayer(handler) - return function(stream; httptestlayer=Ref(false), kw...) - httptestlayer[] = true - return handler(stream; kw...) - end -end - -HTTP.@client (testrequestlayer,) (teststreamlayer,) - -end - -module TestRequest2 - -using HTTP, Test - -function testouterrequestlayer(handler) - return function(req; check=false, kw...) - @test !check - return handler(req; check=true, kw...) - end -end - -function testinnerrequestlayer(handler) - return function(req; check=false, kw...) - @test check - return handler(req; kw...) - end -end - -function testouterstreamlayer(handler) - return function(req; check=false, kw...) - @test !check - return handler(req; check=true, kw...) - end -end - -function testinnerstreamlayer(handler) - return function(req; check=false, kw...) - @test check - return handler(req; kw...) - end -end - -HTTP.@client (first=[testouterrequestlayer], last=[testinnerrequestlayer]) (first=[testouterstreamlayer], last=[testinnerstreamlayer]) - -end - -module ErrorRequest - -using HTTP - -function throwingrequestlayer(handler) - return function(req; request_exception=nothing, kw...) - !isnothing(request_exception) && throw(request_exception) - return handler(req; kw...) - end -end - -function throwingstreamlayer(handler) - return function(stream; stream_exception=nothing, kw...) - !isnothing(stream_exception) && throw(stream_exception) - return handler(stream; kw...) - end -end - -HTTP.@client (throwingrequestlayer,) (throwingstreamlayer,) - -end diff --git a/test/resources/cert.pem b/test/resources/cert.pem deleted file mode 100644 index 85029c67b..000000000 --- a/test/resources/cert.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJAJjLegbzdgUPMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAeFw0xNTA5MjUyMjAyMzlaFw00MzAyMTAyMjAyMzlaMBQx -EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBALxC4OgxUv2v4zJCxChb6c9Qr+kgx5Y3XMsJ5pBcrPyMAyJ8aHAy8HvkUoWi -EmGsghamZujpovTctwUP4CHqDW0NPv5ewmpZNlLJZlpUufeAQ0FIVrWzUL0qKYyB -lilkCoCCpNyTKI9D1QtLOk/EvqD/P8ZoFvDFvgQW/LZSxNj5TWa81rjOT6Ral2b7 -Xdsc6eaJq/aCnt6T9z20NXderh9nFCmkAN98LD12QtV2CP8RCSD8vYqN1KnOLjUr -xtLe3K0bjQF8xTKajsT3ppRV00uWG+6glJVfsqCw+LsCsKWpDmLxooo6b+zC6V61 -c0XLNYYkB+gRKCV9zjXASP+g9hMCAwEAAaNQME4wHQYDVR0OBBYEFJ2Qo9xfnsyo -rbl/Q/28kOsd0D8wMB8GA1UdIwQYMBaAFJ2Qo9xfnsyorbl/Q/28kOsd0D8wMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAD7jtxExdnynxDrPqYHdozAP -WgD3WYgNvygEU7WySz67UECPPjhyH/q/3hMFqntDANYCwFBrQ59zXIrYHDuZ1G5b -BlyZsmKdWaOokpkmAEYZUglj1AHwF5C8XPCuAk7/oP0VYx27T5jEsqTTfpsKVImP -c4HBbdXoy07LRzDRzo/zE8KxXW9Z7divV87AxXbTA7VwBV75uhFfQ58gpuTA4grk -kvF00zTT5P2/hCVKTDhiiLtIZz9c9W0KLT8cJN5zm4S8yPNZu8K5TI1XUTjcrNQM -bcSW0BksGgwB8kH+S5W+eyPsr0y+3a7Omg9S8sMza0Pc6P9GduqN8Q0S+LYRZO4= ------END CERTIFICATE----- diff --git a/test/resources/key.pem b/test/resources/key.pem deleted file mode 100644 index 8387ce8bf..000000000 --- a/test/resources/key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8QuDoMVL9r+My -QsQoW+nPUK/pIMeWN1zLCeaQXKz8jAMifGhwMvB75FKFohJhrIIWpmbo6aL03LcF -D+Ah6g1tDT7+XsJqWTZSyWZaVLn3gENBSFa1s1C9KimMgZYpZAqAgqTckyiPQ9UL -SzpPxL6g/z/GaBbwxb4EFvy2UsTY+U1mvNa4zk+kWpdm+13bHOnmiav2gp7ek/c9 -tDV3Xq4fZxQppADffCw9dkLVdgj/EQkg/L2KjdSpzi41K8bS3tytG40BfMUymo7E -96aUVdNLlhvuoJSVX7KgsPi7ArClqQ5i8aKKOm/swuletXNFyzWGJAfoESglfc41 -wEj/oPYTAgMBAAECggEBAIB6y+7qqo7DWLRWaHR6tchscoERg+R6h/NxIE7pUI1S -KFmCuevId+K1YbQddZn/FxDKI3VU7YdakfT8bqP2jY8c+R60IM5fb/lzxUxkgj3s -5PlKmxKJ+9H9Ujm3vnkk8x3dCxIVxBpx2pVIk9UYmlhZmnaXVwCekx1LatArEHha -D8FGhxHfFKnIVuL4A579+lyGNsHXJl/B7yJEudwFcdT1ZnwkRLidWvLRHkoxnH7m -PsghdjHLNTYaOgYHCVqntr/LInGgZVE7+jxqWJGqTCmTg0MXvabg37QxXSoDgPWa -J8u+64qE+GIKaQ2sOeuNMGR2eB3Hw2h6fsJ33HhzI4ECgYEA4RFP2aJzj1J8jst9 -evSfZcfM7mXev8DReA+wmlcnUjWn0Bz1b7pPUYHDZrTPb9ONTPaB+EUGCDhuPI2b -W0Mk0iPqkfZAssXQBDQIHZnWim/hRoqdkzPe4KWA8hjgKATLrqfcc1MAvhaHOjqe -gAmBykN7WaFRMA/j+cr8FVFo+aECgYEA1iKV+uYIqIbowY6NChc8aBIkSXy7xcz5 -HntR2UgI2UiYCdppZXfQayRlng1CYhTvmFJcd528mZ64sg/1hyD0bT0oZ7Xgn8Tl -IBdEvpZFAafW9xO/akshvt6HSEPrFoZnOlr3MjTr52TRg5JGmM06evfj9sWWsbFD -qFDbckMbWzMCgYATLviRYklbQ/qd6TZOzp7ve/I5t7Eewv6Xry6sWRVe6nfdQzqg -RU8RcXAIRw0PSQbYMoKteKSk+rpaqu88/iIbTzhlLIojMr0iPpUagMxKjHK1Iod/ -zoIGv9SXzgr9HjuGLYSax85eZWktS2XLIARSCyJuZ1OWNySFXAnUf1XlQQKBgERw -VWMVNls2kxmZx/YbqxDQC4z5MsJrWoulemlpnnpju0Qa7GijvJch0OCM+FSEwHb8 -i9UnMuoeUoWGmECSBc0MKOfMt3gY4+o3xZ7sRC3dSNU7GIiObsCkOrScEHzohAGg -pTUEuQkBrfzROYMIxNIcfF2YlStBrpATF7ATRqEFAoGBAI/C2xjcr58U4lyp8Uuq -IYvC0dyz2nPe30wTBtWI0nKEuTGvB0A0xXfkW+OeyDQ2Wmt3OV6hq89EI4kmFxUn -k1LW8w4qOW9lwXnWBEFF0H38EZ8L5Y9Gv9DO3kJPN32ypSfQL7kl0mX7gUaoHskY -jQp1r0nwNJ6zfYXOaj1jr5lz ------END PRIVATE KEY----- diff --git a/test/resources/mwe.html b/test/resources/mwe.html deleted file mode 100644 index c03ae6f01..000000000 --- a/test/resources/mwe.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - -

?

- - - - - - - diff --git a/test/runtests.jl b/test/runtests.jl index d8cd6c2c7..6e0adf105 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,45 +1,11 @@ -using Test, HTTP, JSON +using Test, HTTP, URIs, JSON -# Using this rather than @__DIR__ because then it's easier to run parts of the -# file at the REPL, which is convenient when developing the package. -const dir = joinpath(dirname(pathof(HTTP)), "..", "test") - -# See https://httpbingo.julialang.org/ const httpbin = get(ENV, "JULIA_TEST_HTTPBINGO_SERVER", "httpbingo.julialang.org") - -# A convenient test helper used in a few test files. isok(r) = r.status == 200 -@testset "HTTP" begin - testfiles = [ - "ascii.jl", - "chunking.jl", - "utils.jl", - "client.jl", - # "download.jl", - "multipart.jl", - "parsemultipart.jl", - "sniff.jl", - "cookies.jl", - "parser.jl", - "loopback.jl", - "websockets/deno_client/server.jl", - "messages.jl", - "handlers.jl", - "server.jl", - "async.jl", - "mwe.jl", - "httpversion.jl", - "websockets/autobahn.jl", - "websockets/multiple_writers.jl", - ] - # ARGS can be most easily passed like this: - # import Pkg; Pkg.test("HTTP"; test_args=`ascii.jl parser.jl`) - if !isempty(ARGS) - filter!(in(ARGS), testfiles) - end - for filename in testfiles - println("Running $filename tests...") - include(joinpath(dir, filename)) - end -end +include("utils.jl") +include("sniff.jl") +include("multipart.jl") +include("client.jl") +include("handlers.jl") +include("server.jl") diff --git a/test/server.jl b/test/server.jl index af1a98741..535c314bd 100644 --- a/test/server.jl +++ b/test/server.jl @@ -1,273 +1,51 @@ -module test_server +using Test, HTTP, Logging -import ..httpbin -using HTTP, HTTP.IOExtras, Sockets, Test, MbedTLS - -function testget(url, m=1) - r = [] - @sync for i in 1:m - l = rand([0,0,10,1000,10000]) - body = Vector{UInt8}(rand('A':'Z', l)) - # println("sending request...") - @async push!(r, HTTP.request("GET", "$url/$i", [], body)) - end - return r -end - -const echohandler = req -> HTTP.Response(200, req.body) -const echostreamhandler = HTTP.streamhandler(echohandler) - -@testset "HTTP.listen" begin - server = HTTP.listen!(echostreamhandler; listenany=true) - port = HTTP.port(server) - r = testget("http://127.0.0.1:$port") - @test r[1].status == 200 - close(server) - sleep(0.5) - @test istaskdone(server.task) - - server = HTTP.listen!(echostreamhandler; listenany=true) - port = HTTP.port(server) - server2 = HTTP.serve!(echohandler; listenany=true) - port2 = HTTP.port(server2) - - r = testget("http://127.0.0.1:$port") - @test r[1].status == 200 - - r = testget("http://127.0.0.1:$(port2)") - @test r[1].status == 200 - - rs = testget("http://127.0.0.1:$port/", 20) - foreach(rs) do r - @test r.status == 200 - end - - r = HTTP.get("http://127.0.0.1:$port/"; readtimeout=30) - @test r.status == 200 - @test String(r.body) == "" - - # large headers - tcp = Sockets.connect(ip"127.0.0.1", port) - x = "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n"; - write(tcp, "GET / HTTP/1.1\r\n$(repeat("Foo: Bar\r\n", 10000))\r\n") - sleep(0.1) - try - resp = String(readavailable(tcp)) - @test occursin(r"HTTP/1.1 431 Request Header Fields Too Large", resp) - catch - println("Failed reading bad request response") - end - - # invalid HTTP - tcp = Sockets.connect(ip"127.0.0.1", port) - write(tcp, "GET / HTP/1.1\r\n\r\n") - sleep(0.1) - try - resp = String(readavailable(tcp)) - @test occursin(r"HTTP/1.1 400 Bad Request", resp) - catch - println("Failed reading bad request response") - end - - # no URL - tcp = Sockets.connect(ip"127.0.0.1", port) - write(tcp, "SOMEMETHOD HTTP/1.1\r\nContent-Length: 0\r\n\r\n") - sleep(0.1) - try - resp = String(readavailable(tcp)) - @test occursin(r"HTTP/1.1 400 Bad Request", resp) - catch - println("Failed reading bad request response") - end - - # Expect: 100-continue - tcp = Sockets.connect(ip"127.0.0.1", port) - write(tcp, "POST / HTTP/1.1\r\nContent-Length: 15\r\nExpect: 100-continue\r\n\r\n") - sleep(0.1) - client = String(readavailable(tcp)) - @test client == "HTTP/1.1 100 Continue\r\n\r\n" - - write(tcp, "Body of Request") - sleep(0.1) - client = String(readavailable(tcp)) - - println("client:") - println(client) - @test occursin("HTTP/1.1 200 OK\r\n", client) - @test occursin("Transfer-Encoding: chunked\r\n", client) - @test occursin("Body of Request", client) - - close(server) - close(server2) - - # keep-alive vs. close: issue #81 - hello = HTTP.streamhandler(req -> HTTP.Response("Hello")) - server = HTTP.listen!(hello; listenany=true, verbose=true) - port = HTTP.port(server) - tcp = Sockets.connect(ip"127.0.0.1", port) - write(tcp, "GET / HTTP/1.0\r\n\r\n") - sleep(0.5) +@testset "HTTP.serve" begin + server = HTTP.serve!(req -> HTTP.Response(200, "Hello, World!"); listenany=true) try - resp = String(readavailable(tcp)) - @test resp == "HTTP/1.1 200 OK\r\n\r\nHello" - catch - println("Failed reading bad request response") - end - close(server) - - # SO_REUSEPORT - if HTTP.Servers.supportsreuseaddr() - println("Testing server port reuse") - t1 = HTTP.listen!(hello; listeany=true, reuseaddr=true) - port = HTTP.port(t1) - println("Starting second server listening on same port") - t2 = HTTP.listen!(hello, port; reuseaddr=true) - println("Starting server on same port without port reuse (throws error)") - try - HTTP.listen(hello, port) - catch e - @test e isa Base.IOError - @test startswith(e.msg, "listen") - @test e.code == Base.UV_EADDRINUSE - end - close(t1) - close(t2) - end - - # test automatic forwarding of non-sensitive headers - # this is a server that will "echo" whatever headers were sent to it - t1 = HTTP.listen!(; listenany=true) do http - request::HTTP.Request = http.message - request.body = read(http) - closeread(http) - request.response::HTTP.Response = HTTP.Response(200, request.headers) - request.response.request = request - startwrite(http) - write(http, request.response.body) - end - port = HTTP.port(t1) - - # test that an Authorization header is **not** forwarded to a domain different than initial request - @test !HTTP.hasheader(HTTP.get("https://$httpbin/redirect-to?url=http://127.0.0.1:$port", ["Authorization"=>"auth"]), "Authorization") - - # test that an Authorization header **is** forwarded to redirect in same domain - @test HTTP.hasheader(HTTP.get("https://$httpbin/redirect-to?url=https://$httpbin/response-headers?Authorization=auth"), "Authorization") - close(t1) - - # 318 - dir = joinpath(dirname(pathof(HTTP)), "../test") - sslconfig = MbedTLS.SSLConfig(joinpath(dir, "resources/cert.pem"), joinpath(dir, "resources/key.pem")) - server = HTTP.listen!(; listenany=true, sslconfig = sslconfig, verbose=true) do http::HTTP.Stream - while !eof(http) - println("body data: ", String(readavailable(http))) - end - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - write(http, "response body\n") - write(http, "more response body") - end - port = HTTP.port(server) - r = HTTP.request("GET", "https://127.0.0.1:$port"; require_ssl_verification = false) - @test_throws HTTP.RequestError HTTP.request("GET", "http://127.0.0.1:$port"; require_ssl_verification = false) - close(server) - - # HTTP.listen with server kwarg - let host = Sockets.localhost; port = 8093 - port, server = Sockets.listenany(host, port) - HTTP.listen!(Sockets.localhost, port; server=server) do http - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - end - r = HTTP.get("http://$(host):$(port)/"; readtimeout=30) - @test r.status == 200 + @test server.state == :running + resp = HTTP.get("http://127.0.0.1:8080") + @test resp.status == 200 + @test String(resp.body) == "Hello, World!" + finally close(server) end - - # listen does not break with EOFError during ssl handshake - let host = Sockets.localhost - sslconfig = MbedTLS.SSLConfig(joinpath(dir, "resources/cert.pem"), joinpath(dir, "resources/key.pem")) - server = HTTP.listen!(; listenany=true, sslconfig=sslconfig, verbose=true) do http::HTTP.Stream - HTTP.setstatus(http, 200) - HTTP.startwrite(http) - write(http, "response body\n") - end - - port = HTTP.port(server) - - sock = connect(host, port) - close(sock) - - r = HTTP.get("https://$(host):$(port)/"; readtimeout=30, require_ssl_verification = false) - @test r.status == 200 - - close(server) - end -end # @testset - -@testset "on_shutdown" begin - @test HTTP.Servers.shutdown(nothing) === nothing - - # Shutdown adds 1 - TEST_COUNT = Ref(0) - shutdown_add() = TEST_COUNT[] += 1 - server = HTTP.listen!(x -> nothing; listenany=true, on_shutdown=shutdown_add) - close(server) - - # Shutdown adds 1, performed twice - @test TEST_COUNT[] == 1 - server = HTTP.listen!(x -> nothing; listenany=true, on_shutdown=[shutdown_add, shutdown_add]) - close(server) - @test TEST_COUNT[] == 3 - - # First shutdown function errors, second adds 1 - shutdown_throw() = throw(ErrorException("Broken")) - server = HTTP.listen!(x -> nothing; listenany=true, on_shutdown=[shutdown_throw, shutdown_add]) - @test_logs (:error, r"shutdown function .* failed.*ERROR: Broken.*"s) close(server) - @test TEST_COUNT[] == 4 -end # @testset +end @testset "access logging" begin - local handler = (http) -> begin - if http.message.target == "/internal-error" + local handler = (req) -> begin + if req.target == "/internal-error" error("internal error") end - if http.message.target == "/close" - HTTP.setstatus(http, 444) - close(http.stream) - return - end - HTTP.setstatus(http, 200) - HTTP.setheader(http, "Content-Type" => "text/plain") - msg = "hello, world" - HTTP.setheader(http, "Content-Length" => string(sizeof(msg))) - HTTP.startwrite(http) - if http.message.method == "GET" - HTTP.write(http, msg) + if req.target == "/close" + return HTTP.Response(444, ["content-type" => "text/plain"], nothing) end + return HTTP.Response(200, ["content-type" => "text/plain"], "hello, world") end function with_testserver(f, fmt) logger = Test.TestLogger() - server = Base.CoreLogging.with_logger(logger) do - HTTP.listen!(handler, Sockets.localhost, 32612; access_log=fmt) - end - try - f() - finally - close(server) + with_logger(logger) do + server = HTTP.serve!(handler; listenany=true, access_log=fmt) + port = HTTP.port(server) + try + f(port) + finally + close(server) + end end return filter!(x -> x.group == :access, logger.logs) end # Common Log Format - logs = with_testserver(common_logfmt) do - HTTP.get("http://localhost:32612") - HTTP.get("http://localhost:32612/index.html") - HTTP.get("http://localhost:32612/index.html?a=b") - HTTP.head("http://localhost:32612") - HTTP.get("http://localhost:32612/internal-error"; status_exception=false) - sleep(1) # necessary to properly forget the closed connection from the previous call - try HTTP.get("http://localhost:32612/close"; retry=false) catch end - HTTP.get("http://localhost:32612", ["Connection" => "close"]) + logs = with_testserver(common_logfmt) do port + HTTP.get("http://127.0.0.1:$port") + HTTP.get("http://127.0.0.1:$port/index.html") + HTTP.get("http://127.0.0.1:$port/index.html?a=b") + HTTP.head("http://127.0.0.1:$port") + HTTP.get("http://127.0.0.1:$port/internal-error"; status_exception=false) + # sleep(1) # necessary to properly forget the closed connection from the previous call + try HTTP.get("http://127.0.0.1:$port/close"; retry=false) catch end + HTTP.get("http://127.0.0.1:$port", ["Connection" => "close"]) sleep(1) # we want to make sure the server has time to finish logging before checking logs end @test length(logs) == 7 @@ -281,14 +59,14 @@ end # @testset @test occursin(r"^127.0.0.1 - - \[(\d{2})/.*/(\d{4}):\d{2}:\d{2}:\d{2}.*\] \"GET / HTTP/1.1\" 200 12$", logs[7].message) # Combined Log Format - logs = with_testserver(combined_logfmt) do - HTTP.get("http://localhost:32612", ["Referer" => "julialang.org"]) - HTTP.get("http://localhost:32612/index.html") - useragent = HTTP.HeadersRequest.USER_AGENT[] + logs = with_testserver(combined_logfmt) do port + HTTP.get("http://127.0.0.1:$port", ["Referer" => "julialang.org"]) + HTTP.get("http://127.0.0.1:$port/index.html") + useragent = HTTP.USER_AGENT[] HTTP.setuseragent!(nothing) - HTTP.get("http://localhost:32612/index.html?a=b") + HTTP.get("http://127.0.0.1:$port/index.html?a=b") HTTP.setuseragent!(useragent) - HTTP.head("http://localhost:32612") + HTTP.head("http://127.0.0.1:$port") end @test length(logs) == 4 @test all(x -> x.group === :access, logs) @@ -299,11 +77,11 @@ end # @testset # Custom log format fmt = logfmt"$http_accept $sent_http_content_type $request $request_method $request_uri $remote_addr $remote_port $remote_user $server_protocol $time_iso8601 $time_local $status $body_bytes_sent" - logs = with_testserver(fmt) do - HTTP.get("http://localhost:32612", ["Accept" => "application/json"]) - HTTP.get("http://localhost:32612/index.html") - HTTP.get("http://localhost:32612/index.html?a=b") - HTTP.head("http://localhost:32612") + logs = with_testserver(fmt) do port + HTTP.get("http://127.0.0.1:$port", ["Accept" => "application/json"]) + HTTP.get("http://127.0.0.1:$port/index.html") + HTTP.get("http://127.0.0.1:$port/index.html?a=b") + HTTP.head("http://127.0.0.1:$port") end @test length(logs) == 4 @test all(x -> x.group === :access, logs) @@ -311,6 +89,4 @@ end # @testset @test occursin(r"^\*/\* text/plain GET /index\.html HTTP/1\.1 GET /index\.html 127\.0\.0\.1 \d+ - HTTP/1\.1 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* \d+/.*/\d{4}:\d{2}:\d{2}:\d{2}.* 200 12$", logs[2].message) @test occursin(r"^\*/\* text/plain GET /index\.html\?a=b HTTP/1\.1 GET /index\.html\?a=b 127\.0\.0\.1 \d+ - HTTP/1\.1 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* \d+/.*/\d{4}:\d{2}:\d{2}:\d{2}.* 200 12$", logs[3].message) @test occursin(r"^\*/\* text/plain HEAD / HTTP/1\.1 HEAD / 127\.0\.0\.1 \d+ - HTTP/1\.1 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* \d+/.*/\d{4}:\d{2}:\d{2}:\d{2}.* 200 0$", logs[4].message) -end - -end # module +end \ No newline at end of file diff --git a/test/sniff.jl b/test/sniff.jl index 743ead949..0692563d2 100644 --- a/test/sniff.jl +++ b/test/sniff.jl @@ -2,18 +2,18 @@ const test_cases = [ ("Empty", UInt8[], "text/plain; charset=utf-8"), ("Binary", UInt8[1, 2, 3], "application/octet-stream"), - ("HTML document #1", HTTP.bytes("blah blah blah"), "text/html; charset=utf-8"), - ("HTML document #2", HTTP.bytes(""), "text/html; charset=utf-8"), - ("HTML document #3 (leading whitespace)", HTTP.bytes(" ..."), "text/html; charset=utf-8"), - ("HTML document #4 (leading CRLF)", HTTP.bytes("\r\n..."), "text/html; charset=utf-8"), + ("HTML document #1", b"blah blah blah", "text/html; charset=utf-8"), + ("HTML document #2", b"", "text/html; charset=utf-8"), + ("HTML document #3 (leading whitespace)", b" ...", "text/html; charset=utf-8"), + ("HTML document #4 (leading CRLF)", b"\r\n...", "text/html; charset=utf-8"), - ("Plain text", HTTP.bytes("This is not HTML. It has ☃ though."), "text/plain; charset=utf-8"), + ("Plain text", b"This is not HTML. It has ☃ though.", "text/plain; charset=utf-8"), - ("XML", HTTP.bytes("\n") == "&"'<>" - - @test HTTP.Cookies.isurlchar('\u81') - @test !HTTP.Cookies.isurlchar('\0') - - @test HTTP.Strings.tocameldash("accept") == "Accept" - @test HTTP.Strings.tocameldash("Accept") == "Accept" - @test HTTP.Strings.tocameldash("eXcept-this") == "Except-This" - @test HTTP.Strings.tocameldash("exCept-This") == "Except-This" - @test HTTP.Strings.tocameldash("not-valid") == "Not-Valid" - @test HTTP.Strings.tocameldash("♇") == "♇" - @test HTTP.Strings.tocameldash("bλ-a") == "Bλ-A" - @test HTTP.Strings.tocameldash("not fixable") == "Not fixable" - @test HTTP.Strings.tocameldash("aaaaaaaaaaaaa") == "Aaaaaaaaaaaaa" - @test HTTP.Strings.tocameldash("conTENT-Length") == "Content-Length" - @test HTTP.Strings.tocameldash("Sec-WebSocket-Key2") == "Sec-Websocket-Key2" - @test HTTP.Strings.tocameldash("User-agent") == "User-Agent" - @test HTTP.Strings.tocameldash("Proxy-authorization") == "Proxy-Authorization" - @test HTTP.Strings.tocameldash("HOST") == "Host" - @test HTTP.Strings.tocameldash("ST") == "St" - @test HTTP.Strings.tocameldash("X-\$PrototypeBI-Version") == "X-\$prototypebi-Version" - @test HTTP.Strings.tocameldash("DCLK_imp") == "Dclk_imp" + @test HTTP.escapehtml("&\"'<>") == "&"'<>" + + @test HTTP.tocameldash("accept") == "Accept" + @test HTTP.tocameldash("Accept") == "Accept" + @test HTTP.tocameldash("eXcept-this") == "Except-This" + @test HTTP.tocameldash("exCept-This") == "Except-This" + @test HTTP.tocameldash("not-valid") == "Not-Valid" + @test HTTP.tocameldash("♇") == "♇" + @test HTTP.tocameldash("bλ-a") == "Bλ-A" + @test HTTP.tocameldash("not fixable") == "Not fixable" + @test HTTP.tocameldash("aaaaaaaaaaaaa") == "Aaaaaaaaaaaaa" + @test HTTP.tocameldash("conTENT-Length") == "Content-Length" + @test HTTP.tocameldash("Sec-WebSocket-Key2") == "Sec-Websocket-Key2" + @test HTTP.tocameldash("User-agent") == "User-Agent" + @test HTTP.tocameldash("Proxy-authorization") == "Proxy-Authorization" + @test HTTP.tocameldash("HOST") == "Host" + @test HTTP.tocameldash("ST") == "St" + @test HTTP.tocameldash("X-\$PrototypeBI-Version") == "X-\$prototypebi-Version" + @test HTTP.tocameldash("DCLK_imp") == "Dclk_imp" for (bytes, utf8) in ( (UInt8[], ""), @@ -33,50 +27,6 @@ import HTTP.URIs (UInt8[0x6e, 0x6f, 0xeb, 0x6c], "noël"), (UInt8[0xc4, 0xc6, 0xe4], "ÄÆä"), ) - @test HTTP.Strings.iso8859_1_to_utf8(bytes) == utf8 - end - - withenv("HTTPS_PROXY"=>nothing, "https_proxy"=>nothing) do - @test HTTP.ConnectionRequest.getproxy("https", "https://julialang.org/") === nothing + @test HTTP.iso8859_1_to_utf8(bytes) == utf8 end - withenv("HTTPS_PROXY"=>"") do - # to be compatible with Julia 1.0 - @test HTTP.ConnectionRequest.getproxy("https", "https://julialang.org/") === nothing - end - withenv("https_proxy"=>"") do - @test HTTP.ConnectionRequest.getproxy("https", "https://julialang.org/") === nothing - end - withenv("HTTPS_PROXY"=>"https://user:pass@server:80") do - @test HTTP.ConnectionRequest.getproxy("https", "https://julialang.org/") == "https://user:pass@server:80" - end - withenv("https_proxy"=>"https://user:pass@server:80") do - @test HTTP.ConnectionRequest.getproxy("https", "https://julialang.org/") == "https://user:pass@server:80" - end - - withenv("HTTP_PROXY"=>nothing, "http_proxy"=>nothing) do - @test HTTP.ConnectionRequest.getproxy("http", "http://julialang.org/") === nothing - end - withenv("HTTP_PROXY"=>"") do - @test HTTP.ConnectionRequest.getproxy("http", "http://julialang.org/") === nothing - end - withenv("http_proxy"=>"") do - @test HTTP.ConnectionRequest.getproxy("http", "http://julialang.org/") === nothing - end - withenv("HTTP_PROXY"=>"http://user:pass@server:80") do - @test HTTP.ConnectionRequest.getproxy("http", "http://julialang.org/") == "http://user:pass@server:80" - end - withenv("http_proxy"=>"http://user:pass@server:80") do - @test HTTP.ConnectionRequest.getproxy("http", "http://julialang.org/") == "http://user:pass@server:80" - end -end # testset - - -@testset "Conditions" begin - function foo(x, y) - HTTP.@require x > 10 - HTTP.@ensure y > 10 - end - - @test_throws ArgumentError("foo() requires `x > 10`") foo(1, 11) - @test_throws AssertionError("foo() failed to ensure `y > 10`\ny = 1\n10 = 10") foo(11, 1) -end +end # testset \ No newline at end of file diff --git a/test/websockets/autobahn.jl b/test/websockets/autobahn.jl index 8aea00ce0..5b373c7ca 100644 --- a/test/websockets/autobahn.jl +++ b/test/websockets/autobahn.jl @@ -21,7 +21,7 @@ end @testset "Client" begin # Run the autobahn test suite in a docker container - serverproc = run(Cmd(`docker run --rm -v "$DIR/config:/config" -v "$DIR/reports:/reports" -p 9001:9001 --name fuzzingserver crossbario/autobahn-testsuite`; dir=DIR), stdin, stdout, stdout; wait=false) + serverproc = run(Cmd(`docker run --rm --platform linux/amd64 -v "$DIR/config:/config" -v "$DIR/reports:/reports" -p 9001:9001 --name fuzzingserver crossbario/autobahn-testsuite`; dir=DIR), stdin, stdout, stdout; wait=false) try sleep(5) # give time for server to get setup cases = Ref(0)