diff --git a/CMakeLists.txt b/CMakeLists.txt index d01de87..9504b71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ endif() include(GNUInstallDirs) if(BOOST_HTTP_IO_IS_ROOT) - set(BOOST_INCLUDE_LIBRARIES http_io http_proto url) + set(BOOST_INCLUDE_LIBRARIES http_io http_proto program_options url) set(BOOST_EXCLUDE_LIBRARIES http_io) set(CMAKE_FOLDER Dependencies) add_subdirectory(../.. Dependencies/boost EXCLUDE_FROM_ALL) diff --git a/example/client/flex_await/CMakeLists.txt b/example/client/flex_await/CMakeLists.txt index 16d4fbc..98dc2ad 100644 --- a/example/client/flex_await/CMakeLists.txt +++ b/example/client/flex_await/CMakeLists.txt @@ -27,6 +27,7 @@ find_package(ZLIB) target_link_libraries(http_io_example_client_flex_await Boost::http_io Boost::http_proto + Boost::program_options OpenSSL::SSL OpenSSL::Crypto) diff --git a/example/client/flex_await/Jamfile b/example/client/flex_await/Jamfile index 3875926..5c55dd4 100644 --- a/example/client/flex_await/Jamfile +++ b/example/client/flex_await/Jamfile @@ -16,6 +16,7 @@ project /boost/http_proto//boost_http_proto [ ac.check-library /boost/http_proto//boost_http_proto_zlib : /boost/http_proto//boost_http_proto_zlib : ] /boost/http_io//boost_http_io + /boost/program_options//boost_program_options /openssl//ssl/shared /openssl//crypto/shared . diff --git a/example/client/flex_await/main.cpp b/example/client/flex_await/main.cpp index 9d4e134..a22b149 100644 --- a/example/client/flex_await/main.cpp +++ b/example/client/flex_await/main.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -24,9 +25,16 @@ namespace asio = boost::asio; namespace core = boost::core; namespace http_io = boost::http_io; namespace http_proto = boost::http_proto; +namespace po = boost::program_options; namespace ssl = boost::asio::ssl; namespace urls = boost::urls; +#ifdef BOOST_HTTP_PROTO_HAS_ZLIB +inline const bool http_proto_has_zlib = true; +#else +inline const bool http_proto_has_zlib = false; +#endif + class any_stream { public: @@ -129,9 +137,8 @@ class any_stream std::visit( [&](auto& s) { - if constexpr(std::is_same_v< - decltype(s), - ssl_stream_type&>) + if constexpr( + std::is_same_v) { s.async_shutdown(std::move(self)); } @@ -154,21 +161,19 @@ class any_stream }; asio::awaitable -connect( - ssl::context& ssl_ctx, - core::string_view host, - core::string_view service) +connect(ssl::context& ssl_ctx, urls::url_view url) { - auto exec = co_await asio::this_coro::executor; - auto resolver = asio::ip::tcp::resolver{ exec }; - auto rresults = co_await resolver.async_resolve(host, service); + auto executor = co_await asio::this_coro::executor; + auto resolver = asio::ip::tcp::resolver{ executor }; + auto service = url.has_port() ? url.port() : url.scheme(); + auto rresults = co_await resolver.async_resolve(url.host(), service); if(service == "https") { - auto stream = ssl::stream{ exec, ssl_ctx }; + auto stream = ssl::stream{ executor, ssl_ctx }; co_await asio::async_connect(stream.lowest_layer(), rresults); - if(auto host_s = std::string{ host }; + if(auto host_s = std::string{ url.host() }; !SSL_set_tlsext_host_name(stream.native_handle(), host_s.c_str())) { throw boost::system::system_error( @@ -180,113 +185,188 @@ connect( co_return stream; } - auto stream = asio::ip::tcp::socket{ exec }; + auto stream = asio::ip::tcp::socket{ executor }; co_await asio::async_connect(stream, rresults); co_return stream; } -bool -is_redirect(http_proto::response_view resp) noexcept +auto +is_redirect(http_proto::status status) noexcept { - switch(resp.status()) + struct result_t + { + bool is_redirect; + bool need_method_change; + }; + + // The specifications do not intend for 301 and 302 redirects to change the + // HTTP method, but most user agents do change the method in practice. + switch(status) { case http_proto::status::moved_permanently: case http_proto::status::found: + case http_proto::status::see_other: + return result_t{ true, true }; case http_proto::status::temporary_redirect: case http_proto::status::permanent_redirect: - return true; + return result_t{ true, false }; default: - return false; + return result_t{ false, false }; } } -asio::awaitable -write_get_request( - http_proto::context& http_proto_ctx, - any_stream& stream, - core::string_view host, - core::string_view target) +core::string_view +get_traget(urls::url_view url) noexcept +{ + if(url.encoded_target().empty()) + return "/"; + + return url.encoded_target(); +} + +http_proto::request +create_request(const po::variables_map& vm, urls::url_view url) { - auto exec = co_await asio::this_coro::executor; + using http_proto::field; + using http_proto::method; + using http_proto::version; + auto request = http_proto::request{}; - request.set_method(http_proto::method::get); - request.set_target(target); - request.set_version(http_proto::version::http_1_1); + request.set_method(vm.count("head") ? method::head : method::get); + + if(vm.count("request")) + request.set_method(vm.at("request").as()); + + request.set_version( + vm.count("http1.0") ? version::http_1_0 : version::http_1_1); + + request.set_target(get_traget(url)); request.set_keep_alive(false); - request.set(http_proto::field::host, host); - request.set(http_proto::field::user_agent, "Boost.Http.Io"); + request.set(field::host, url.host()); - #ifdef BOOST_HTTP_PROTO_HAS_ZLIB - request.set(http_proto::field::accept_encoding, "gzip, deflate"); - #endif + if(vm.count("continue-at")) + { + auto value = "bytes=" + + std::to_string(vm.at("continue-at").as()) + "-"; + request.set(field::range, value); + } - auto serializer = http_proto::serializer{ http_proto_ctx }; - serializer.start(request); - co_await http_io::async_write(stream, serializer); + if(vm.count("user-agent")) + { + request.set(field::user_agent, vm.at("user-agent").as()); + } + else + { + request.set(field::user_agent, "Boost.Http.Io"); + } + + if(vm.count("referer")) + request.set(field::referer, vm.at("referer").as()); + + if(vm.count("user")) + { + // TODO: use base64 encoding for basic authentication + request.set(field::authorization, vm.at("user").as()); + } + + if(vm.count("compressed") && http_proto_has_zlib) + request.set(field::accept_encoding, "gzip, deflate"); + + // Set user provided headers + if(vm.count("header")) + { + for(auto& header : vm.at("header").as>()) + { + if(auto pos = header.find(':'); pos != std::string::npos) + request.set(header.substr(0, pos), header.substr(pos + 1)); + } + } + + return request; } // Performs an HTTP GET and prints the response asio::awaitable request( + const po::variables_map& vm, ssl::context& ssl_ctx, http_proto::context& http_proto_ctx, - core::string_view host, - core::string_view service, - core::string_view target) + http_proto::request request, + urls::url_view url) { - auto exec = co_await asio::this_coro::executor; - auto stream = co_await connect(ssl_ctx, host, service); + auto stream = co_await connect(ssl_ctx, url); + auto parser = http_proto::response_parser{ http_proto_ctx }; + auto serializer = http_proto::serializer{ http_proto_ctx }; - co_await write_get_request(http_proto_ctx, stream, host, target); + serializer.start(request); + co_await http_io::async_write(stream, serializer); - auto parser = http_proto::response_parser{ http_proto_ctx }; parser.reset(); parser.start(); - co_await http_io::async_read_header(stream, parser); - while(is_redirect(parser.get())) + auto referer_url = urls::url{ url }; + for(;;) { - auto resp = parser.get(); - if(auto it = resp.find(http_proto::field::location); it != resp.end()) + auto [is_redirect, need_method_change] = + ::is_redirect(parser.get().status()); + + if(!is_redirect || !vm.count("location")) + break; + + auto response = parser.get(); + if(auto it = response.find(http_proto::field::location); + it != response.end()) { - auto url = urls::parse_uri(it->value); - auto host = url->host(); - auto service = url->has_port() ? url->port() : url->scheme(); - auto target = - !url->encoded_target().empty() ? url->encoded_target() : "/"; + auto redirect_url = urls::parse_uri(it->value).value(); - // TODO: reuse the connection when possible + // TODO: reuse the established connection when possible co_await stream.async_shutdown(asio::as_tuple); - stream = co_await connect(ssl_ctx, host, service); - co_await write_get_request(http_proto_ctx, stream, host, target); + stream = co_await connect(ssl_ctx, redirect_url); + + // Change the method according to RFC 9110, Section 15.4.4. + if(need_method_change && !vm.count("head")) + { + request.set_method(http_proto::method::get); + // TODO: drop the request body + } + request.set_target(get_traget(redirect_url)); + request.set(http_proto::field::host, redirect_url.host()); + request.set(http_proto::field::referer, referer_url); + + referer_url = redirect_url; + + serializer.reset(); + serializer.start(request); + co_await http_io::async_write(stream, serializer); parser.reset(); parser.start(); - co_await http_io::async_read_header(stream, parser); } else { - throw std::runtime_error{ "Bad redirect response"}; + throw std::runtime_error{ "Bad redirect response" }; } } + if(vm.count("head") || vm.count("show-headers")) + std::cout << parser.get().buffer() << std::flush; + for(;;) { for(auto cb : parser.pull_body()) { - std::cout.write(static_cast( - cb.data()), cb.size()); + std::cout.write(static_cast(cb.data()), cb.size()); parser.consume_body(cb.size()); } if(parser.is_complete()) break; - auto [ec, _] = co_await http_io::async_read_some( - stream, parser, asio::as_tuple); + auto [ec, _] = + co_await http_io::async_read_some(stream, parser, asio::as_tuple); if(ec && ec != http_proto::condition::need_more_input) throw boost::system::system_error{ ec }; } @@ -301,25 +381,67 @@ main(int argc, char* argv[]) { try { - if(argc != 2) + auto odesc = po::options_description{"Options"}; + odesc.add_options() + ("help,h", "produce help message") + ("head,I", "Show document info only") + ("header,H", + po::value>()->value_name("
"), + "Pass custom header(s) to server") + ("location,L", "Follow redirects") + ("continue-at,C", + po::value()->value_name(""), + "Resume transfer offset") + ("request,X", + po::value()->value_name(""), + "Specify request method to use") + ("show-headers,i", "Show response headers in the output") + ("referer,e", + po::value()->value_name(""), + "Referer URL") + ("user,u", + po::value()->value_name(""), + "Server user and password") + ("user-agent,A", + po::value()->value_name(""), + "Send User-Agent to server") + ("url", + po::value()->value_name(""), + "URL to work with") + ("compressed", "Request compressed response") + ("http1.0", "Use HTTP 1.0"); + + auto podesc = po::positional_options_description{}; + podesc.add("url", 1); + + po::variables_map vm; + po::store( + po::command_line_parser{ argc, argv } + .options(odesc) + .positional(podesc) + .run(), + vm); + po::notify(vm); + + if(vm.count("help") || !vm.count("url")) { - std::cerr << "Usage: flex_await \n" - << "Example:\n" - << " flex_await https://www.example.com\n"; + std::cerr + << "Usage: flex_await [options...] \n" + << "Example:\n" + << " flex_await https://www.example.com\n" + << " flex_await -L http://httpstat.us/301\n" + << odesc; return EXIT_FAILURE; } - auto url = urls::parse_uri(argv[1]); + auto url = urls::parse_uri(vm.at("url").as()); if(url.has_error()) { - std::cerr << "Failed to parse URL\n" - << "Error: " << url.error().what() << std::endl; + std::cerr + << "Failed to parse URL\n" + << "Error: " << url.error().what() << std::endl; return EXIT_FAILURE; - } - auto host = url->host(); - auto service = url->has_port() ? url->port() : url->scheme(); - auto target = - !url->encoded_target().empty() ? url->encoded_target() : "/"; + } auto ioc = asio::io_context{}; auto ssl_ctx = ssl::context{ ssl::context::tlsv12_client }; @@ -331,17 +453,23 @@ main(int argc, char* argv[]) http_proto::response_parser::config cfg; cfg.body_limit = std::numeric_limits::max(); cfg.min_buffer = 1024 * 1024; - #ifdef BOOST_HTTP_PROTO_HAS_ZLIB - cfg.apply_gzip_decoder = true; - cfg.apply_deflate_decoder = true; - http_proto::zlib::install_service(http_proto_ctx); - #endif + if(http_proto_has_zlib) + { + cfg.apply_gzip_decoder = true; + cfg.apply_deflate_decoder = true; + http_proto::zlib::install_service(http_proto_ctx); + } http_proto::install_parser_service(http_proto_ctx, cfg); } asio::co_spawn( ioc, - request(ssl_ctx, http_proto_ctx, host, service, target), + request( + vm, + ssl_ctx, + http_proto_ctx, + create_request(vm, url.value()), + url.value()), [](std::exception_ptr ep) { if(ep)