From 3c8916491dd6f304ceb2620693800ee22f339943 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Thu, 11 Jun 2026 13:00:17 +0000 Subject: [PATCH 1/7] multipart_form: add bytes() for in-memory file parts --- include/boost/burl/multipart_form.hpp | 34 +++++++ src/multipart_form.cpp | 25 ++++++ test/unit/multipart_form.cpp | 124 ++++++++++++++++---------- 3 files changed, 138 insertions(+), 45 deletions(-) diff --git a/include/boost/burl/multipart_form.hpp b/include/boost/burl/multipart_form.hpp index 498d7e5..5392a13 100644 --- a/include/boost/burl/multipart_form.hpp +++ b/include/boost/burl/multipart_form.hpp @@ -146,6 +146,40 @@ class multipart_form std::string_view filename = {}, std::string_view content_type = {}); + /** Append a file part with in-memory contents to the form. + + The part is written with a `filename` in its + `Content-Disposition` header, so servers + treat it as a file upload, but the contents + are held in memory rather than streamed from + disk. + + @par Exception Safety + Calls to allocate may throw. + + @param name The name of the form field. + + @param data The contents of the part. + + @param filename The filename to report in + the part header. + + @param content_type The value for the + `Content-Type` header of the part. Deduced + from the filename extension when empty, with + `application/octet-stream` as the fallback. + + @return A reference to this object, for + chaining. + */ + BOOST_BURL_DECL + multipart_form& + bytes( + std::string_view name, + std::string data, + std::string_view filename, + std::string_view content_type = {}); + private: static std::string generate_boundary(); diff --git a/src/multipart_form.cpp b/src/multipart_form.cpp index c84cd78..87311b7 100644 --- a/src/multipart_form.cpp +++ b/src/multipart_form.cpp @@ -80,6 +80,31 @@ multipart_form::file( return *this; } +multipart_form& +multipart_form::bytes( + std::string_view name, + std::string data, + std::string_view filename, + std::string_view content_type) +{ + std::string content_type_buf; + if(content_type.empty()) + { + content_type_buf = http::mime_types::content_type(filename); + if(content_type_buf.empty()) + content_type_buf = "application/octet-stream"; + content_type = content_type_buf; + } + auto size = data.size(); + parts_.push_back( + part{ .header = make_header(name, filename, content_type), + .is_file = false, + .text = std::move(data), + .path = {}, + .size = size }); + return *this; +} + std::string multipart_form::generate_boundary() { diff --git a/test/unit/multipart_form.cpp b/test/unit/multipart_form.cpp index a8f64cb..a6c1b2d 100644 --- a/test/unit/multipart_form.cpp +++ b/test/unit/multipart_form.cpp @@ -81,6 +81,21 @@ struct multipart_form_test check_body(body, expected); } + static void + check_io(any_request_body const& body, std::string_view expected) + { + auto cl = body.content_length(); + BOOST_TEST(cl.has_value()); + BOOST_TEST_EQ(cl.value(), expected.size()); + + capy::test::buffer_sink bs; + capy::any_buffer_sink sink(&bs); + + auto ec = drive_body(body, sink); + BOOST_TEST(!ec); + BOOST_TEST_EQ(bs.data(), expected); + } + void testEmpty() { @@ -186,61 +201,87 @@ struct multipart_form_test part(boundary, "doc", contents, "report.txt", "text/plain") + "--" + boundary + "--\r\n"; - auto cl = body.content_length(); - BOOST_TEST(cl.has_value()); - BOOST_TEST_EQ(cl.value(), expected.size()); + check_io(body, expected); + } - capy::test::buffer_sink bs; - capy::any_buffer_sink sink(&bs); + void + testFilePartDeduced() + { + auto check_one = [&](std::string_view extension, + std::string_view expected_ct) { + std::string contents = "a\nb\nc\n"; + temp_file tmp(contents, extension); - auto ec = drive_body(body, sink); - BOOST_TEST(!ec); - BOOST_TEST_EQ(bs.data(), expected); + multipart_form form; + form.file("doc", tmp.path); + + auto body = + tag_invoke(body_from_tag{}, std::move(form)); + + auto boundary = boundary_of(body); + auto expected = + part( + boundary, + "doc", + contents, + tmp.path.filename().string(), + expected_ct) + + "--" + boundary + "--\r\n"; + + check_io(body, expected); + }; + + check_one(".txt", "text/plain; charset=UTF-8"); + check_one(".png", "image/png"); + + // fall back to octet-stream. + check_one(".zzz", "application/octet-stream"); + check_one("", "application/octet-stream"); } void - check_deduced(std::string_view extension, std::string_view expected_ct) + testBytesPart() { - std::string contents = "a\nb\nc\n"; - temp_file tmp(contents, extension); - multipart_form form; - form.file("doc", tmp.path); + + // bytes() returns *this for chaining + auto& ref = + form.bytes("report", "x,y\n1,2\n", "report.csv", "text/csv"); + BOOST_TEST_EQ(&ref, &form); auto body = tag_invoke(body_from_tag{}, std::move(form)); auto boundary = boundary_of(body); auto expected = - part( - boundary, - "doc", - contents, - tmp.path.filename().string(), - expected_ct) + + part(boundary, "report", "x,y\n1,2\n", "report.csv", "text/csv") + "--" + boundary + "--\r\n"; - auto cl = body.content_length(); - BOOST_TEST(cl.has_value()); - BOOST_TEST_EQ(cl.value(), expected.size()); - - capy::test::buffer_sink bs; - capy::any_buffer_sink sink(&bs); - - auto ec = drive_body(body, sink); - BOOST_TEST(!ec); - BOOST_TEST_EQ(bs.data(), expected); + check(body, expected); } void - testFilePartDeduced() + testBytesDeduced() { - check_deduced(".txt", "text/plain; charset=UTF-8"); - check_deduced(".png", "image/png"); + auto check_one = [&](std::string_view filename, + std::string_view expected_ct) { + multipart_form form; + form.bytes("doc", "a\nb\nc\n", filename); - // fall back to octet-stream. - check_deduced(".zzz", "application/octet-stream"); - check_deduced("", "application/octet-stream"); + auto body = + tag_invoke(body_from_tag{}, std::move(form)); + + auto boundary = boundary_of(body); + auto expected = + part(boundary, "doc", "a\nb\nc\n", filename, expected_ct) + + "--" + boundary + "--\r\n"; + + check(body, expected); + }; + + check_one("notes.txt", "text/plain; charset=UTF-8"); + check_one("img.png", "image/png"); + check_one("data.zzz", "application/octet-stream"); } void @@ -263,16 +304,7 @@ struct multipart_form_test part(boundary, "attachment", contents, "crash.log", "text/plain") + "--" + boundary + "--\r\n"; - auto cl = body.content_length(); - BOOST_TEST(cl.has_value()); - BOOST_TEST_EQ(cl.value(), expected.size()); - - capy::test::buffer_sink bs; - capy::any_buffer_sink sink(&bs); - - auto ec = drive_body(body, sink); - BOOST_TEST(!ec); - BOOST_TEST_EQ(bs.data(), expected); + check_io(body, expected); } void @@ -285,6 +317,8 @@ struct multipart_form_test testEmptyValue(); testFilePart(); testFilePartDeduced(); + testBytesPart(); + testBytesDeduced(); testMixedParts(); } }; From 2bd364117b4f624f9fa404c8ad3f8920c747c777 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Thu, 11 Jun 2026 13:21:37 +0000 Subject: [PATCH 2/7] request_builder: add per-request followlocation override --- example/usage.cpp | 19 +++++++++++++----- include/boost/burl/request.hpp | 10 ++++++++++ include/boost/burl/request_builder.hpp | 27 ++++++++++++++++++++++++++ src/client.cpp | 3 ++- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/example/usage.cpp b/example/usage.cpp index 1b3ac14..3af343f 100644 --- a/example/usage.cpp +++ b/example/usage.cpp @@ -387,13 +387,22 @@ follow_redirects(corosio::tls_context tls_ctx) burl::client client(co_await capy::this_coro::executor, tls_ctx, cfg); - auto [ec, r] = co_await client.get("http://boost.org").send(); + auto [ec1, r1] = co_await client.get("http://boost.org").send(); - if(ec) - throw std::system_error(ec); + if(ec1) + throw std::system_error(ec1); + + // The final URL after following redirects + std::cout << r1.url() << '\n'; + + auto [ec2, r2] = co_await client.get("http://boost.org") + .followlocation(false) // per-request override + .send(); + + if(ec2) + throw std::system_error(ec2); - // Final URL after following redirects, e.g. https://www.boost.org - std::cout << r.url() << '\n'; + std::cout << r2.status_int() << '\n'; // e.g. 301 } //============================================================== diff --git a/include/boost/burl/request.hpp b/include/boost/burl/request.hpp index 67aee69..8df6c9d 100644 --- a/include/boost/burl/request.hpp +++ b/include/boost/burl/request.hpp @@ -64,6 +64,16 @@ struct request @see @ref request_builder::timeout. */ std::optional timeout; + + /** Follow redirect responses automatically. + + When set, overrides + @ref client::config::followlocation for + this request. + + @see @ref request_builder::followlocation. + */ + std::optional followlocation; }; /** The request method. diff --git a/include/boost/burl/request_builder.hpp b/include/boost/burl/request_builder.hpp index b8f0de2..a22401b 100644 --- a/include/boost/burl/request_builder.hpp +++ b/include/boost/burl/request_builder.hpp @@ -196,6 +196,33 @@ class request_builder return std::move(*this); } + /** Set whether redirect responses are followed. + + Overrides + @ref client::config::followlocation for + this request. When disabled, the redirect + response is returned as-is; its `Location` + header can be inspected through the + @ref response. + + @par Example + @code + auto [ec, r] = co_await c.get("https://example.com/moved") + .followlocation(false) + .send(); + @endcode + + @param enable `true` to follow redirects. + + @return The builder, for chaining. + */ + request_builder&& + followlocation(bool enable) && + { + request_.options.followlocation = enable; + return std::move(*this); + } + /** Set the request body. The value is converted to a request body by diff --git a/src/client.cpp b/src/client.cpp index 1e2edfc..5a4db18 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -278,6 +278,7 @@ client::execute_impl( auto url = request.url; auto trusted = true; + auto followlocation = request.options.followlocation.value_or(config_.followlocation); auto maxredirs = config_.maxredirs; auto request_cookies = request.headers.value_or(field::cookie, ""); for(;;) @@ -346,7 +347,7 @@ client::execute_impl( auto [is_redirect, need_method_change] = burl::is_redirect(parser.get().status(), config_); - if(!is_redirect || !config_.followlocation) + if(!is_redirect || !followlocation) { auto ec = std::error_code{}; auto status_int = parser.get().status_int(); From de900cc5203bfb88833e02d9b4146ee99dafa735 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Thu, 11 Jun 2026 15:33:57 +0000 Subject: [PATCH 3/7] urlencoded_form: add range constructor and append overload --- include/boost/burl/urlencoded_form.hpp | 61 ++++++++++++++++++++++++++ test/unit/urlencoded_form.cpp | 29 ++++++++++++ 2 files changed, 90 insertions(+) diff --git a/include/boost/burl/urlencoded_form.hpp b/include/boost/burl/urlencoded_form.hpp index 8e0ef64..a554b86 100644 --- a/include/boost/burl/urlencoded_form.hpp +++ b/include/boost/burl/urlencoded_form.hpp @@ -14,7 +14,9 @@ #include #include +#include #include +#include #include #include #include @@ -88,6 +90,38 @@ class urlencoded_form std::initializer_list< std::pair> fields); + /** Constructor. + + Constructs a form containing the name and + value pairs from the passed range. + + @par Example + @code + std::map fields = { + { "user", "John" }, + { "lang", "En" } }; + + auto r = co_await c.post("https://example.com/post") + .body(fields) + .send(); + @endcode + + @par Exception Safety + Calls to allocate may throw. + + @param fields The range of name and value + pairs to append. + */ + template + requires std::ranges::input_range && + std::convertible_to< + std::ranges::range_reference_t, + std::pair> + urlencoded_form(Range&& fields) + { + append(std::forward(fields)); + } + /** Append a field to the form. The name and value are percent-encoded, @@ -107,6 +141,33 @@ class urlencoded_form urlencoded_form& append(std::string_view name, std::string_view value); + /** Append fields to the form. + + Appends the name and value pairs from the + passed range. + + @par Exception Safety + Calls to allocate may throw. + + @param fields The range of name and value + pairs to append. + + @return A reference to this object, for + chaining. + */ + template + requires std::ranges::input_range && + std::convertible_to< + std::ranges::range_reference_t, + std::pair> + urlencoded_form& + append(Range&& fields) + { + for(std::pair field : fields) + append(field.first, field.second); + return *this; + } + private: friend BOOST_BURL_DECL any_request_body tag_invoke(body_from_tag, urlencoded_form form); diff --git a/test/unit/urlencoded_form.cpp b/test/unit/urlencoded_form.cpp index abd146c..8b1bb8a 100644 --- a/test/unit/urlencoded_form.cpp +++ b/test/unit/urlencoded_form.cpp @@ -20,8 +20,11 @@ #include "test_suite.hpp" +#include +#include #include #include +#include namespace boost { @@ -88,6 +91,31 @@ struct urlencoded_form_test "user=John&lang=En"); } + void + testRange() + { + std::map map + { + { "lang", "En" }, + { "user", "John" } + }; + check(map, "lang=En&user=John"); + + std::vector> vec + { + { "user", "John" }, + { "lang", "En" } + }; + check(urlencoded_form(vec), "user=John&lang=En"); + + // Range append returns *this for chaining. + urlencoded_form form; + auto& ref = form.append(vec); + BOOST_TEST_EQ(&ref, &form); + form.append(map); + check(std::move(form), "user=John&lang=En&lang=En&user=John"); + } + void testEncoding() { @@ -115,6 +143,7 @@ struct urlencoded_form_test testEmpty(); testAppend(); testInitializerList(); + testRange(); testEncoding(); } }; From f5ca922a64c88139c9c4b4b90c02a36511f944f1 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 12 Jun 2026 07:21:03 +0000 Subject: [PATCH 4/7] response: reuse connection when an unread body has already arrived --- include/boost/burl/connection_pool.hpp | 25 +++++++++----- include/boost/burl/response.hpp | 12 +++---- src/client.cpp | 9 ++--- src/connection_pool.cpp | 34 +++++++++--------- src/detail/reuse.cpp | 48 ++++++++++++++++++++++++++ src/detail/reuse.hpp | 29 ++++++++++++++++ src/response.cpp | 14 ++++---- 7 files changed, 127 insertions(+), 44 deletions(-) create mode 100644 src/detail/reuse.cpp create mode 100644 src/detail/reuse.hpp diff --git a/include/boost/burl/connection_pool.hpp b/include/boost/burl/connection_pool.hpp index b92c66f..b1c9898 100644 --- a/include/boost/burl/connection_pool.hpp +++ b/include/boost/burl/connection_pool.hpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -194,6 +193,8 @@ class connection_pool friend class response; std::unique_ptr conn_; + connection_pool* pool_ = nullptr; + std::string key_; std::optional io_timeout_; capy::detail::buffer_array<8, false> rba_; // TODO capy::detail::buffer_array<8, true> wba_; // TODO @@ -202,8 +203,12 @@ class connection_pool pooled_connection( std::unique_ptr conn, + connection_pool* pool, + std::string key, std::optional io_timeout = std::nullopt) : conn_(std::move(conn)) + , pool_(pool) + , key_(std::move(key)) , io_timeout_(io_timeout) { } @@ -230,6 +235,16 @@ class connection_pool return capy::timeout(conn_->write_some(wba_), *io_timeout_); return conn_->write_some(wba_); } + + explicit + operator bool() const noexcept + { + return conn_ != nullptr; + } + + BOOST_BURL_DECL + void + return_to_pool(); }; struct idle_connection @@ -238,18 +253,12 @@ class connection_pool config::clock::time_point idle_since; }; - BOOST_BURL_DECL capy::io_task acquire(urls::url_view url); - BOOST_BURL_DECL void - release( - urls::url_view url, - pooled_connection pc, - http::response_parser const& parser); + release(pooled_connection pc); - BOOST_BURL_DECL capy::io_task> connect(urls::url_view url) const; diff --git a/include/boost/burl/response.hpp b/include/boost/burl/response.hpp index 2c44507..22d5dcf 100644 --- a/include/boost/burl/response.hpp +++ b/include/boost/burl/response.hpp @@ -48,9 +48,8 @@ namespace burl The response owns the connection it was received on. Upon destruction, the connection is returned - to the pool for reuse when the body was read to - completion and the connection can be kept alive; - otherwise it is closed. + to the pool for reuse when it can be kept alive + and the entire message has arrived. @par Example @code @@ -75,14 +74,12 @@ class response urls::url url_; connection_pool::pooled_connection conn_; - connection_pool* pool_ = nullptr; http::response_parser parser_; std::optional deadline_; response( urls::url url, connection_pool::pooled_connection conn, - connection_pool* pool, http::response_parser parser, std::optional deadline); @@ -127,9 +124,8 @@ class response /** Destructor. Returns the connection to the pool for reuse - when the body was read to completion and the - connection can be kept alive; otherwise the - connection is closed. + when it can be kept alive and the entire + message has arrived. */ BOOST_BURL_DECL ~response(); diff --git a/src/client.cpp b/src/client.cpp index 5a4db18..24bccaa 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -11,6 +11,7 @@ #include #include "detail/base64.hpp" +#include "detail/reuse.hpp" #include #include @@ -274,7 +275,6 @@ client::execute_impl( serializer.set_message(headers); http::response_parser parser(http::make_parser_config(parser_cfg)); - parser.reset(); auto url = request.url; auto trusted = true; @@ -324,6 +324,7 @@ client::execute_impl( co_return { wec, {} }; } + parser.reset(); if(headers.method() == http::method::head) parser.start_head_response(); else @@ -356,8 +357,7 @@ client::execute_impl( co_return { ec, - response{ - url, std::move(conn), &pool_, std::move(parser), deadline } + response{ url, std::move(conn), std::move(parser), deadline } }; } @@ -365,7 +365,8 @@ client::execute_impl( parser.set_body_limit(64 * 1024); if(auto [ec] = co_await parser.read(conn); ec) parser.reset(); - pool_.release(url, std::move(conn), parser); + if(detail::can_reuse_conn(parser)) + conn.return_to_pool(); if(maxredirs-- == 0) co_return { error::too_many_redirects, {} }; diff --git a/src/connection_pool.cpp b/src/connection_pool.cpp index 44cb16b..9e01247 100644 --- a/src/connection_pool.cpp +++ b/src/connection_pool.cpp @@ -407,7 +407,7 @@ connection_pool::connect(urls::url_view url) const capy::io_task connection_pool::acquire(urls::url_view url) { - auto const key = origin(url); + auto key = origin(url); auto const now = config::clock::now(); auto [it, last] = idle_.equal_range(key); @@ -422,7 +422,9 @@ connection_pool::acquire(urls::url_view url) if(!entry.conn->is_open()) continue; - co_return { {}, { std::move(entry.conn), config_.io_timeout } }; + co_return { + {}, { std::move(entry.conn), this, std::move(key), config_.io_timeout } + }; } auto [ec, conn] = @@ -430,30 +432,30 @@ connection_pool::acquire(urls::url_view url) if(ec) co_return { ec, {} }; - co_return { {}, { std::move(conn), config_.io_timeout } }; + co_return { + {}, { std::move(conn), this, std::move(key), config_.io_timeout } + }; } void -connection_pool::release( - urls::url_view url, - connection_pool::pooled_connection pc, - http::response_parser const& parser) +connection_pool::release(pooled_connection pc) { - if(!parser.is_complete()) - return; - - if(!parser.get().keep_alive()) - return; - if(!pc.conn_ || !pc.conn_->is_open()) return; - auto const key = origin(url); - if(idle_.count(key) >= config_.max_idle_per_host) + if(idle_.count(pc.key_) >= config_.max_idle_per_host) return; idle_.emplace( - key, idle_connection{ std::move(pc.conn_), config::clock::now() }); + std::move(pc.key_), + idle_connection{ std::move(pc.conn_), config::clock::now() }); +} + +void +connection_pool::pooled_connection::return_to_pool() +{ + if(pool_) + pool_->release(std::move(*this)); } } // namespace burl diff --git a/src/detail/reuse.cpp b/src/detail/reuse.cpp new file mode 100644 index 0000000..d3ded92 --- /dev/null +++ b/src/detail/reuse.cpp @@ -0,0 +1,48 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#include "reuse.hpp" + +namespace boost +{ +namespace burl +{ +namespace detail +{ + +bool +can_reuse_conn(http::response_parser& parser) noexcept +{ + if(!parser.got_header()) + return false; + + if(!parser.get().keep_alive()) + return false; + + if(!parser.is_complete()) + { + // The rest of the message may already sit in the + // parser's buffer; parsing it needs no I/O and makes + // the connection reusable. + try + { + system::error_code ec; + parser.parse(ec); + } + catch(...) + { + } + } + + return parser.is_complete(); +} + +} // namespace detail +} // namespace burl +} // namespace boost diff --git a/src/detail/reuse.hpp b/src/detail/reuse.hpp new file mode 100644 index 0000000..36b6e6e --- /dev/null +++ b/src/detail/reuse.hpp @@ -0,0 +1,29 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_SRC_DETAIL_REUSE_HPP +#define BOOST_BURL_SRC_DETAIL_REUSE_HPP + +#include + +namespace boost +{ +namespace burl +{ +namespace detail +{ + +bool +can_reuse_conn(http::response_parser& parser) noexcept; + +} // namespace detail +} // namespace burl +} // namespace boost + +#endif diff --git a/src/response.cpp b/src/response.cpp index b15eb66..d03a393 100644 --- a/src/response.cpp +++ b/src/response.cpp @@ -10,6 +10,8 @@ #include #include +#include "detail/reuse.hpp" + #include #include @@ -24,12 +26,10 @@ namespace burl response::response( urls::url url, connection_pool::pooled_connection conn, - connection_pool* pool, http::response_parser parser, std::optional deadline) : url_(std::move(url)) , conn_(std::move(conn)) - , pool_(pool) , parser_(std::move(parser)) , deadline_(deadline) { @@ -38,7 +38,6 @@ response::response( response::response(response&& other) noexcept : url_(std::move(other.url_)) , conn_(std::move(other.conn_)) - , pool_(std::exchange(other.pool_, nullptr)) , parser_(std::move(other.parser_)) , deadline_(other.deadline_) { @@ -49,11 +48,10 @@ response::operator=(response&& other) noexcept { if(this != &other) { - if(pool_) - pool_->release(url_, std::move(conn_), parser_); + if(conn_ && detail::can_reuse_conn(parser_)) + conn_.return_to_pool(); url_ = std::move(other.url_); conn_ = std::move(other.conn_); - pool_ = std::exchange(other.pool_, nullptr); parser_ = std::move(other.parser_); deadline_ = other.deadline_; } @@ -62,8 +60,8 @@ response::operator=(response&& other) noexcept response::~response() { - if(pool_) - pool_->release(url_, std::move(conn_), parser_); + if(conn_ && detail::can_reuse_conn(parser_)) + conn_.return_to_pool(); } capy::io_task From dd890dd503920282372e1c74388eb90e05660212 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 12 Jun 2026 11:44:42 +0000 Subject: [PATCH 5/7] connection_pool: allow responses to outlive the client --- include/boost/burl/connection_pool.hpp | 42 +-- include/boost/burl/response.hpp | 5 + src/connection_pool.cpp | 374 ++++++++++++++----------- 3 files changed, 230 insertions(+), 191 deletions(-) diff --git a/include/boost/burl/connection_pool.hpp b/include/boost/burl/connection_pool.hpp index b1c9898..ccff57b 100644 --- a/include/boost/burl/connection_pool.hpp +++ b/include/boost/burl/connection_pool.hpp @@ -26,7 +26,6 @@ #include #include #include -#include #include namespace boost @@ -52,6 +51,9 @@ class response; Each @ref client owns a connection pool, configured through @ref client::config::pool. + This class is a shared handle to the pool state. + Copies share the same underlying pool. + @see @ref client. */ class connection_pool @@ -155,17 +157,15 @@ class connection_pool @param cfg The configuration settings. */ + BOOST_BURL_DECL connection_pool( capy::executor_ref exec, corosio::tls_context tls_ctx, - config cfg) - : exec_(exec) - , tls_ctx_(std::move(tls_ctx)) - , config_(std::move(cfg)) - { - } + config cfg); private: + class impl; + class connection { public: @@ -184,16 +184,13 @@ class connection_pool virtual ~connection() = default; }; - class tcp_connection; - class tls_connection; - class pooled_connection { - friend class connection_pool; + friend class impl; friend class response; std::unique_ptr conn_; - connection_pool* pool_ = nullptr; + std::weak_ptr pool_; std::string key_; std::optional io_timeout_; capy::detail::buffer_array<8, false> rba_; // TODO @@ -203,11 +200,11 @@ class connection_pool pooled_connection( std::unique_ptr conn, - connection_pool* pool, + std::weak_ptr pool, std::string key, std::optional io_timeout = std::nullopt) : conn_(std::move(conn)) - , pool_(pool) + , pool_(std::move(pool)) , key_(std::move(key)) , io_timeout_(io_timeout) { @@ -247,25 +244,10 @@ class connection_pool return_to_pool(); }; - struct idle_connection - { - std::unique_ptr conn; - config::clock::time_point idle_since; - }; - capy::io_task acquire(urls::url_view url); - void - release(pooled_connection pc); - - capy::io_task> - connect(urls::url_view url) const; - - capy::executor_ref exec_; - corosio::tls_context tls_ctx_; - std::unordered_multimap idle_; - config config_; + std::shared_ptr impl_; }; } // namespace burl diff --git a/include/boost/burl/response.hpp b/include/boost/burl/response.hpp index 22d5dcf..54901b7 100644 --- a/include/boost/burl/response.hpp +++ b/include/boost/burl/response.hpp @@ -51,6 +51,11 @@ namespace burl to the pool for reuse when it can be kept alive and the entire message has arrived. + A response remains usable after the client which + produced it is destroyed; in that case the + connection is closed upon destruction instead of + being returned to the pool. + @par Example @code auto [ec, r] = co_await c.get("https://example.com").send(); diff --git a/src/connection_pool.cpp b/src/connection_pool.cpp index 9e01247..088fbcc 100644 --- a/src/connection_pool.cpp +++ b/src/connection_pool.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include namespace boost @@ -42,85 +43,6 @@ namespace boost namespace burl { -class connection_pool::tcp_connection final : public connection -{ - corosio::tcp_socket socket_; - -public: - explicit tcp_connection(corosio::tcp_socket socket) - : socket_(std::move(socket)) - { - } - - virtual capy::io_task - read_some(std::span buffers) override - { - co_return co_await socket_.read_some(buffers); - } - - virtual capy::io_task - write_some(std::span buffers) override - { - co_return co_await socket_.write_some(buffers); - } - - capy::io_task<> - shutdown() override - { - socket_.shutdown(corosio::shutdown_both); - co_return {}; - } - - bool - is_open() override - { - return socket_.is_open(); - } -}; - -class connection_pool::tls_connection final : public connection -{ - corosio::tcp_socket socket_; - corosio::openssl_stream stream_; - -public: - tls_connection(corosio::tcp_socket socket, const corosio::tls_context& ctx) - : socket_(std::move(socket)) - , stream_(&socket_, ctx) - { - } - - virtual capy::io_task - read_some(std::span buffers) override - { - return stream_.read_some(buffers); - } - - virtual capy::io_task - write_some(std::span buffers) override - { - return stream_.write_some(buffers); - } - - capy::io_task<> - handshake() - { - return stream_.handshake(corosio::openssl_stream::client); - } - - capy::io_task<> - shutdown() override - { - return stream_.shutdown(); - } - - bool - is_open() override - { - return socket_.is_open(); - } -}; - namespace { @@ -339,123 +261,253 @@ connect_socks5_proxy( } // namespace -capy::io_task> -connection_pool::connect(urls::url_view url) const +class connection_pool::impl + : public std::enable_shared_from_this { - auto target_port = effective_port(url); - if(target_port.empty()) - co_return { error::unsupported_url_scheme, {} }; + struct idle_connection + { + std::unique_ptr conn; + config::clock::time_point idle_since; + }; + + capy::executor_ref exec_; + corosio::tls_context tls_ctx_; + std::unordered_multimap idle_; + config config_; - corosio::tcp_socket socket(exec_); +public: + impl( + capy::executor_ref exec, + corosio::tls_context tls_ctx, + config cfg) + : exec_(exec) + , tls_ctx_(std::move(tls_ctx)) + , config_(std::move(cfg)) + { + } - if(config_.proxy) + capy::io_task + acquire(urls::url_view url) { - auto const& proxy = *config_.proxy; - auto proxy_port = effective_port(proxy); - if(proxy_port.empty()) - co_return { error::unsupported_proxy_scheme, {} }; + auto key = origin(url); + auto [it, last] = idle_.equal_range(key); + while(it != last) + { + auto entry = std::move(it->second); + it = idle_.erase(it); + + if(config::clock::now() - entry.idle_since >= config_.idle_timeout) + continue; + + if(!entry.conn->is_open()) + continue; + + co_return { + {}, + { std::move(entry.conn), + weak_from_this(), + std::move(key), + config_.io_timeout } + }; + } - auto [ec] = co_await connect_tcp( - socket, exec_, config_, proxy.encoded_host(), proxy_port); + auto [ec, conn] = + co_await capy::timeout(connect(url), config_.connect_timeout); if(ec) co_return { ec, {} }; - if(proxy.scheme() == "http") + co_return { + {}, + { std::move(conn), + weak_from_this(), + std::move(key), + config_.io_timeout } + }; + } + + void + release(pooled_connection pc) + { + if(!pc.conn_ || !pc.conn_->is_open()) + return; + + if(idle_.count(pc.key_) >= config_.max_idle_per_host) + return; + + idle_.emplace( + std::move(pc.key_), + idle_connection{ std::move(pc.conn_), config::clock::now() }); + } + +private: + class tcp_connection final : public connection + { + corosio::tcp_socket socket_; + + public: + explicit tcp_connection(corosio::tcp_socket socket) + : socket_(std::move(socket)) { - auto [ec] = co_await connect_http_proxy( - socket, url.encoded_host(), target_port, proxy); - if(ec) - co_return { ec, {} }; } - else if(proxy.scheme() == "socks5" || proxy.scheme() == "socks5h") + + virtual capy::io_task + read_some(std::span buffers) override { - auto [ec] = co_await connect_socks5_proxy( - socket, url.encoded_host(), target_port, proxy); - if(ec) - co_return { ec, {} }; + co_return co_await socket_.read_some(buffers); } - else + + virtual capy::io_task + write_some(std::span buffers) override { - co_return { error::unsupported_proxy_scheme, {} }; + co_return co_await socket_.write_some(buffers); } - } - else - { - auto [ec] = co_await connect_tcp( - socket, exec_, config_, url.encoded_host(), target_port); - if(ec) - co_return { ec, {} }; - } - if(url.scheme_id() == urls::scheme::https) + capy::io_task<> + shutdown() override + { + socket_.shutdown(corosio::shutdown_both); + co_return {}; + } + + bool + is_open() override + { + return socket_.is_open(); + } + }; + + class tls_connection final : public connection { - auto tls_ctx = tls_ctx_; - tls_ctx.set_hostname(url.encoded_host()); + corosio::tcp_socket socket_; + corosio::openssl_stream stream_; + + public: + tls_connection( + corosio::tcp_socket socket, + const corosio::tls_context& ctx) + : socket_(std::move(socket)) + , stream_(&socket_, ctx) + { + } - auto conn = - std::make_unique(std::move(socket), tls_ctx); - auto [hec] = co_await conn->handshake(); - if(hec) - co_return { hec, {} }; + virtual capy::io_task + read_some(std::span buffers) override + { + return stream_.read_some(buffers); + } - co_return { {}, std::move(conn) }; - } + virtual capy::io_task + write_some(std::span buffers) override + { + return stream_.write_some(buffers); + } - co_return { {}, std::make_unique(std::move(socket)) }; -} + capy::io_task<> + handshake() + { + return stream_.handshake(corosio::openssl_stream::client); + } -capy::io_task -connection_pool::acquire(urls::url_view url) -{ - auto key = origin(url); - auto const now = config::clock::now(); + capy::io_task<> + shutdown() override + { + return stream_.shutdown(); + } - auto [it, last] = idle_.equal_range(key); - while(it != last) + bool + is_open() override + { + return socket_.is_open(); + } + }; + + capy::io_task> + connect(urls::url_view url) const { - auto entry = std::move(it->second); - it = idle_.erase(it); + auto target_port = effective_port(url); + if(target_port.empty()) + co_return { error::unsupported_url_scheme, {} }; - if(now - entry.idle_since >= config_.idle_timeout) - continue; + corosio::tcp_socket socket(exec_); - if(!entry.conn->is_open()) - continue; + if(config_.proxy) + { + auto const& proxy = *config_.proxy; + auto proxy_port = effective_port(proxy); + if(proxy_port.empty()) + co_return { error::unsupported_proxy_scheme, {} }; - co_return { - {}, { std::move(entry.conn), this, std::move(key), config_.io_timeout } - }; - } + auto [ec] = co_await connect_tcp( + socket, exec_, config_, proxy.encoded_host(), proxy_port); + if(ec) + co_return { ec, {} }; - auto [ec, conn] = - co_await capy::timeout(connect(url), config_.connect_timeout); - if(ec) - co_return { ec, {} }; + if(proxy.scheme() == "http") + { + auto [ec] = co_await connect_http_proxy( + socket, url.encoded_host(), target_port, proxy); + if(ec) + co_return { ec, {} }; + } + else if(proxy.scheme() == "socks5" || proxy.scheme() == "socks5h") + { + auto [ec] = co_await connect_socks5_proxy( + socket, url.encoded_host(), target_port, proxy); + if(ec) + co_return { ec, {} }; + } + else + { + co_return { error::unsupported_proxy_scheme, {} }; + } + } + else + { + auto [ec] = co_await connect_tcp( + socket, exec_, config_, url.encoded_host(), target_port); + if(ec) + co_return { ec, {} }; + } - co_return { - {}, { std::move(conn), this, std::move(key), config_.io_timeout } - }; -} + if(url.scheme_id() == urls::scheme::https) + { + auto tls_ctx = tls_ctx_; + tls_ctx.set_hostname(url.encoded_host()); -void -connection_pool::release(pooled_connection pc) -{ - if(!pc.conn_ || !pc.conn_->is_open()) - return; + auto conn = + std::make_unique(std::move(socket), tls_ctx); + auto [hec] = co_await conn->handshake(); + if(hec) + co_return { hec, {} }; - if(idle_.count(pc.key_) >= config_.max_idle_per_host) - return; + co_return { {}, std::move(conn) }; + } + + co_return { {}, std::make_unique(std::move(socket)) }; + } +}; - idle_.emplace( - std::move(pc.key_), - idle_connection{ std::move(pc.conn_), config::clock::now() }); +connection_pool::connection_pool( + capy::executor_ref exec, + corosio::tls_context tls_ctx, + config cfg) + : impl_( + std::make_shared( + exec, std::move(tls_ctx), std::move(cfg))) +{ +} + +capy::io_task +connection_pool::acquire(urls::url_view url) +{ + return impl_->acquire(url); } void connection_pool::pooled_connection::return_to_pool() { - if(pool_) - pool_->release(std::move(*this)); + if(auto pool = pool_.lock()) + pool->release(std::move(*this)); } } // namespace burl From b10c2f25ca3cecfdbe66a12345f5a60107bb4699 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 12 Jun 2026 15:03:00 +0000 Subject: [PATCH 6/7] client: apply a timeout when draining bodies for connection reuse --- include/boost/burl/response.hpp | 1 - src/client.cpp | 8 +++-- src/detail/drain.hpp | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 src/detail/drain.hpp diff --git a/include/boost/burl/response.hpp b/include/boost/burl/response.hpp index 54901b7..90dbe2a 100644 --- a/include/boost/burl/response.hpp +++ b/include/boost/burl/response.hpp @@ -26,7 +26,6 @@ #include #include -#include #include #include #include diff --git a/src/client.cpp b/src/client.cpp index 24bccaa..255c88d 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -11,6 +11,7 @@ #include #include "detail/base64.hpp" +#include "detail/drain.hpp" #include "detail/reuse.hpp" #include @@ -362,9 +363,10 @@ client::execute_impl( } // Read and discard small bodies so the connection can be reused - parser.set_body_limit(64 * 1024); - if(auto [ec] = co_await parser.read(conn); ec) - parser.reset(); + auto [dec] = co_await capy::timeout( + detail::drain_body(parser, conn, 1024 * 1024), + std::chrono::seconds(2)); + if(detail::can_reuse_conn(parser)) conn.return_to_pool(); diff --git a/src/detail/drain.hpp b/src/detail/drain.hpp new file mode 100644 index 0000000..40a5ff2 --- /dev/null +++ b/src/detail/drain.hpp @@ -0,0 +1,58 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_SRC_DETAIL_DRAIN_HPP +#define BOOST_BURL_SRC_DETAIL_DRAIN_HPP + +#include +#include +#include +#include + +#include + +namespace boost +{ +namespace burl +{ +namespace detail +{ + +/** Read and discard the remaining body. +*/ +template +capy::io_task<> +drain_body( + http::response_parser& parser, + Stream& conn, + std::uint64_t limit) +{ + auto source = parser.source_for(conn); + for(;;) + { + capy::const_buffer arr[2]; + auto [ec, bufs] = co_await source.pull(arr); + if(ec == capy::cond::eof) + co_return {}; + if(ec) + co_return { ec }; + + auto n = capy::buffer_size(bufs); + if(n > limit) + co_return {}; + limit -= n; + source.consume(n); + } +} + +} // namespace detail +} // namespace burl +} // namespace boost + +#endif From 0515f8e0804bfa8dd1b522da116b8a464d1a8ad8 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 12 Jun 2026 15:14:47 +0000 Subject: [PATCH 7/7] update ci.yml --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b1c5da..c657414 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,21 +142,21 @@ jobs: build-type: "Release" build-cmake: true - - compiler: "gcc" - version: "15" - cxxstd: "20" - latest-cxxstd: "20" - cxx: "g++-15" - cc: "gcc-15" - runs-on: "ubuntu-latest" - container: "ubuntu:25.04" - b2-toolset: "gcc" - is-latest: true - name: "GCC 15: C++20 (asan+ubsan)" - shared: true - asan: true - ubsan: true - build-type: "RelWithDebInfo" + # - compiler: "gcc" + # version: "15" + # cxxstd: "20" + # latest-cxxstd: "20" + # cxx: "g++-15" + # cc: "gcc-15" + # runs-on: "ubuntu-latest" + # container: "ubuntu:25.04" + # b2-toolset: "gcc" + # is-latest: true + # name: "GCC 15: C++20 (asan+ubsan)" + # shared: true + # asan: true + # ubsan: true + # build-type: "RelWithDebInfo" - compiler: "gcc" version: "13"