TLA Line data Source code
1 : //
2 : // Copyright (c) 2025 Steve Gerbino (steve@gerbino.co)
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/corosio
8 : //
9 :
10 : #ifndef BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
11 : #define BOOST_COROSIO_DETAIL_OBJECT_POOL_HPP
12 :
13 : #include <boost/corosio/detail/config.hpp>
14 : #include <boost/corosio/detail/intrusive.hpp>
15 : #include <atomic>
16 : #include <cstddef>
17 : #include <initializer_list>
18 : #include <limits>
19 : #include <mutex>
20 : #include <utility>
21 :
22 : namespace boost::corosio::detail {
23 :
24 : /** Per-service recycling pool for I/O implementations.
25 :
26 : Replaces per-impl `shared_ptr` ownership: a service news an impl
27 : once, then `recycle()` (the impl's `retire()` action) parks
28 : it on the free list and `acquire()` hands it back out, so steady-state
29 : construct/destroy allocates nothing. Unbounded; memory is released
30 : only on pool destruction. In debug builds, recycled impls are
31 : poisoned by the service's `reuse()` discipline, not here (the pool
32 : cannot know which bytes are resettable).
33 :
34 : A service tears down its impls by calling `shutdown()`, which
35 : enters shutting-down mode and visits every live impl in one
36 : critical section; subsequent zero-ref crossings delete instead of
37 : recycling.
38 :
39 : `adopt()` and `remove()` are protected ownership-transfer
40 : primitives for sanctioned extensions — a derived pool exposing
41 : them to a caller that tracks some of its live impls through its
42 : own structure instead of this pool's `live_` list (the timer
43 : service, via its expiry heap) — rather than part of the public
44 : surface every ordinary service uses.
45 :
46 : @par Thread Safety
47 : Distinct objects: Safe. Shared objects: Safe; all operations lock
48 : an internal mutex.
49 : */
50 : template<class Impl>
51 : class object_pool
52 : {
53 : std::mutex mutex_;
54 : intrusive_list<Impl> live_;
55 : intrusive_list<Impl> free_;
56 : bool shutting_down_ = false;
57 :
58 : /** Marks `refs_` on an impl currently parked in `free_`.
59 :
60 : `intrusive_list::remove()` cannot distinguish "linked in
61 : `free_`" from "linked in `live_`" by inspecting the node
62 : alone — both lists share one `next_`/`prev_` pair per impl —
63 : so `recycle()` cannot safely call `live_.remove()` on an impl
64 : it does not already know is live. Stamping this sentinel into
65 : `refs_` the moment an impl joins `free_`, and checking it
66 : before touching either list, is what lets `recycle()` and
67 : `remove()` tell the two cases apart in every build mode, not
68 : just under asserts. `acquire()` always overwrites it with 1
69 : before the impl is usable again.
70 : */
71 : static constexpr std::size_t pooled_sentinel =
72 : (std::numeric_limits<std::size_t>::max)();
73 :
74 : /// Check whether @p impl is parked in `free_` (`refs_` reads the
75 : /// sentinel). Callers must check this before any list operation —
76 : /// see the class-level comment on `pooled_sentinel`.
77 HIT 56698 : static bool is_pooled(Impl const* impl) noexcept
78 : {
79 56698 : return impl->refs_.load(std::memory_order_relaxed) ==
80 56698 : pooled_sentinel;
81 : }
82 :
83 : protected:
84 : /** Track a fresh or reset impl as live.
85 :
86 : @pre @p impl is not parked in `free_` (`refs_` must not read
87 : `pooled_sentinel`). The same list-corruption hazard
88 : `recycle()`'s class-level comment documents applies here: a
89 : free-listed impl pushed onto `live_` directly, bypassing
90 : `acquire()`, would end up linked in both lists through one shared
91 : `next_`/`prev_` pair.
92 :
93 : @pre The caller is not racing `shutdown()`. `acquire()` is
94 : this method's only caller, and a service's `construct()` —
95 : `acquire()`'s only caller — is expected to stop being called
96 : before the service's own `shutdown()` runs; this is checked
97 : only in debug builds because the failure mode is a silently
98 : orphaned impl (no close callback), not memory corruption.
99 : */
100 21531 : void adopt(Impl* impl)
101 : {
102 21531 : std::lock_guard lock(mutex_);
103 21531 : if (is_pooled(impl))
104 : {
105 MIS 0 : BOOST_COROSIO_ASSERT(false);
106 : return;
107 : }
108 HIT 21531 : BOOST_COROSIO_ASSERT(!shutting_down_);
109 21531 : live_.push_back(impl);
110 21531 : }
111 :
112 : /** Unlink a live impl without recycling (ownership transfer).
113 :
114 : Used by a caller that is taking an impl out of this pool's
115 : bookkeeping entirely (a thread-local cache slot, a direct
116 : delete during shutdown) rather than parking it on `free_`.
117 :
118 : @pre @p impl is not parked in `free_` (`refs_` must not read
119 : `pooled_sentinel`). The same list-corruption hazard
120 : `recycle()`'s class-level comment documents applies here: a
121 : free-listed impl would be spliced out using `live_`'s
122 : boundary pointers instead of `free_`'s.
123 :
124 : @return True if @p impl was linked in `live_` and has now
125 : been removed; false if it was already detached.
126 : */
127 14847 : bool remove(Impl* impl) noexcept
128 : {
129 14847 : std::lock_guard lock(mutex_);
130 14847 : if (is_pooled(impl))
131 : {
132 MIS 0 : BOOST_COROSIO_ASSERT(false);
133 : return false;
134 : }
135 HIT 14847 : return live_.remove(impl);
136 14847 : }
137 :
138 : public:
139 : /** Destroy the pool, freeing every impl still parked or live.
140 :
141 : @pre `shutdown()` has already run, or no impls are live.
142 : Without it, deleting a live impl here would let that impl's
143 : own destructor reenter `recycle()` outside the shutting-down
144 : branch that makes a reentrant retirement (a self-referential
145 : op's `object_ref` releasing its last reference while this
146 : destructor's own `delete` is still on the stack) a safe no-op
147 : — it would instead look exactly like the accounting-bug case
148 : `recycle()` deletes a second time.
149 : */
150 5075 : ~object_pool()
151 : {
152 5075 : BOOST_COROSIO_ASSERT(shutting_down_ || live_.empty());
153 15225 : for (auto* list : {&free_, &live_})
154 16498 : while (auto* impl = list->pop_front())
155 6348 : delete impl;
156 5075 : }
157 :
158 5069 : object_pool() = default;
159 : object_pool(object_pool const&) = delete;
160 : object_pool& operator=(object_pool const&) = delete;
161 :
162 : /** Pop-or-create an impl with the acquisition protocol applied.
163 :
164 : Recycled impls are `reuse()`-reset and their reference count
165 : restored from the pooled sentinel; fresh impls are constructed
166 : in place from `args` on the cold path. Either way the impl is
167 : adopted as live. This is the sequence every service's
168 : `construct()` used to hand-roll. Making the pool itself the
169 : only place that calls `new Impl` gives the whole library one
170 : audit point for allocation policy.
171 :
172 : @pre Not called concurrently with, or after, `shutdown()` on
173 : this pool. Every caller reaches this through a service's
174 : `construct()`, and service shutdown order guarantees no
175 : `construct()` outlives that service's own `shutdown()`; the
176 : window is only asserted in debug builds (see `adopt()`)
177 : because the failure mode is an orphaned impl, not corruption.
178 :
179 : @param args Constructor arguments for `Impl` (cold path only).
180 :
181 : @return The acquired impl, `refs_ == 1`, tracked live.
182 : */
183 : template<class... Args>
184 20830 : Impl* acquire(Args&&... args)
185 : {
186 : Impl* impl;
187 : {
188 20830 : std::lock_guard lock(mutex_);
189 20830 : BOOST_COROSIO_ASSERT(!shutting_down_);
190 20830 : impl = free_.pop_front();
191 20830 : if (impl)
192 : {
193 13656 : impl->refs_.store(1, std::memory_order_relaxed);
194 13656 : live_.push_back(impl);
195 : }
196 20830 : }
197 : // reuse() runs unlocked: acquire()'s precondition excludes a
198 : // concurrent shutdown() visiting the impl mid-reset.
199 20830 : if (impl)
200 : {
201 13656 : impl->reuse();
202 13656 : return impl;
203 : }
204 7174 : impl = new Impl(std::forward<Args>(args)...);
205 7174 : adopt(impl);
206 7174 : return impl;
207 : }
208 :
209 : /** Move a zero-ref impl from live to the free list.
210 :
211 : Called from `retire()`, possibly on a scheduler thread.
212 : Deletes instead when shutting down. The impl must not be
213 : touched after this returns.
214 :
215 : Two accounting bugs are possible here, neither from this
216 : pool's own bookkeeping but from a caller's refcounting (e.g.
217 : a duplicate `retire()` for one zero-crossing). Both are
218 : made safe in every build mode, not just under asserts, because
219 : neither can be told apart from the correct case by a
220 : `BOOST_COROSIO_ASSERT` alone that still runs the list
221 : operations around it:
222 :
223 : - `impl` is already parked in `free_` — the list-corruption
224 : hazard the class-level `pooled_sentinel` comment documents.
225 : The `is_pooled()` check runs first and returns before either
226 : list is touched.
227 : - `impl` is fully detached from `live_` and is not the
228 : sentinel (not shutting down): `live_.remove()` correctly
229 : returns false, but then the impl is reachable through
230 : neither `acquire()` nor `shutdown()`'s visit while still
231 : allocated — stranded. Deleted instead of left to leak.
232 :
233 : The one legitimate "already detached" case — an impl with an
234 : embedded op whose `object_ref` points back at itself reaching
235 : zero *during* `delete impl`, e.g. `~object_pool()`'s own
236 : unconditional sweep tearing down that op's `object_ref` member
237 : before the outer `delete` returns — is distinguished from the
238 : accounting-bug case by `shutting_down_`: that reentrant path
239 : only exists because `shutdown()` — called with a no-op
240 : callback by a caller like the timer service that tracks its
241 : live impls through its own structure instead of this pool's
242 : `live_` list, just to flip the flag — always precedes
243 : `~object_pool()`'s sweep (service shutdown order), so a
244 : not-live, not-pooled impl while `shutting_down_` is exactly
245 : that reentrant no-op, and the (already false) `was_live` means
246 : skip the delete — the outer frame's `delete` is still running.
247 : */
248 20320 : void recycle(Impl* impl)
249 : {
250 20320 : bool do_delete = false;
251 : {
252 20320 : std::lock_guard lock(mutex_);
253 20320 : if (is_pooled(impl))
254 : {
255 MIS 0 : BOOST_COROSIO_ASSERT(false);
256 : return;
257 : }
258 :
259 HIT 20320 : bool const was_live = live_.remove(impl);
260 20320 : if (shutting_down_)
261 : {
262 338 : do_delete = was_live;
263 : }
264 19982 : else if (was_live)
265 : {
266 19982 : impl->refs_.store(
267 : pooled_sentinel, std::memory_order_relaxed);
268 : // LIFO: the most recently retired impl is the
269 : // cache-warmest candidate for the next acquire().
270 19982 : free_.push_front(impl);
271 : }
272 : else
273 : {
274 MIS 0 : BOOST_COROSIO_ASSERT(false);
275 : do_delete = true;
276 : }
277 HIT 20320 : }
278 20320 : if (do_delete)
279 337 : delete impl;
280 : }
281 :
282 : /** Enter shutdown mode and visit every live impl.
283 :
284 : Sets shutting-down (subsequent zero-crossings delete instead
285 : of recycling) and runs `f` on each live impl, all under one
286 : critical section.
287 :
288 : @note The callback runs under the pool's non-recursive mutex;
289 : it must not call acquire(), recycle(), adopt(), or remove()
290 : on this pool.
291 : */
292 : template<class F>
293 5071 : void shutdown(F&& f)
294 : {
295 5071 : std::lock_guard lock(mutex_);
296 5071 : shutting_down_ = true;
297 5071 : live_.for_each(std::forward<F>(f));
298 5071 : }
299 : };
300 :
301 : } // namespace boost::corosio::detail
302 :
303 : #endif
|
