. /include/boost/corosio /detail /timer_service.hpp
96.85% Lines (215/222) 100.00% Functions (28/28)
[ ] prev/next file n p prev/next uncovered line
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // Copyright (c) 2026 Steve Gerbino 3   // Copyright (c) 2026 Steve Gerbino
4   // 4   //
5   // Distributed under the Boost Software License, Version 1.0. (See accompanying 5   // Distributed under the Boost Software License, Version 1.0. (See accompanying
6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7   // 7   //
8   // Official repository: https://github.com/cppalliance/corosio 8   // Official repository: https://github.com/cppalliance/corosio
9   // 9   //
10   10  
11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 11   #ifndef BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP 12   #define BOOST_COROSIO_DETAIL_TIMER_SERVICE_HPP
13   13  
14   #include <boost/corosio/detail/timer.hpp> 14   #include <boost/corosio/detail/timer.hpp>
15   #include <boost/corosio/detail/scheduler.hpp> 15   #include <boost/corosio/detail/scheduler.hpp>
16   #include <boost/corosio/detail/scheduler_op.hpp> 16   #include <boost/corosio/detail/scheduler_op.hpp>
  17 + #include <boost/corosio/detail/object_pool.hpp>
  18 + #include <boost/corosio/detail/object_ref.hpp>
17   #include <boost/corosio/detail/intrusive.hpp> 19   #include <boost/corosio/detail/intrusive.hpp>
18   #include <boost/corosio/detail/thread_local_ptr.hpp> 20   #include <boost/corosio/detail/thread_local_ptr.hpp>
19   #include <boost/capy/error.hpp> 21   #include <boost/capy/error.hpp>
20   #include <boost/capy/ex/execution_context.hpp> 22   #include <boost/capy/ex/execution_context.hpp>
21   #include <boost/capy/ex/executor_ref.hpp> 23   #include <boost/capy/ex/executor_ref.hpp>
22   #include <system_error> 24   #include <system_error>
23   25  
24   #include <atomic> 26   #include <atomic>
25   #include <chrono> 27   #include <chrono>
26   #include <coroutine> 28   #include <coroutine>
27   #include <cstddef> 29   #include <cstddef>
28   #include <limits> 30   #include <limits>
29   #include <mutex> 31   #include <mutex>
30   #include <stop_token> 32   #include <stop_token>
31   #include <utility> 33   #include <utility>
32   #include <vector> 34   #include <vector>
33   35  
34   namespace boost::corosio::detail { 36   namespace boost::corosio::detail {
35   37  
36   struct scheduler; 38   struct scheduler;
37   39  
38   /* 40   /*
39   Timer Service 41   Timer Service
40   ============= 42   =============
41   43  
42   Data Structures 44   Data Structures
43   --------------- 45   ---------------
44   waiter_node (defined in timer.hpp) holds per-waiter state: 46   waiter_node (defined in timer.hpp) holds per-waiter state:
45   coroutine handle, executor, error output, embedded 47   coroutine handle, executor, error output, embedded
46   completion_op. Each concurrent co_await t.wait() embeds one 48   completion_op. Each concurrent co_await t.wait() embeds one
47   waiter_node in the awaitable on the suspended coroutine's 49   waiter_node in the awaitable on the suspended coroutine's
48   frame — waits perform no allocation. 50   frame — waits perform no allocation.
49   51  
50   timer::implementation holds per-timer state: expiry, heap 52   timer::implementation holds per-timer state: expiry, heap
51   index, and the single published waiter. Each timer holds 53   index, and the single published waiter. Each timer holds
52   at most one waiter; process_expired's local cross-timer drain 54   at most one waiter; process_expired's local cross-timer drain
53   list still threads waiters through their intrusive hooks when 55   list still threads waiters through their intrusive hooks when
54   collecting several timers' waiters past the lock. 56   collecting several timers' waiters past the lock.
55   57  
56 - timer_service owns a min-heap of active timers and a free list 58 + timer_service owns a min-heap of active timers and recycles
57 - of recycled impls. The heap is ordered by expiry time; the 59 + retired impls through a per-service object_pool, fronted by a
58 - scheduler queries nearest_expiry() to set the epoll/timerfd 60 + thread-local single-slot cache. The heap is ordered by expiry
59 - timeout. 61 + time; the scheduler queries nearest_expiry() to set the
  62 + epoll/timerfd timeout.
60   63  
61   Optimization Strategy 64   Optimization Strategy
62   --------------------- 65   ---------------------
63   1. Deferred heap insertion — expires_after() stores the expiry 66   1. Deferred heap insertion — expires_after() stores the expiry
64   but does not insert into the heap. Insertion happens in wait(). 67   but does not insert into the heap. Insertion happens in wait().
65 - 2. Thread-local impl cache — single-slot per-thread cache. 68 + 2. Thread-local impl cache — single-slot per-thread cache, in
  69 + front of the object_pool fallback.
66   3. Frame-resident waiter_node with embedded completion_op — 70   3. Frame-resident waiter_node with embedded completion_op —
67   eliminates heap allocation per wait/fire/cancel. 71   eliminates heap allocation per wait/fire/cancel.
68   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry(). 72   4. Cached nearest expiry — atomic avoids mutex in nearest_expiry().
69   5. might_have_pending_waits_ flag — skips lock when no wait issued. 73   5. might_have_pending_waits_ flag — skips lock when no wait issued.
70   74  
71   Concurrency 75   Concurrency
72   ----------- 76   -----------
73   stop_token callbacks can fire from any thread. The impl_ 77   stop_token callbacks can fire from any thread. The impl_
74   pointer on waiter_node is used as a "still in list" marker. 78   pointer on waiter_node is used as a "still in list" marker.
75   A waiter_node's storage is the suspended coroutine's frame: 79   A waiter_node's storage is the suspended coroutine's frame:
76   every completion path must finish touching the node before 80   every completion path must finish touching the node before
77   posting the continuation or destroying the handle. 81   posting the continuation or destroying the handle.
78   */ 82   */
79   83  
80   inline void timer_service_invalidate_cache() noexcept; 84   inline void timer_service_invalidate_cache() noexcept;
81   85  
  86 + /** The timer service's recycling pool.
  87 +
  88 + Adds no behavior of its own — it exists only to re-expose
  89 + `adopt()` and `remove()` as public, so the timer service's
  90 + thread-local cache can transfer a cached impl's `live_` tracking
  91 + on both sides of the handoff: `construct()` re-adopts a
  92 + TL-cached impl as live, `shutdown()` detaches one before its
  93 + direct delete.
  94 + */
  95 + class timer_object_pool : public object_pool<timer::implementation>
  96 + {
  97 + public:
  98 + using object_pool::adopt;
  99 + using object_pool::remove;
  100 + };
  101 +
82   // timer_service class body — member function definitions are 102   // timer_service class body — member function definitions are
83   // out-of-class (after implementation and waiter_node are complete) 103   // out-of-class (after implementation and waiter_node are complete)
84   class BOOST_COROSIO_DECL timer_service final 104   class BOOST_COROSIO_DECL timer_service final
85   : public capy::execution_context::service 105   : public capy::execution_context::service
86   , public io_object::io_service 106   , public io_object::io_service
87   { 107   {
  108 + friend struct timer::implementation;
  109 +
88   public: 110   public:
89   using clock_type = std::chrono::steady_clock; 111   using clock_type = std::chrono::steady_clock;
90   using time_point = clock_type::time_point; 112   using time_point = clock_type::time_point;
91   113  
92   /// Type-erased callback for earliest-expiry-changed notifications. 114   /// Type-erased callback for earliest-expiry-changed notifications.
93   class callback 115   class callback
94   { 116   {
95   void* ctx_ = nullptr; 117   void* ctx_ = nullptr;
96   void (*fn_)(void*) = nullptr; 118   void (*fn_)(void*) = nullptr;
97   119  
98   public: 120   public:
99   /// Construct an empty callback. 121   /// Construct an empty callback.
HITCBC 100   2243 callback() = default; 122   2310 callback() = default;
101   123  
102   /// Construct a callback with the given context and function. 124   /// Construct a callback with the given context and function.
HITCBC 103   2243 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {} 125   2310 callback(void* ctx, void (*fn)(void*)) noexcept : ctx_(ctx), fn_(fn) {}
104   126  
105   /// Return true if the callback is non-empty. 127   /// Return true if the callback is non-empty.
106   explicit operator bool() const noexcept 128   explicit operator bool() const noexcept
107   { 129   {
108   return fn_ != nullptr; 130   return fn_ != nullptr;
109   } 131   }
110   132  
111   /// Invoke the callback. 133   /// Invoke the callback.
HITCBC 112   7006 void operator()() const 134   7496 void operator()() const
113   { 135   {
HITCBC 114   7006 if (fn_) 136   7496 if (fn_)
HITCBC 115   7006 fn_(ctx_); 137   7496 fn_(ctx_);
HITCBC 116   7006 } 138   7496 }
117   }; 139   };
118   140  
119   private: 141   private:
120   struct heap_entry 142   struct heap_entry
121   { 143   {
122   time_point time_; 144   time_point time_;
123   timer::implementation* timer_; 145   timer::implementation* timer_;
124   }; 146   };
125   147  
126   scheduler* sched_ = nullptr; 148   scheduler* sched_ = nullptr;
127   BOOST_COROSIO_MSVC_WARNING_PUSH 149   BOOST_COROSIO_MSVC_WARNING_PUSH
128   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface 150   BOOST_COROSIO_MSVC_WARNING_DISABLE(4251) // std:: members, dll-interface
129   mutable std::mutex mutex_; 151   mutable std::mutex mutex_;
130   std::vector<heap_entry> heap_; 152   std::vector<heap_entry> heap_;
131 - timer::implementation* free_list_ = nullptr; 153 + timer_object_pool pool_;
132   callback on_earliest_changed_; 154   callback on_earliest_changed_;
133   bool shutting_down_ = false; 155   bool shutting_down_ = false;
134   // Avoids mutex in nearest_expiry() and empty() 156   // Avoids mutex in nearest_expiry() and empty()
135   mutable std::atomic<std::int64_t> cached_nearest_ns_{ 157   mutable std::atomic<std::int64_t> cached_nearest_ns_{
136   (std::numeric_limits<std::int64_t>::max)()}; 158   (std::numeric_limits<std::int64_t>::max)()};
137   BOOST_COROSIO_MSVC_WARNING_POP 159   BOOST_COROSIO_MSVC_WARNING_POP
138   160  
139   public: 161   public:
140   /// Construct the timer service bound to a scheduler. 162   /// Construct the timer service bound to a scheduler.
HITCBC 141   2243 inline timer_service(capy::execution_context&, scheduler& sched) 163   2310 inline timer_service(capy::execution_context&, scheduler& sched)
HITCBC 142   2243 : sched_(&sched) 164   2310 : sched_(&sched)
143   { 165   {
HITCBC 144   2243 } 166   2310 }
145   167  
146   /// Return the associated scheduler. 168   /// Return the associated scheduler.
HITCBC 147   28371 inline scheduler& get_scheduler() noexcept 169   32737 inline scheduler& get_scheduler() noexcept
148   { 170   {
HITCBC 149   28371 return *sched_; 171   32737 return *sched_;
150   } 172   }
151   173  
152   /// Destroy the timer service. 174   /// Destroy the timer service.
HITCBC 153   4486 ~timer_service() override = default; 175   4620 ~timer_service() override = default;
154   176  
155   timer_service(timer_service const&) = delete; 177   timer_service(timer_service const&) = delete;
156   timer_service& operator=(timer_service const&) = delete; 178   timer_service& operator=(timer_service const&) = delete;
157   179  
158   /// Register a callback invoked when the earliest expiry changes. 180   /// Register a callback invoked when the earliest expiry changes.
HITCBC 159   2243 inline void set_on_earliest_changed(callback cb) 181   2310 inline void set_on_earliest_changed(callback cb)
160   { 182   {
HITCBC 161   2243 on_earliest_changed_ = cb; 183   2310 on_earliest_changed_ = cb;
HITCBC 162   2243 } 184   2310 }
163   185  
164   /// Return true if no timers are in the heap. 186   /// Return true if no timers are in the heap.
165   inline bool empty() const noexcept 187   inline bool empty() const noexcept
166   { 188   {
167   return cached_nearest_ns_.load(std::memory_order_acquire) == 189   return cached_nearest_ns_.load(std::memory_order_acquire) ==
168   (std::numeric_limits<std::int64_t>::max)(); 190   (std::numeric_limits<std::int64_t>::max)();
169   } 191   }
170   192  
171   /// Return the nearest timer expiry without acquiring the mutex. 193   /// Return the nearest timer expiry without acquiring the mutex.
HITCBC 172   280966 inline time_point nearest_expiry() const noexcept 194   265754 inline time_point nearest_expiry() const noexcept
173   { 195   {
HITCBC 174   280966 auto ns = cached_nearest_ns_.load(std::memory_order_acquire); 196   265754 auto ns = cached_nearest_ns_.load(std::memory_order_acquire);
HITCBC 175   280966 return time_point(time_point::duration(ns)); 197   265754 return time_point(time_point::duration(ns));
176   } 198   }
177   199  
178   /// Cancel all pending timers and free cached resources. 200   /// Cancel all pending timers and free cached resources.
179   inline void shutdown() override; 201   inline void shutdown() override;
180   202  
181   /// Construct a new timer implementation. 203   /// Construct a new timer implementation.
182   inline io_object::implementation* construct() override; 204   inline io_object::implementation* construct() override;
183   205  
184   /// Destroy a timer implementation, cancelling pending waiters. 206   /// Destroy a timer implementation, cancelling pending waiters.
185   inline void destroy(io_object::implementation* p) override; 207   inline void destroy(io_object::implementation* p) override;
186   208  
187   /// Cancel and recycle a timer implementation. 209   /// Cancel and recycle a timer implementation.
188   inline void destroy_impl(timer::implementation& impl); 210   inline void destroy_impl(timer::implementation& impl);
189   211  
190   /// Publish the timer's waiter and insert the timer into the heap. 212   /// Publish the timer's waiter and insert the timer into the heap.
191   inline void insert_waiter(timer::implementation& impl, waiter_node* w); 213   inline void insert_waiter(timer::implementation& impl, waiter_node* w);
192   214  
193   /// Cancel the timer's published waiter, if any. 215   /// Cancel the timer's published waiter, if any.
194   inline void cancel_timer(timer::implementation& impl); 216   inline void cancel_timer(timer::implementation& impl);
195   217  
196   /// Cancel one specific waiter ( stop_token callback path ). 218   /// Cancel one specific waiter ( stop_token callback path ).
197   inline void cancel_waiter(waiter_node* w); 219   inline void cancel_waiter(waiter_node* w);
198   220  
199   /// Complete all waiters whose timers have expired. 221   /// Complete all waiters whose timers have expired.
200   inline std::size_t process_expired(); 222   inline std::size_t process_expired();
201   223  
202   private: 224   private:
HITCBC 203   322042 inline void refresh_cached_nearest() noexcept 225   313523 inline void refresh_cached_nearest() noexcept
204   { 226   {
HITCBC 205   322042 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)() 227   313523 auto ns = heap_.empty() ? (std::numeric_limits<std::int64_t>::max)()
HITCBC 206   316271 : heap_[0].time_.time_since_epoch().count(); 228   306716 : heap_[0].time_.time_since_epoch().count();
HITCBC 207   322042 cached_nearest_ns_.store(ns, std::memory_order_release); 229   313523 cached_nearest_ns_.store(ns, std::memory_order_release);
HITCBC 208   322042 } 230   313523 }
209   231  
210   inline void remove_timer_impl(timer::implementation& impl); 232   inline void remove_timer_impl(timer::implementation& impl);
211   inline void up_heap(std::size_t index); 233   inline void up_heap(std::size_t index);
212   inline void down_heap(std::size_t index); 234   inline void down_heap(std::size_t index);
213   inline void swap_heap(std::size_t i1, std::size_t i2); 235   inline void swap_heap(std::size_t i1, std::size_t i2);
214   }; 236   };
215   237  
216   // Thread-local cache avoids hot-path mutex acquisitions: 238   // Thread-local cache avoids hot-path mutex acquisitions:
217   // single-slot impl cache, validated by comparing svc_. Cleared by 239   // single-slot impl cache, validated by comparing svc_. Cleared by
218   // timer_service_invalidate_cache() during shutdown. 240   // timer_service_invalidate_cache() during shutdown.
219   241  
220   inline thread_local_ptr<timer::implementation> tl_cached_impl; 242   inline thread_local_ptr<timer::implementation> tl_cached_impl;
221   243  
222   // The POD TLS slot above never runs destructors, so a short-lived 244   // The POD TLS slot above never runs destructors, so a short-lived
223   // run() thread would leak its cached impl. Each push arms this 245   // run() thread would leak its cached impl. Each push arms this
224   // owner, whose destructor frees the slot at thread exit. A cached 246   // owner, whose destructor frees the slot at thread exit. A cached
225   // entry is a quiescent heap object (nothing in the heap or free 247   // entry is a quiescent heap object (nothing in the heap or free
226   // list) and deletion touches no service state, so it is safe after 248   // list) and deletion touches no service state, so it is safe after
227   // the owning service is gone (the stale-entry path in 249   // the owning service is gone (the stale-entry path in
228   // try_pop_tl_cache deletes the same way). 250   // try_pop_tl_cache deletes the same way).
229   struct tl_cache_owner 251   struct tl_cache_owner
230   { 252   {
HITCBC 231   42 ~tl_cache_owner() 253   64 ~tl_cache_owner()
232   { 254   {
HITCBC 233   42 delete tl_cached_impl.get(); 255   64 delete tl_cached_impl.get();
HITCBC 234   42 tl_cached_impl.set(nullptr); 256   64 tl_cached_impl.set(nullptr);
HITCBC 235   42 } 257   64 }
236   }; 258   };
237   259  
238   inline void 260   inline void
HITCBC 239   13884 arm_tl_cache_cleanup() noexcept 261   14815 arm_tl_cache_cleanup() noexcept
240   { 262   {
HITCBC 241   13884 [[maybe_unused]] thread_local tl_cache_owner owner; 263   14815 [[maybe_unused]] thread_local tl_cache_owner owner;
HITCBC 242   13884 } 264   14815 }
243   265  
244   inline timer::implementation* 266   inline timer::implementation*
HITCBC 245   15055 try_pop_tl_cache(timer_service* svc) noexcept 267   17235 try_pop_tl_cache(timer_service* svc) noexcept
246   { 268   {
HITCBC 247   15055 auto* impl = tl_cached_impl.get(); 269   17235 auto* impl = tl_cached_impl.get();
HITCBC 248   15055 if (impl) 270   17235 if (impl)
249   { 271   {
HITCBC 250   13471 tl_cached_impl.set(nullptr); 272   14357 tl_cached_impl.set(nullptr);
HITCBC 251   13471 if (impl->svc_ == svc) 273   14357 if (impl->svc_ == svc)
HITCBC 252   13471 return impl; 274   14357 return impl;
253   // Stale impl from a destroyed service 275   // Stale impl from a destroyed service
MISUBC 254   ✗ delete impl; 276   ✗ delete impl;
255   } 277   }
HITCBC 256   1584 return nullptr; 278   2878 return nullptr;
257   } 279   }
258   280  
259   inline bool 281   inline bool
HITCBC 260   15026 try_push_tl_cache(timer::implementation* impl) noexcept 282   17206 try_push_tl_cache(timer::implementation* impl) noexcept
261   { 283   {
HITCBC 262   15026 if (!tl_cached_impl.get()) 284   17206 if (!tl_cached_impl.get())
263   { 285   {
HITCBC 264   13884 arm_tl_cache_cleanup(); 286   14815 arm_tl_cache_cleanup();
HITCBC 265   13884 tl_cached_impl.set(impl); 287   14815 tl_cached_impl.set(impl);
HITCBC 266   13884 return true; 288   14815 return true;
267   } 289   }
HITCBC 268   1142 return false; 290   2391 return false;
269   } 291   }
270   292  
271   inline void 293   inline void
HITCBC 272   2243 timer_service_invalidate_cache() noexcept 294   2310 timer_service_invalidate_cache() noexcept
273   { 295   {
HITCBC 274   2243 delete tl_cached_impl.get(); 296   2310 delete tl_cached_impl.get();
HITCBC 275   2243 tl_cached_impl.set(nullptr); 297   2310 tl_cached_impl.set(nullptr);
HITCBC 276   2243 } 298   2310 }
277   299  
278   // timer_service out-of-class member function definitions 300   // timer_service out-of-class member function definitions
279   301  
280   inline void 302   inline void
HITCBC 281   2243 timer_service::shutdown() 303   2310 timer_service::shutdown()
282   { 304   {
HITCBC 283   2243 timer_service_invalidate_cache(); 305   2310 timer_service_invalidate_cache();
HITCBC 284   2243 shutting_down_ = true; 306   2310 shutting_down_ = true;
  307 + // Flag only; the real drain walks the expiry heap below, not
  308 + // object_pool's live_ list, so the visit callback is a no-op.
HITGNC   309 + 2310 pool_.shutdown([](timer::implementation*) {});
285   310  
286   // Snapshot impls and detach them from the heap so that 311   // Snapshot impls and detach them from the heap so that
287   // coroutine-owned timer destructors (triggered by h.destroy() 312   // coroutine-owned timer destructors (triggered by h.destroy()
288   // below) cannot re-enter remove_timer_impl() and mutate the 313   // below) cannot re-enter remove_timer_impl() and mutate the
289   // vector during iteration. 314   // vector during iteration.
HITCBC 290   2243 std::vector<timer::implementation*> impls; 315   2310 std::vector<timer::implementation*> impls;
HITCBC 291   2243 impls.reserve(heap_.size()); 316   2310 impls.reserve(heap_.size());
HITCBC 292   2272 for (auto& entry : heap_) 317   2339 for (auto& entry : heap_)
293   { 318   {
HITCBC 294   29 entry.timer_->heap_index_.store( 319   29 entry.timer_->heap_index_.store(
295   (std::numeric_limits<std::size_t>::max)(), 320   (std::numeric_limits<std::size_t>::max)(),
296   std::memory_order_relaxed); 321   std::memory_order_relaxed);
HITCBC 297   29 impls.push_back(entry.timer_); 322   29 impls.push_back(entry.timer_);
298   } 323   }
HITCBC 299   2243 heap_.clear(); 324   2310 heap_.clear();
HITCBC 300   2243 cached_nearest_ns_.store( 325   2310 cached_nearest_ns_.store(
301   (std::numeric_limits<std::int64_t>::max)(), std::memory_order_release); 326   (std::numeric_limits<std::int64_t>::max)(), std::memory_order_release);
302   327  
303   // Cancel waiting timers. Each waiter called work_started() 328   // Cancel waiting timers. Each waiter called work_started()
304   // in implementation::wait(). On IOCP the scheduler shutdown 329   // in implementation::wait(). On IOCP the scheduler shutdown
305   // loop exits when outstanding_work_ reaches zero, so we must 330   // loop exits when outstanding_work_ reaches zero, so we must
306   // call work_finished() here to balance it. On other backends 331   // call work_finished() here to balance it. On other backends
307   // this is harmless. 332   // this is harmless.
HITCBC 308   2272 for (auto* impl : impls) 333   2339 for (auto* impl : impls)
309   { 334   {
HITCBC 310   29 if (auto* w = std::exchange(impl->waiter_, nullptr)) 335   29 if (auto* w = std::exchange(impl->waiter_, nullptr))
311   { 336   {
HITCBC 312   29 w->reset_stop_cb(); 337   29 w->reset_stop_cb();
HITCBC 313   29 auto h = std::exchange(w->h_, {}); 338   29 auto h = std::exchange(w->h_, {});
HITCBC 314   29 sched_->work_finished(); 339   29 sched_->work_finished();
315   // Destroying the frame also ends the node's storage 340   // Destroying the frame also ends the node's storage
HITCBC 316   29 if (h) 341   29 if (h)
HITCBC 317   29 h.destroy(); 342   29 h.destroy();
318   } 343   }
  344 + // Unlink from the pool's live_ list before the direct delete
  345 + // below, or ~object_pool()'s unconditional sweep would delete
  346 + // this impl a second time.
HITGNC   347 + 29 [[maybe_unused]] bool const removed = pool_.remove(impl);
HITGNC   348 + 29 BOOST_COROSIO_ASSERT(removed);
HITCBC 319   29 delete impl; 349   29 delete impl;
320   } 350   }
321   351  
322 - // Delete free-listed impls 352 + // Anything still parked in pool_ (free-listed or live with no
ECB 323 - 3383 while (free_list_) 353 + // pending wait at shutdown) is freed by object_pool's own destructor.
324 - {  
DCB 325 - 1140 auto* next = free_list_->next_free_;  
DCB 326 - 1140 delete free_list_;  
DCB 327 - 1140 free_list_ = next;  
328 - }  
HITCBC 329   2243 } 354   2310 }
330   355  
331   inline io_object::implementation* 356   inline io_object::implementation*
HITCBC 332   15055 timer_service::construct() 357   17235 timer_service::construct()
333   { 358   {
HITCBC 334   15055 timer::implementation* impl = try_pop_tl_cache(this); 359   17235 timer::implementation* impl = try_pop_tl_cache(this);
HITCBC 335   15055 if (impl) 360   17235 if (impl)
336   { 361   {
HITCBC 337   13471 impl->svc_ = this; 362   14357 impl->svc_ = this;
HITGIC 338 - // Reset expiry_ too: a recycled impl must behave like a fresh 363 + 14357 impl->reuse();
HITGIC 339 - // one, whose default expiry reads as already elapsed 364 + 14357 impl->refs_.store(1, std::memory_order_relaxed);
ECB 340 - 13471 impl->expiry_ = {}; 365 + // The TL push that parked impl here also removed it from
ECB 341 - 13471 impl->heap_index_.store( 366 + // live_ (see destroy_impl) — re-adopt it now that it is live
342 - (std::numeric_limits<std::size_t>::max)(), 367 + // again.
HITGIC 343 - std::memory_order_relaxed); 368 + 14357 pool_.adopt(impl);
DCB 344 - 13471 impl->might_have_pending_waits_.store(false, std::memory_order_relaxed);  
DCB 345 - 13471 BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr);  
HITCBC 346   13471 return impl; 369   14357 return impl;
347   } 370   }
348   371  
HITCBC 349 - 1584 std::lock_guard lock(mutex_); 372 + 2878 return pool_.acquire(*this);
DCB 350 - 1584 if (free_list_)  
351 - {  
DCB 352 - 2 impl = free_list_;  
DCB 353 - 2 free_list_ = impl->next_free_;  
DCB 354 - 2 impl->next_free_ = nullptr;  
DCB 355 - 2 impl->svc_ = this;  
DCB 356 - 2 impl->expiry_ = {};  
DCB 357 - 2 impl->heap_index_.store(  
358 - (std::numeric_limits<std::size_t>::max)(),  
359 - std::memory_order_relaxed);  
DCB 360 - 2 impl->might_have_pending_waits_.store(false, std::memory_order_relaxed);  
DCB 361 - 2 BOOST_COROSIO_ASSERT(impl->waiter_ == nullptr);  
362 - }  
363 - else  
364 - {  
DCB 365 - 1582 impl = new timer::implementation(*this);  
366 - }  
DCB 367 - 1584 return impl;  
ECB 368   1584 } 373   }
369   374  
370   inline void 375   inline void
HITCBC 371   15055 timer_service::destroy(io_object::implementation* p) 376   17235 timer_service::destroy(io_object::implementation* p)
372   { 377   {
373   // During shutdown the drain loop owns every impl and deletes 378   // During shutdown the drain loop owns every impl and deletes
374   // them directly. A frame destroyed by that loop can unwind a 379   // them directly. A frame destroyed by that loop can unwind a
375   // handle whose impl was freed in an earlier iteration (a 380   // handle whose impl was freed in an earlier iteration (a
376   // timeout's parent frame owns the timeout timer while 381   // timeout's parent frame owns the timeout timer while
377   // suspended on the inner delay's timer), so bail out before 382   // suspended on the inner delay's timer), so bail out before
378   // even downcasting the pointer. 383   // even downcasting the pointer.
HITCBC 379   15055 if (shutting_down_) 384   17235 if (shutting_down_)
HITCBC 380   29 return; 385   29 return;
HITCBC 381   15026 destroy_impl(static_cast<timer::implementation&>(*p)); 386   17206 destroy_impl(static_cast<timer::implementation&>(*p));
382   } 387   }
383   388  
384   inline void 389   inline void
HITCBC 385   15026 timer_service::destroy_impl(timer::implementation& impl) 390   17206 timer_service::destroy_impl(timer::implementation& impl)
386   { 391   {
387   // During shutdown the impl is owned by the shutdown loop. 392   // During shutdown the impl is owned by the shutdown loop.
388   // Re-entering here (from a coroutine-owned timer destructor 393   // Re-entering here (from a coroutine-owned timer destructor
389   // triggered by h.destroy()) must not modify the heap or 394   // triggered by h.destroy()) must not modify the heap or
390   // recycle the impl — shutdown deletes it directly. 395   // recycle the impl — shutdown deletes it directly.
HITCBC 391   15026 if (shutting_down_) 396   17206 if (shutting_down_)
MISLBC 392   13884 return; 397   ✗ return;
393   398  
HITCBC 394   15026 cancel_timer(impl); 399   17206 cancel_timer(impl);
395   400  
HITCBC 396   30052 if (impl.heap_index_.load(std::memory_order_relaxed) != 401   34412 if (impl.heap_index_.load(std::memory_order_relaxed) !=
HITCBC 397   15026 (std::numeric_limits<std::size_t>::max)()) 402   17206 (std::numeric_limits<std::size_t>::max)())
398   { 403   {
MISUBC 399   ✗ std::lock_guard lock(mutex_); 404   ✗ std::lock_guard lock(mutex_);
MISUBC 400   ✗ remove_timer_impl(impl); 405   ✗ remove_timer_impl(impl);
MISUBC 401   ✗ refresh_cached_nearest(); 406   ✗ refresh_cached_nearest();
MISUBC 402   ✗ } 407   ✗ }
403   408  
HITCBC 404   15026 if (try_push_tl_cache(&impl)) 409   17206 if (try_push_tl_cache(&impl))
  410 + {
  411 + // The TL slot takes sole ownership: it is deleted directly by
  412 + // tl_cache_owner / the stale-service path, never through this
  413 + // pool, so it must leave live_ now or ~object_pool() would see
  414 + // it as abandoned (and, at the wrong moment, double-delete it
  415 + // after a later construct() re-adopts it).
HITGNC   416 + 14815 [[maybe_unused]] bool const removed = pool_.remove(&impl);
HITGNC   417 + 14815 BOOST_COROSIO_ASSERT(removed);
HITCBC 405   13884 return; 418   14815 return;
  419 + }
406   420  
ECB 407 - 1142 std::lock_guard lock(mutex_); 421 + // No op keepalives on a timer impl: this is the only reference,
ECB 408 - 1142 impl.next_free_ = free_list_; 422 + // so release() always recycles immediately via retire().
HITCBC 409 - 1142 free_list_ = &impl; 423 + 2391 release(&impl);
ECB 410   1142 } 424   }
411   425  
412   inline void 426   inline void
HITCBC 413   18016 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w) 427   22780 timer_service::insert_waiter(timer::implementation& impl, waiter_node* w)
414   { 428   {
HITCBC 415   18016 bool notify = false; 429   22780 bool notify = false;
HITCBC 416   18016 bool lost_cancel = false; 430   22780 bool lost_cancel = false;
417   { 431   {
HITCBC 418   18016 std::lock_guard lock(mutex_); 432   22780 std::lock_guard lock(mutex_);
419   // Grow before publishing anything, so the push_back below 433   // Grow before publishing anything, so the push_back below
420   // cannot throw: a failure here leaves the waiter untouched, 434   // cannot throw: a failure here leaves the waiter untouched,
421   // the strong guarantee rearm_wait's recovery relies on. 435   // the strong guarantee rearm_wait's recovery relies on.
HITCBC 422   18016 if (impl.heap_index_.load(std::memory_order_relaxed) == 436   22780 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 423   36032 (std::numeric_limits<std::size_t>::max)() && 437   45560 (std::numeric_limits<std::size_t>::max)() &&
HITCBC 424   18016 heap_.size() == heap_.capacity()) 438   22780 heap_.size() == heap_.capacity())
HITCBC 425   428 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity()); 439   499 heap_.reserve(heap_.capacity() == 0 ? 16 : 2 * heap_.capacity());
426   // Publish: from here the waiter is visible to the fire path and 440   // Publish: from here the waiter is visible to the fire path and
427   // to its own stop callback (impl_ non-null enables cancel_waiter). 441   // to its own stop callback (impl_ non-null enables cancel_waiter).
HITCBC 428   18016 w->impl_ = &impl; 442   22780 w->impl_ = &impl;
HITCBC 429   36032 if (impl.heap_index_.load(std::memory_order_relaxed) == 443   45560 if (impl.heap_index_.load(std::memory_order_relaxed) ==
HITCBC 430   18016 (std::numeric_limits<std::size_t>::max)()) 444   22780 (std::numeric_limits<std::size_t>::max)())
431   { 445   {
HITCBC 432   18016 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed); 446   22780 impl.heap_index_.store(heap_.size(), std::memory_order_relaxed);
HITCBC 433   18016 heap_.push_back({impl.expiry_, &impl}); 447   22780 heap_.push_back({impl.expiry_, &impl});
HITCBC 434   18016 up_heap(heap_.size() - 1); 448   22780 up_heap(heap_.size() - 1);
HITCBC 435   18016 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0); 449   22780 notify = (impl.heap_index_.load(std::memory_order_relaxed) == 0);
HITCBC 436   18016 refresh_cached_nearest(); 450   22780 refresh_cached_nearest();
437   } 451   }
HITCBC 438   18016 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr); 452   22780 BOOST_COROSIO_ASSERT(impl.waiter_ == nullptr);
HITCBC 439   18016 impl.waiter_ = w; 453   22780 impl.waiter_ = w;
440   454  
441   // Lost-cancel re-check: a stop requested after the canceller was 455   // Lost-cancel re-check: a stop requested after the canceller was
442   // armed in wait() but before this publication found impl_ null 456   // armed in wait() but before this publication found impl_ null
443   // and returned a no-op. Observe it now and undo the insertion. 457   // and returned a no-op. Observe it now and undo the insertion.
HITCBC 444   18016 if (w->token_->stop_requested()) 458   22780 if (w->token_->stop_requested())
445   { 459   {
HITCBC 446   3 w->impl_ = nullptr; 460   477 w->impl_ = nullptr;
HITCBC 447   3 impl.waiter_ = nullptr; 461   477 impl.waiter_ = nullptr;
HITCBC 448   3 remove_timer_impl(impl); 462   477 remove_timer_impl(impl);
HITCBC 449   3 impl.might_have_pending_waits_.store( 463   477 impl.might_have_pending_waits_.store(
450   false, std::memory_order_relaxed); 464   false, std::memory_order_relaxed);
HITCBC 451   3 refresh_cached_nearest(); 465   477 refresh_cached_nearest();
HITCBC 452   3 lost_cancel = true; 466   477 lost_cancel = true;
HITCBC 453   3 notify = false; // insertion undone; nearest unchanged 467   477 notify = false; // insertion undone; nearest unchanged
454   } 468   }
HITCBC 455   18016 } 469   22780 }
HITCBC 456   18016 if (notify) 470   22780 if (notify)
HITCBC 457   7006 on_earliest_changed_(); 471   7496 on_earliest_changed_();
HITCBC 458   18016 if (lost_cancel) 472   22780 if (lost_cancel)
459   { 473   {
HITCBC 460   3 w->ec_ = make_error_code(capy::error::canceled); 474   477 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 461   3 sched_->post(&w->op_); 475   477 sched_->post(&w->op_);
462   } 476   }
HITCBC 463   18016 } 477   22780 }
464   478  
465   inline void 479   inline void
HITCBC 466   15026 timer_service::cancel_timer(timer::implementation& impl) 480   17206 timer_service::cancel_timer(timer::implementation& impl)
467   { 481   {
HITCBC 468   15026 if (!impl.might_have_pending_waits_.load(std::memory_order_relaxed)) 482   17206 if (!impl.might_have_pending_waits_.load(std::memory_order_relaxed))
HITCBC 469   15024 return; 483   17204 return;
470   484  
471   // No unlocked already-done fast-out here: it would need the 485   // No unlocked already-done fast-out here: it would need the
472   // non-atomic waiter_ (a race with concurrent drains), and an 486   // non-atomic waiter_ (a race with concurrent drains), and an
473   // index-only check is lifetime-unsafe because npos is stored 487   // index-only check is lifetime-unsafe because npos is stored
474   // before the drain finishes touching the impl. A stale-true 488   // before the drain finishes touching the impl. A stale-true
475   // flag is rare with the stateless API; the locked path below 489   // flag is rare with the stateless API; the locked path below
476   // re-validates. 490   // re-validates.
477   491  
HITCBC 478   2 waiter_node* canceled = nullptr; 492   2 waiter_node* canceled = nullptr;
479   493  
480   { 494   {
HITCBC 481   2 std::lock_guard lock(mutex_); 495   2 std::lock_guard lock(mutex_);
HITCBC 482   2 remove_timer_impl(impl); 496   2 remove_timer_impl(impl);
HITCBC 483   2 canceled = std::exchange(impl.waiter_, nullptr); 497   2 canceled = std::exchange(impl.waiter_, nullptr);
HITCBC 484   2 if (canceled) 498   2 if (canceled)
HITCBC 485   2 canceled->impl_ = nullptr; 499   2 canceled->impl_ = nullptr;
486   // Store false as the final touch of the impl under the lock so 500   // Store false as the final touch of the impl under the lock so
487   // a pre-lock false-flag check trusts it unqualified. 501   // a pre-lock false-flag check trusts it unqualified.
HITCBC 488   2 impl.might_have_pending_waits_.store(false, std::memory_order_relaxed); 502   2 impl.might_have_pending_waits_.store(false, std::memory_order_relaxed);
HITCBC 489   2 refresh_cached_nearest(); 503   2 refresh_cached_nearest();
HITCBC 490   2 } 504   2 }
491   505  
HITCBC 492   2 if (canceled) 506   2 if (canceled)
493   { 507   {
HITCBC 494   2 canceled->ec_ = make_error_code(capy::error::canceled); 508   2 canceled->ec_ = make_error_code(capy::error::canceled);
HITCBC 495   2 sched_->post(&canceled->op_); 509   2 sched_->post(&canceled->op_);
496   } 510   }
497   } 511   }
498   512  
499   inline void 513   inline void
HITCBC 500   1642 timer_service::cancel_waiter(waiter_node* w) 514   2689 timer_service::cancel_waiter(waiter_node* w)
501   { 515   {
502   { 516   {
HITCBC 503   1642 std::lock_guard lock(mutex_); 517   2689 std::lock_guard lock(mutex_);
504   // Already removed by another drain: cancel_timer, 518   // Already removed by another drain: cancel_timer,
505   // process_expired, or insert_waiter's lost-cancel recheck 519   // process_expired, or insert_waiter's lost-cancel recheck
HITCBC 506   1642 if (!w->impl_) 520   2689 if (!w->impl_)
HITCBC 507   3 return; 521   506 return;
HITCBC 508   1639 auto* impl = w->impl_; 522   2183 auto* impl = w->impl_;
HITCBC 509   1639 w->impl_ = nullptr; 523   2183 w->impl_ = nullptr;
HITCBC 510   1639 impl->waiter_ = nullptr; 524   2183 impl->waiter_ = nullptr;
HITCBC 511   1639 remove_timer_impl(*impl); 525   2183 remove_timer_impl(*impl);
HITCBC 512   1639 impl->might_have_pending_waits_.store(false, std::memory_order_relaxed); 526   2183 impl->might_have_pending_waits_.store(false, std::memory_order_relaxed);
HITCBC 513   1639 refresh_cached_nearest(); 527   2183 refresh_cached_nearest();
HITCBC 514   1642 } 528   2689 }
515   529  
HITCBC 516   1639 w->ec_ = make_error_code(capy::error::canceled); 530   2183 w->ec_ = make_error_code(capy::error::canceled);
HITCBC 517   1639 sched_->post(&w->op_); 531   2183 sched_->post(&w->op_);
518   } 532   }
519   533  
520   inline std::size_t 534   inline std::size_t
HITCBC 521   302382 timer_service::process_expired() 535   288081 timer_service::process_expired()
522   { 536   {
HITCBC 523   302382 intrusive_list<waiter_node> expired; 537   288081 intrusive_list<waiter_node> expired;
524   538  
525   { 539   {
HITCBC 526   302382 std::lock_guard lock(mutex_); 540   288081 std::lock_guard lock(mutex_);
HITCBC 527   302382 auto now = clock_type::now(); 541   288081 auto now = clock_type::now();
528   542  
HITCBC 529   318725 while (!heap_.empty() && heap_[0].time_ <= now) 543   308170 while (!heap_.empty() && heap_[0].time_ <= now)
530   { 544   {
HITCBC 531   16343 timer::implementation* t = heap_[0].timer_; 545   20089 timer::implementation* t = heap_[0].timer_;
HITCBC 532   16343 remove_timer_impl(*t); 546   20089 remove_timer_impl(*t);
HITCBC 533   16343 if (auto* w = std::exchange(t->waiter_, nullptr)) 547   20089 if (auto* w = std::exchange(t->waiter_, nullptr))
534   { 548   {
HITCBC 535   16343 w->impl_ = nullptr; 549   20089 w->impl_ = nullptr;
HITCBC 536   16343 w->ec_ = {}; 550   20089 w->ec_ = {};
HITCBC 537   16343 expired.push_back(w); 551   20089 expired.push_back(w);
538   } 552   }
HITCBC 539   16343 t->might_have_pending_waits_.store( 553   20089 t->might_have_pending_waits_.store(
540   false, std::memory_order_relaxed); 554   false, std::memory_order_relaxed);
541   } 555   }
542   556  
HITCBC 543   302382 refresh_cached_nearest(); 557   288081 refresh_cached_nearest();
HITCBC 544   302382 } 558   288081 }
545   559  
HITCBC 546   302382 std::size_t count = 0; 560   288081 std::size_t count = 0;
HITCBC 547   318725 while (auto* w = expired.pop_front()) 561   308170 while (auto* w = expired.pop_front())
548   { 562   {
HITCBC 549   16343 sched_->post(&w->op_); 563   20089 sched_->post(&w->op_);
HITCBC 550   16343 ++count; 564   20089 ++count;
HITCBC 551   16343 } 565   20089 }
552   566  
HITCBC 553   302382 return count; 567   288081 return count;
554   } 568   }
555   569  
556   inline void 570   inline void
HITCBC 557   17987 timer_service::remove_timer_impl(timer::implementation& impl) 571   22751 timer_service::remove_timer_impl(timer::implementation& impl)
558   { 572   {
HITCBC 559   17987 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed); 573   22751 std::size_t index = impl.heap_index_.load(std::memory_order_relaxed);
HITCBC 560   17987 if (index >= heap_.size()) 574   22751 if (index >= heap_.size())
MISUBC 561   ✗ return; // Not in heap 575   ✗ return; // Not in heap
562   576  
HITCBC 563   17987 if (index == heap_.size() - 1) 577   22751 if (index == heap_.size() - 1)
564   { 578   {
565   // Last element, just pop 579   // Last element, just pop
HITCBC 566   2416 impl.heap_index_.store( 580   3233 impl.heap_index_.store(
567   (std::numeric_limits<std::size_t>::max)(), 581   (std::numeric_limits<std::size_t>::max)(),
568   std::memory_order_relaxed); 582   std::memory_order_relaxed);
HITCBC 569   2416 heap_.pop_back(); 583   3233 heap_.pop_back();
570   } 584   }
571   else 585   else
572   { 586   {
573   // Swap with last and reheapify 587   // Swap with last and reheapify
HITCBC 574   15571 swap_heap(index, heap_.size() - 1); 588   19518 swap_heap(index, heap_.size() - 1);
HITCBC 575   15571 impl.heap_index_.store( 589   19518 impl.heap_index_.store(
576   (std::numeric_limits<std::size_t>::max)(), 590   (std::numeric_limits<std::size_t>::max)(),
577   std::memory_order_relaxed); 591   std::memory_order_relaxed);
HITCBC 578   15571 heap_.pop_back(); 592   19518 heap_.pop_back();
579   593  
HITCBC 580   15571 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_) 594   19518 if (index > 0 && heap_[index].time_ < heap_[(index - 1) / 2].time_)
HITCBC 581   1 up_heap(index); 595   3 up_heap(index);
582   else 596   else
HITCBC 583   15570 down_heap(index); 597   19515 down_heap(index);
584   } 598   }
585   } 599   }
586   600  
587   inline void 601   inline void
HITCBC 588   18017 timer_service::up_heap(std::size_t index) 602   22783 timer_service::up_heap(std::size_t index)
589   { 603   {
HITCBC 590   25246 while (index > 0) 604   30601 while (index > 0)
591   { 605   {
HITCBC 592   18237 std::size_t parent = (index - 1) / 2; 606   22738 std::size_t parent = (index - 1) / 2;
HITCBC 593   18237 if (!(heap_[index].time_ < heap_[parent].time_)) 607   22738 if (!(heap_[index].time_ < heap_[parent].time_))
HITCBC 594   11008 break; 608   14920 break;
HITCBC 595   7229 swap_heap(index, parent); 609   7818 swap_heap(index, parent);
HITCBC 596   7229 index = parent; 610   7818 index = parent;
597   } 611   }
HITCBC 598   18017 } 612   22783 }
599   613  
600   inline void 614   inline void
HITCBC 601   15570 timer_service::down_heap(std::size_t index) 615   19515 timer_service::down_heap(std::size_t index)
602   { 616   {
HITCBC 603   15570 std::size_t child = index * 2 + 1; 617   19515 std::size_t child = index * 2 + 1;
HITCBC 604   33005 while (child < heap_.size()) 618   46071 while (child < heap_.size())
605   { 619   {
HITCBC 606   19216 std::size_t min_child = (child + 1 == heap_.size() || 620   29555 std::size_t min_child = (child + 1 == heap_.size() ||
HITCBC 607   15694 heap_[child].time_ < heap_[child + 1].time_) 621   25385 heap_[child].time_ < heap_[child + 1].time_)
HITCBC 608   34910 ? child 622   54940 ? child
HITCBC 609   19216 : child + 1; 623   29555 : child + 1;
610   624  
HITCBC 611   19216 if (heap_[index].time_ < heap_[min_child].time_) 625   29555 if (heap_[index].time_ < heap_[min_child].time_)
HITCBC 612   1781 break; 626   2999 break;
613   627  
HITCBC 614   17435 swap_heap(index, min_child); 628   26556 swap_heap(index, min_child);
HITCBC 615   17435 index = min_child; 629   26556 index = min_child;
HITCBC 616   17435 child = index * 2 + 1; 630   26556 child = index * 2 + 1;
617   } 631   }
HITCBC 618   15570 } 632   19515 }
619   633  
620   inline void 634   inline void
HITCBC 621   40235 timer_service::swap_heap(std::size_t i1, std::size_t i2) 635   53892 timer_service::swap_heap(std::size_t i1, std::size_t i2)
622   { 636   {
HITCBC 623   40235 heap_entry tmp = heap_[i1]; 637   53892 heap_entry tmp = heap_[i1];
HITCBC 624   40235 heap_[i1] = heap_[i2]; 638   53892 heap_[i1] = heap_[i2];
HITCBC 625   40235 heap_[i2] = tmp; 639   53892 heap_[i2] = tmp;
HITCBC 626   40235 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed); 640   53892 heap_[i1].timer_->heap_index_.store(i1, std::memory_order_relaxed);
HITCBC 627   40235 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed); 641   53892 heap_[i2].timer_->heap_index_.store(i2, std::memory_order_relaxed);
HITCBC 628   40235 } 642   53892 }
629   643  
630   // waiter_node's completion_op and canceller members are defined in 644   // waiter_node's completion_op and canceller members are defined in
631   // timer.cpp alongside implementation::wait(), for the same reason 645   // timer.cpp alongside implementation::wait(), for the same reason
632   // wait() lives there (see below). 646   // wait() lives there (see below).
633   647  
634   // timer::implementation::wait() is defined in timer.cpp, not here. 648   // timer::implementation::wait() is defined in timer.cpp, not here.
635   // It must be a non-inline definition in a translation unit that is 649   // It must be a non-inline definition in a translation unit that is
636   // always pulled into the link whenever detail::timer is used (every 650   // always pulled into the link whenever detail::timer is used (every
637   // consumer needs timer's constructors from that same object file). 651   // consumer needs timer's constructors from that same object file).
638   // An inline definition in this header would only be emitted in 652   // An inline definition in this header would only be emitted in
639   // translation units that happen to also include this header, which 653   // translation units that happen to also include this header, which
640   // is not guaranteed for every caller of wait_awaitable::await_suspend 654   // is not guaranteed for every caller of wait_awaitable::await_suspend
641   // in timer.hpp (e.g. code that only reaches timer.hpp through 655   // in timer.hpp (e.g. code that only reaches timer.hpp through
642   // delay.hpp, without transitively including a scheduler header). 656   // delay.hpp, without transitively including a scheduler header).
643   657  
644   // Free functions 658   // Free functions
645   659  
646   inline timer_service& 660   inline timer_service&
HITCBC 647   2243 get_timer_service(capy::execution_context& ctx, scheduler& sched) 661   2310 get_timer_service(capy::execution_context& ctx, scheduler& sched)
648   { 662   {
HITCBC 649   2243 return ctx.make_service<timer_service>(sched); 663   2310 return ctx.make_service<timer_service>(sched);
650   } 664   }
651   665  
652   } // namespace boost::corosio::detail 666   } // namespace boost::corosio::detail
653   667  
654   #endif 668   #endif