seams-core v2.14.0
libyodaLib, the C++ engine of d-SEAMS
Loading...
Searching...
No Matches
thread_team.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <algorithm>
4#include <cstddef>
5#include <cstdlib>
6#include <cstring>
7#include <mutex>
8#include <vector>
9
10// One width for OpenMP and Rayon. OpenMP already follows
11// OMP_NUM_THREADS. Linkcell's pool does not, so a process whose
12// hardware reports one cpu keeps the neighbour walk on one thread
13// even when OpenMP was given several. lc_set_threads is read by the
14// next linkcell search.
15//
16// OMP_PROC_BIND makes the OpenMP runtime pin the calling thread to its
17// first place when the library loads, and threads created afterwards, a
18// Rayon pool among them, inherit that one cpu. widenAffinity gives the
19// caller back the cpus of every place, which the runtime took from the
20// launch mask (taskset, numactl, a cgroup cpuset) and OMP_PLACES. It adds
21// no cpu outside those places. A caller the runtime did not bind, and a
22// one-thread width, keep their mask. The pool is built once and its
23// workers sleep between searches. The OpenMP team is not paused: a pause
24// makes the next region pay a wake-up that moves by tens of milliseconds.
25
26#ifdef SEAMS_HAS_OPENMP
27#include <omp.h>
28#endif
29#ifdef __linux__
30#include <sched.h>
31#endif
32
33extern "C" void lc_set_threads(int n);
34extern "C" int lc_num_threads(void);
35
36namespace seams {
37
38// Once per process, on the first call that may widen. C API callers on
39// several threads may arrive here together.
40inline void widenAffinity() {
41#if defined(__linux__) && defined(SEAMS_HAS_OPENMP)
42 static std::mutex lock;
43 static bool done = false;
44 if (omp_in_parallel() || omp_get_max_threads() < 2 || omp_get_place_num() < 0) {
45 return;
46 }
47 const std::lock_guard<std::mutex> hold(lock);
48 if (done) {
49 return;
50 }
51 done = true;
52 cpu_set_t mask;
53 CPU_ZERO(&mask);
54 if (sched_getaffinity(0, sizeof(mask), &mask) != 0) {
55 return;
56 }
57 const int places = omp_get_num_places();
58 for (int place = 0; place < places; ++place) {
59 const int count = omp_get_place_num_procs(place);
60 if (count <= 0) {
61 continue;
62 }
63 std::vector<int> ids(static_cast<std::size_t>(count));
64 omp_get_place_proc_ids(place, ids.data());
65 for (const int cpu : ids) {
66 if (cpu >= 0 && cpu < CPU_SETSIZE) {
67 CPU_SET(static_cast<std::size_t>(cpu), &mask);
68 }
69 }
70 }
71 sched_setaffinity(0, sizeof(mask), &mask);
72#endif
73}
74
75inline void releaseTeam() {
76#ifdef SEAMS_HAS_OPENMP
77 omp_pause_resource_all(omp_pause_soft);
78#endif
79}
80
81// OpenMP work that does not hand the cores to Rayon. Pausing the team
82// here parks every worker and the next parallel region pays a wake-up
83// that moves by several milliseconds from call to call. Widen the mask
84// and freeze the team size, and leave the workers asleep only under
85// the passive wait policy.
86inline void prepareCompute() {
87 // Linkcell sleeps between searches. Forcing a passive OpenMP wait
88 // on top of that made the next region late by tens of milliseconds,
89 // which dominated a k-nearest call of about a millisecond. Leave the
90 // wait policy the process was started with.
92#ifdef SEAMS_HAS_OPENMP
93 omp_set_dynamic(0);
94#endif
95}
96
97inline void adoptThreads() {
98 // Record the Rayon width and leave the OpenMP team running. Pausing
99 // it, or forcing a passive wait, made the next region late by tens of
100 // milliseconds. The pool sleeps on its own condvar between searches.
102#ifdef SEAMS_HAS_OPENMP
103 if (omp_in_parallel()) {
104#ifdef SEAMS_HAS_LINKCELL
106#endif
107 return;
108 }
109 int n = omp_get_max_threads();
110#else
111 int n = 1;
112#endif
113 if (n < 1) {
114 n = 1;
115 }
116#ifdef SEAMS_HAS_LINKCELL
118#else
119 (void)n;
120#endif
121}
122
123// Width the caller asked for. The neighbour and q_l walks are one
124// OpenMP team; shrinking that team by the atom count left eight
125// requested threads idle on the sizes where the search does scale.
126inline int teamFor(int n) {
127#ifdef SEAMS_HAS_OPENMP
128 (void)n;
129 return std::max(1, omp_get_max_threads());
130#else
131 (void)n;
132 return 1;
133#endif
134}
135
136// Sets the OpenMP width from teamFor for the rest of the scope, then
137// puts the caller's width back. An OpenMP-only kernel uses this instead
138// of adoptThreads, which would also start the Rayon pool.
139struct TeamWidth {
140 int saved = 1;
141 bool on = false;
142#ifdef __linux__
143 cpu_set_t oldMask;
144#endif
145 explicit TeamWidth(int n) {
146#ifdef SEAMS_HAS_OPENMP
147 if (!omp_in_parallel()) {
148 saved = std::max(1, omp_get_max_threads());
149 const int want = teamFor(n);
150 if (want != saved) {
151 omp_set_num_threads(want);
152 on = true;
153 }
154#ifdef __linux__
155 // A four-thread team on an eight-core mask still migrates across
156 // all eight. Bind it to the first cores the process is allowed,
157 // before the workers are created, so they inherit that mask.
158 if (want > 0 && want < saved &&
159 sched_getaffinity(0, sizeof(oldMask), &oldMask) == 0) {
160 // CPU_ISSET and CPU_SET take the cpu number as a size_t.
161 std::size_t cpus[256];
162 int ncpu = 0;
163 for (std::size_t cpu = 0; cpu < CPU_SETSIZE && ncpu < want && ncpu < 256; ++cpu) {
164 if (CPU_ISSET(cpu, &oldMask)) {
165 cpus[ncpu++] = cpu;
166 }
167 }
168 if (ncpu > 0) {
169 // Once per process. Repeating the bind on a call of a few
170 // hundred microseconds was the whole gap between four threads
171 // requested directly and eight narrowed to four. OpenMP's own
172 // bind would put the workers back on the whole allowed set,
173 // so it is turned off before the team is created.
174 // Thread 0 is the caller and stays on its own mask: a pinned
175 // caller hands its one cpu to every OpenMP worker and Rayon
176 // thread it creates later, and the next eight-thread region
177 // then runs five threads on that cpu.
178 static int bound = 0;
179 static std::mutex boundMu;
180 const std::lock_guard<std::mutex> lock(boundMu);
181 if (bound != want) {
182 setenv("OMP_PROC_BIND", "false", 1);
183#pragma omp parallel num_threads(want)
184 {
185 const int tid = omp_get_thread_num();
186 if (tid != 0) {
187 cpu_set_t mask;
188 CPU_ZERO(&mask);
189 CPU_SET(cpus[tid < ncpu ? tid : 0], &mask);
190 sched_setaffinity(0, sizeof(mask), &mask);
191 }
192 }
193 bound = want;
194 }
195 }
196 }
197#endif
198 }
199#else
200 (void)n;
201#endif
202 }
204#ifdef SEAMS_HAS_OPENMP
205 if (on) {
206 omp_set_num_threads(saved);
207 }
208#endif
209 }
210 TeamWidth(const TeamWidth &) = delete;
211 TeamWidth &operator=(const TeamWidth &) = delete;
212};
213
214// Team width for one library call on a machine other processes share.
215// When their threads hold the cores, a full-width team waits at each fork
216// that follows serial work for workers the scheduler has parked, a slice
217// at a time, and a call of a few milliseconds grows tenfold. A narrower
218// team leaves cores free and does not wait. One small region at entry,
219// after the caller's own gap, takes at most a few hundred microseconds on
220// an idle machine, where sleeping workers wake, and milliseconds on a
221// contended one. Two slow entries in a row halve the team, down to a
222// quarter of the requested width; at most once a second a narrowed team
223// tries twice its width again. Results do not depend on the width.
224// SEAMS_FAIR_WIDTH=0 keeps the requested width.
225struct FairState {
226 std::mutex lock;
227 int base = 0;
228 int granted = 0;
229 int slow = 0;
230 double tried = 0.0;
231};
232
234 static FairState state;
235 return state;
236}
237
238inline bool fairEnabled() {
239 static const bool on = [] {
240 const char *flag = std::getenv("SEAMS_FAIR_WIDTH");
241 return flag == nullptr || std::strcmp(flag, "0") != 0;
242 }();
243 return on;
244}
245
247public:
248 explicit FairWidth(bool adopt = false) {
249#ifdef SEAMS_HAS_OPENMP
250 if (depth()++ == 0 && !omp_in_parallel() && fairEnabled()) {
251 choose();
252 }
253#endif
254 if (adopt) {
255 adoptThreads();
256 }
257 }
259#ifdef SEAMS_HAS_OPENMP
260 if (set_) {
261 omp_set_num_threads(asked_);
262 }
263 --depth();
264#endif
265 }
266 FairWidth(const FairWidth &) = delete;
267 FairWidth &operator=(const FairWidth &) = delete;
268
269private:
270#ifdef SEAMS_HAS_OPENMP
271 static constexpr double kSlowFork = 1.0e-3;
272 static constexpr double kRetry = 1.0;
273 int asked_ = 1;
274 bool set_ = false;
275
276 static int &depth() {
277 thread_local int level = 0;
278 return level;
279 }
280
281 static double forkSeconds(int width) {
282 const double start = omp_get_wtime();
283#pragma omp parallel num_threads(width)
284 {
285 [[maybe_unused]] volatile int touch = omp_get_thread_num();
286 }
287 return omp_get_wtime() - start;
288 }
289
290 void choose() {
291 asked_ = std::max(1, omp_get_max_threads());
292 if (asked_ < 2) {
293 return;
294 }
295 FairState &state = fairState();
296 const std::lock_guard<std::mutex> hold(state.lock);
297 if (state.base != asked_) {
298 state.base = asked_;
299 state.granted = asked_;
300 state.slow = 0;
301 state.tried = 0.0;
302 }
303 const int floor = std::max(1, state.base / 4);
304 const double now = omp_get_wtime();
305 int width = state.granted;
306 if (width < state.base && now - state.tried >= kRetry) {
307 width = std::min(state.base, 2 * width);
308 state.tried = now;
309 }
310 if (forkSeconds(width) <= kSlowFork) {
311 state.slow = 0;
312 state.granted = width;
313 } else if (width == state.granted && ++state.slow >= 2 && width > floor) {
314 state.granted = std::max(floor, width / 2);
315 state.slow = 0;
316 state.tried = now;
317 }
318 if (state.granted != asked_) {
319 omp_set_num_threads(state.granted);
320 set_ = true;
321 }
322 }
323#endif
324};
325
326} // namespace seams
FairWidth(const FairWidth &)=delete
FairWidth(bool adopt=false)
FairWidth & operator=(const FairWidth &)=delete
void widenAffinity()
FairState & fairState()
void prepareCompute()
int teamFor(int n)
void releaseTeam()
bool fairEnabled()
void adoptThreads()
TeamWidth & operator=(const TeamWidth &)=delete
TeamWidth(const TeamWidth &)=delete
void lc_set_threads(int n)
int lc_num_threads(void)