d-SEAMS
Install
Output
Methods
Docs
Engine & CLI
Python
Rust
Julia
R
Lua / Fennel
Fortran
PLUMED
linkcell
Wiki
Cite
GitHub
seams-core
v2.14.0
libyodaLib, the C++ engine of d-SEAMS
Toggle main menu visibility
Loading...
Searching...
No Matches
thread_team.hpp
Go to the documentation of this file.
1
#pragma once
2
3
#include <algorithm>
4
#include <cstddef>
5
#include <cstdlib>
6
#include <cstring>
7
#include <mutex>
8
#include <vector>
9
10
// One width for OpenMP and Rayon. OpenMP already follows
11
// OMP_NUM_THREADS. Linkcell's pool does not, so a process whose
12
// hardware reports one cpu keeps the neighbour walk on one thread
13
// even when OpenMP was given several. lc_set_threads is read by the
14
// next linkcell search.
15
//
16
// OMP_PROC_BIND makes the OpenMP runtime pin the calling thread to its
17
// first place when the library loads, and threads created afterwards, a
18
// Rayon pool among them, inherit that one cpu. widenAffinity gives the
19
// caller back the cpus of every place, which the runtime took from the
20
// launch mask (taskset, numactl, a cgroup cpuset) and OMP_PLACES. It adds
21
// no cpu outside those places. A caller the runtime did not bind, and a
22
// one-thread width, keep their mask. The pool is built once and its
23
// workers sleep between searches. The OpenMP team is not paused: a pause
24
// makes the next region pay a wake-up that moves by tens of milliseconds.
25
26
#ifdef SEAMS_HAS_OPENMP
27
#include <omp.h>
28
#endif
29
#ifdef __linux__
30
#include <sched.h>
31
#endif
32
33
extern
"C"
void
lc_set_threads
(
int
n);
34
extern
"C"
int
lc_num_threads
(
void
);
35
36
namespace
seams
{
37
38
// Once per process, on the first call that may widen. C API callers on
39
// several threads may arrive here together.
40
inline
void
widenAffinity
() {
41
#if defined(__linux__) && defined(SEAMS_HAS_OPENMP)
42
static
std::mutex lock;
43
static
bool
done =
false
;
44
if
(omp_in_parallel() || omp_get_max_threads() < 2 || omp_get_place_num() < 0) {
45
return
;
46
}
47
const
std::lock_guard<std::mutex> hold(lock);
48
if
(done) {
49
return
;
50
}
51
done =
true
;
52
cpu_set_t mask;
53
CPU_ZERO(&mask);
54
if
(sched_getaffinity(0,
sizeof
(mask), &mask) != 0) {
55
return
;
56
}
57
const
int
places = omp_get_num_places();
58
for
(
int
place = 0; place < places; ++place) {
59
const
int
count = omp_get_place_num_procs(place);
60
if
(count <= 0) {
61
continue
;
62
}
63
std::vector<int> ids(
static_cast<
std::size_t
>
(count));
64
omp_get_place_proc_ids(place, ids.data());
65
for
(
const
int
cpu
: ids) {
66
if
(
cpu
>= 0 &&
cpu
< CPU_SETSIZE) {
67
CPU_SET(
static_cast<
std::size_t
>
(
cpu
), &mask);
68
}
69
}
70
}
71
sched_setaffinity(0,
sizeof
(mask), &mask);
72
#endif
73
}
74
75
inline
void
releaseTeam
() {
76
#ifdef SEAMS_HAS_OPENMP
77
omp_pause_resource_all(omp_pause_soft);
78
#endif
79
}
80
81
// OpenMP work that does not hand the cores to Rayon. Pausing the team
82
// here parks every worker and the next parallel region pays a wake-up
83
// that moves by several milliseconds from call to call. Widen the mask
84
// and freeze the team size, and leave the workers asleep only under
85
// the passive wait policy.
86
inline
void
prepareCompute
() {
87
// Linkcell sleeps between searches. Forcing a passive OpenMP wait
88
// on top of that made the next region late by tens of milliseconds,
89
// which dominated a k-nearest call of about a millisecond. Leave the
90
// wait policy the process was started with.
91
widenAffinity
();
92
#ifdef SEAMS_HAS_OPENMP
93
omp_set_dynamic(0);
94
#endif
95
}
96
97
inline
void
adoptThreads
() {
98
// Record the Rayon width and leave the OpenMP team running. Pausing
99
// it, or forcing a passive wait, made the next region late by tens of
100
// milliseconds. The pool sleeps on its own condvar between searches.
101
prepareCompute
();
102
#ifdef SEAMS_HAS_OPENMP
103
if
(omp_in_parallel()) {
104
#ifdef SEAMS_HAS_LINKCELL
105
lc_set_threads
(1);
106
#endif
107
return
;
108
}
109
int
n = omp_get_max_threads();
110
#else
111
int
n = 1;
112
#endif
113
if
(n < 1) {
114
n = 1;
115
}
116
#ifdef SEAMS_HAS_LINKCELL
117
lc_set_threads
(n);
118
#else
119
(void)n;
120
#endif
121
}
122
123
// Width the caller asked for. The neighbour and q_l walks are one
124
// OpenMP team; shrinking that team by the atom count left eight
125
// requested threads idle on the sizes where the search does scale.
126
inline
int
teamFor
(
int
n) {
127
#ifdef SEAMS_HAS_OPENMP
128
(void)n;
129
return
std::max(1, omp_get_max_threads());
130
#else
131
(void)n;
132
return
1;
133
#endif
134
}
135
136
// Sets the OpenMP width from teamFor for the rest of the scope, then
137
// puts the caller's width back. An OpenMP-only kernel uses this instead
138
// of adoptThreads, which would also start the Rayon pool.
139
struct
TeamWidth
{
140
int
saved
= 1;
141
bool
on
=
false
;
142
#ifdef __linux__
143
cpu_set_t oldMask;
144
#endif
145
explicit
TeamWidth
(
int
n) {
146
#ifdef SEAMS_HAS_OPENMP
147
if
(!omp_in_parallel()) {
148
saved
= std::max(1, omp_get_max_threads());
149
const
int
want =
teamFor
(n);
150
if
(want !=
saved
) {
151
omp_set_num_threads(want);
152
on
=
true
;
153
}
154
#ifdef __linux__
155
// A four-thread team on an eight-core mask still migrates across
156
// all eight. Bind it to the first cores the process is allowed,
157
// before the workers are created, so they inherit that mask.
158
if
(want > 0 && want <
saved
&&
159
sched_getaffinity(0,
sizeof
(oldMask), &oldMask) == 0) {
160
// CPU_ISSET and CPU_SET take the cpu number as a size_t.
161
std::size_t cpus[256];
162
int
ncpu = 0;
163
for
(std::size_t
cpu
= 0;
cpu
< CPU_SETSIZE && ncpu < want && ncpu < 256; ++
cpu
) {
164
if
(CPU_ISSET(
cpu
, &oldMask)) {
165
cpus[ncpu++] =
cpu
;
166
}
167
}
168
if
(ncpu > 0) {
169
// Once per process. Repeating the bind on a call of a few
170
// hundred microseconds was the whole gap between four threads
171
// requested directly and eight narrowed to four. OpenMP's own
172
// bind would put the workers back on the whole allowed set,
173
// so it is turned off before the team is created.
174
// Thread 0 is the caller and stays on its own mask: a pinned
175
// caller hands its one cpu to every OpenMP worker and Rayon
176
// thread it creates later, and the next eight-thread region
177
// then runs five threads on that cpu.
178
static
int
bound = 0;
179
static
std::mutex boundMu;
180
const
std::lock_guard<std::mutex> lock(boundMu);
181
if
(bound != want) {
182
setenv(
"OMP_PROC_BIND"
,
"false"
, 1);
183
#pragma omp parallel num_threads(want)
184
{
185
const
int
tid = omp_get_thread_num();
186
if
(tid != 0) {
187
cpu_set_t mask;
188
CPU_ZERO(&mask);
189
CPU_SET(cpus[tid < ncpu ? tid : 0], &mask);
190
sched_setaffinity(0,
sizeof
(mask), &mask);
191
}
192
}
193
bound = want;
194
}
195
}
196
}
197
#endif
198
}
199
#else
200
(void)n;
201
#endif
202
}
203
~TeamWidth
() {
204
#ifdef SEAMS_HAS_OPENMP
205
if
(
on
) {
206
omp_set_num_threads(
saved
);
207
}
208
#endif
209
}
210
TeamWidth
(
const
TeamWidth
&) =
delete
;
211
TeamWidth
&
operator=
(
const
TeamWidth
&) =
delete
;
212
};
213
214
// Team width for one library call on a machine other processes share.
215
// When their threads hold the cores, a full-width team waits at each fork
216
// that follows serial work for workers the scheduler has parked, a slice
217
// at a time, and a call of a few milliseconds grows tenfold. A narrower
218
// team leaves cores free and does not wait. One small region at entry,
219
// after the caller's own gap, takes at most a few hundred microseconds on
220
// an idle machine, where sleeping workers wake, and milliseconds on a
221
// contended one. Two slow entries in a row halve the team, down to a
222
// quarter of the requested width; at most once a second a narrowed team
223
// tries twice its width again. Results do not depend on the width.
224
// SEAMS_FAIR_WIDTH=0 keeps the requested width.
225
struct
FairState
{
226
std::mutex
lock
;
227
int
base
= 0;
228
int
granted
= 0;
229
int
slow
= 0;
230
double
tried
= 0.0;
231
};
232
233
inline
FairState
&
fairState
() {
234
static
FairState
state;
235
return
state;
236
}
237
238
inline
bool
fairEnabled
() {
239
static
const
bool
on = [] {
240
const
char
*flag = std::getenv(
"SEAMS_FAIR_WIDTH"
);
241
return
flag ==
nullptr
|| std::strcmp(flag,
"0"
) != 0;
242
}();
243
return
on;
244
}
245
246
class
FairWidth
{
247
public
:
248
explicit
FairWidth
(
bool
adopt =
false
) {
249
#ifdef SEAMS_HAS_OPENMP
250
if
(depth()++ == 0 && !omp_in_parallel() &&
fairEnabled
()) {
251
choose();
252
}
253
#endif
254
if
(adopt) {
255
adoptThreads
();
256
}
257
}
258
~FairWidth
() {
259
#ifdef SEAMS_HAS_OPENMP
260
if
(set_) {
261
omp_set_num_threads(asked_);
262
}
263
--depth();
264
#endif
265
}
266
FairWidth
(
const
FairWidth
&) =
delete
;
267
FairWidth
&
operator=
(
const
FairWidth
&) =
delete
;
268
269
private
:
270
#ifdef SEAMS_HAS_OPENMP
271
static
constexpr
double
kSlowFork = 1.0e-3;
272
static
constexpr
double
kRetry = 1.0;
273
int
asked_ = 1;
274
bool
set_ =
false
;
275
276
static
int
&depth() {
277
thread_local
int
level = 0;
278
return
level;
279
}
280
281
static
double
forkSeconds(
int
width) {
282
const
double
start = omp_get_wtime();
283
#pragma omp parallel num_threads(width)
284
{
285
[[maybe_unused]]
volatile
int
touch = omp_get_thread_num();
286
}
287
return
omp_get_wtime() - start;
288
}
289
290
void
choose() {
291
asked_ = std::max(1, omp_get_max_threads());
292
if
(asked_ < 2) {
293
return
;
294
}
295
FairState &state =
fairState
();
296
const
std::lock_guard<std::mutex> hold(state.lock);
297
if
(state.base != asked_) {
298
state.base = asked_;
299
state.granted = asked_;
300
state.slow = 0;
301
state.tried = 0.0;
302
}
303
const
int
floor = std::max(1, state.base / 4);
304
const
double
now = omp_get_wtime();
305
int
width = state.granted;
306
if
(width < state.base && now - state.tried >= kRetry) {
307
width = std::min(state.base, 2 * width);
308
state.tried = now;
309
}
310
if
(forkSeconds(width) <= kSlowFork) {
311
state.slow = 0;
312
state.granted = width;
313
}
else
if
(width == state.granted && ++state.slow >= 2 && width > floor) {
314
state.granted = std::max(floor, width / 2);
315
state.slow = 0;
316
state.tried = now;
317
}
318
if
(state.granted != asked_) {
319
omp_set_num_threads(state.granted);
320
set_ =
true
;
321
}
322
}
323
#endif
324
};
325
326
}
// namespace seams
seams::FairWidth::FairWidth
FairWidth(const FairWidth &)=delete
seams::FairWidth::FairWidth
FairWidth(bool adopt=false)
Definition
thread_team.hpp:248
seams::FairWidth::~FairWidth
~FairWidth()
Definition
thread_team.hpp:258
seams::FairWidth::operator=
FairWidth & operator=(const FairWidth &)=delete
seams::cpu
Definition
cpu_features.hpp:30
seams
Definition
cpu_features.hpp:30
seams::widenAffinity
void widenAffinity()
Definition
thread_team.hpp:40
seams::fairState
FairState & fairState()
Definition
thread_team.hpp:233
seams::prepareCompute
void prepareCompute()
Definition
thread_team.hpp:86
seams::teamFor
int teamFor(int n)
Definition
thread_team.hpp:126
seams::releaseTeam
void releaseTeam()
Definition
thread_team.hpp:75
seams::fairEnabled
bool fairEnabled()
Definition
thread_team.hpp:238
seams::adoptThreads
void adoptThreads()
Definition
thread_team.hpp:97
seams::FairState
Definition
thread_team.hpp:225
seams::FairState::slow
int slow
Definition
thread_team.hpp:229
seams::FairState::granted
int granted
Definition
thread_team.hpp:228
seams::FairState::tried
double tried
Definition
thread_team.hpp:230
seams::FairState::lock
std::mutex lock
Definition
thread_team.hpp:226
seams::FairState::base
int base
Definition
thread_team.hpp:227
seams::TeamWidth::saved
int saved
Definition
thread_team.hpp:140
seams::TeamWidth::~TeamWidth
~TeamWidth()
Definition
thread_team.hpp:203
seams::TeamWidth::TeamWidth
TeamWidth(int n)
Definition
thread_team.hpp:145
seams::TeamWidth::operator=
TeamWidth & operator=(const TeamWidth &)=delete
seams::TeamWidth::TeamWidth
TeamWidth(const TeamWidth &)=delete
seams::TeamWidth::on
bool on
Definition
thread_team.hpp:141
lc_set_threads
void lc_set_threads(int n)
lc_num_threads
int lc_num_threads(void)
src
include
internal
thread_team.hpp