StarPU Internal Handbook
coherency.h
Go to the documentation of this file.
1 /* StarPU --- Runtime system for heterogeneous multicore architectures.
2  *
3  * Copyright (C) 2008-2023 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
4  *
5  * StarPU is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU Lesser General Public License as published by
7  * the Free Software Foundation; either version 2.1 of the License, or (at
8  * your option) any later version.
9  *
10  * StarPU is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13  *
14  * See the GNU Lesser General Public License in COPYING.LGPL for more details.
15  */
16 
17 #ifndef __COHERENCY__H__
18 #define __COHERENCY__H__
19 
22 #include <starpu.h>
23 #include <common/config.h>
24 
25 #include <common/starpu_spinlock.h>
26 #include <common/rwlock.h>
27 #include <common/timing.h>
28 #include <common/fxt.h>
29 #include <common/list.h>
30 
32 #include <datawizard/datastats.h>
33 #include <datawizard/memstats.h>
35 
36 #pragma GCC visibility push(hidden)
37 
38 enum _starpu_cache_state
39 {
40  STARPU_OWNER,
41  STARPU_SHARED,
42  STARPU_INVALID
43 };
44 
47 {
48  starpu_data_handle_t handle;
49 
52 
54  int refcnt;
55 
56  char memory_node;
57 
59  enum _starpu_cache_state state: 2;
60 
63  unsigned relaxed_coherency:2;
64 
66  unsigned initialized:1;
67 
69  unsigned allocated:1;
76 
82  unsigned map_write:1;
83 
84 #define STARPU_UNMAPPED -1
87  int mapped;
88 
96  uint32_t requested;
97 
99  struct _starpu_data_request *request[STARPU_MAXNODES];
101  struct _starpu_data_request *last_request[STARPU_MAXNODES];
102 
103  /* Which request is loading data here */
104  struct _starpu_data_request *load_request;
105 
111 
113  struct _starpu_mem_chunk * mc;
114 };
115 
116 struct _starpu_data_requester_prio_list;
117 
119 {
120  unsigned long id;
121  struct _starpu_jobid_list *next;
122 };
123 
126 {
127  struct starpu_task *task;
128  struct _starpu_task_wrapper_list *next;
129 };
130 
133 {
134  struct starpu_task *task;
135  struct _starpu_task_wrapper_dlist *next;
136  struct _starpu_task_wrapper_dlist *prev;
137 };
138 
139 extern int _starpu_has_not_important_data;
140 
141 typedef void (*_starpu_data_handle_unregister_hook)(starpu_data_handle_t);
142 
145 {
146  int magic;
147  struct _starpu_data_requester_prio_list req_list;
151  unsigned refcnt;
153  unsigned unlocking_reqs;
157  enum starpu_data_access_mode current_mode;
160 
165  unsigned busy_count;
167  unsigned busy_waiting;
168  starpu_pthread_mutex_t busy_mutex;
169  starpu_pthread_cond_t busy_cond;
170 
174  starpu_data_handle_t *active_children;
176  starpu_data_handle_t **active_readonly_children;
180  unsigned nsiblings;
181  starpu_data_handle_t *siblings;
182  unsigned sibling_index;
183  unsigned depth;
185 #ifdef STARPU_BUBBLE
186  starpu_pthread_mutex_t unpartition_mutex;
187 #endif
188 
190  starpu_data_handle_t children;
191  unsigned nchildren;
193  unsigned nplans;
195  struct starpu_codelet *switch_cl;
204  unsigned partitioned;
206  unsigned part_readonly:1;
207 
209  unsigned active:1;
210  unsigned active_ro:1;
211 
215  struct _starpu_data_replicate *per_worker;
216 
217  struct starpu_data_interface_ops *ops;
218 
220  uint32_t footprint;
221 
222  /* The following bitfields are set from the application initialization */
223 
226  unsigned is_not_important:1;
228  unsigned ooc:1;
232  unsigned readonly:1;
233 
236 
238  uint32_t wt_mask;
239 
242  unsigned aliases;
245  starpu_data_handle_t readonly_dup;
248  starpu_data_handle_t readonly_dup_of;
249 
250  /* The following bitfields are set from the application submission thread */
251 
254  unsigned initialized:1;
255 
256 #ifdef STARPU_OPENMP
257  unsigned removed_from_context_hash:1;
258 #endif
259 
260  /* The following field is set by StarPU at execution time */
261 
263  unsigned char lazy_unregister;
264 
267  starpu_pthread_mutex_t sequential_consistency_mutex;
268 
273  enum starpu_data_access_mode last_submitted_mode;
274  struct starpu_task *last_sync_task;
275  struct _starpu_task_wrapper_dlist last_submitted_accessors;
276 
284  unsigned long last_submitted_ghost_sync_id;
285  struct _starpu_jobid_list *last_submitted_ghost_accessors_id;
286 
289  unsigned post_sync_tasks_cnt;
290 
291  /*
292  * Reductions
293  */
294 
299  struct starpu_codelet *redux_cl;
300  struct starpu_codelet *init_cl;
301  void *redux_cl_arg;
302  void *init_cl_arg;
303 
308 
312  struct _starpu_data_requester_prio_list reduction_req_list;
313 
314  starpu_data_handle_t *reduction_tmp_handles;
315 
317  struct _starpu_data_request *write_invalidation_req;
318 
320  void *mpi_data;
321 
322  _starpu_memory_stats_t memory_stats;
323 
324  unsigned int mf_node; //XXX
325 
327  _starpu_data_handle_unregister_hook unregister_hook;
328 
329  struct starpu_arbiter *arbiter;
331  struct _starpu_data_requester_prio_list arbitered_req_list;
332 
337 
340  unsigned dimensions;
341  int coordinates[5];
342 
345  void *user_data;
346 
349  void *sched_data;
350 };
351 
358 int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate,
359  enum starpu_data_access_mode mode, unsigned detached,
360  struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async,
361  void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
363 void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask,
364  enum starpu_data_access_mode down_to_mode,
365  struct _starpu_data_replicate *replicate);
366 
367 void _starpu_update_data_state(starpu_data_handle_t handle,
368  struct _starpu_data_replicate *requesting_replicate,
369  enum starpu_data_access_mode mode);
370 
371 uint32_t _starpu_get_data_refcnt(struct _starpu_data_state *state, unsigned node);
372 
373 size_t _starpu_data_get_size(starpu_data_handle_t handle);
374 size_t _starpu_data_get_alloc_size(starpu_data_handle_t handle);
375 starpu_ssize_t _starpu_data_get_max_size(starpu_data_handle_t handle);
376 
377 uint32_t _starpu_data_get_footprint(starpu_data_handle_t handle);
378 
379 void __starpu_push_task_output(struct _starpu_job *j);
382 
383 struct _starpu_worker;
384 STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
388 int _starpu_fetch_task_input(struct starpu_task *task, struct _starpu_job *j, int async);
389 void _starpu_fetch_task_input_tail(struct starpu_task *task, struct _starpu_job *j, struct _starpu_worker *worker);
390 void _starpu_fetch_nowhere_task_input(struct _starpu_job *j);
391 
392 int _starpu_select_src_node(struct _starpu_data_state *state, unsigned destination);
393 int _starpu_determine_request_path(starpu_data_handle_t handle,
394  int src_node, int dst_node,
395  enum starpu_data_access_mode mode, int max_len,
396  unsigned *src_nodes, unsigned *dst_nodes,
397  unsigned *handling_nodes, unsigned write_invalidation);
398 
403 struct _starpu_data_request *_starpu_create_request_to_fetch_data(starpu_data_handle_t handle,
404  struct _starpu_data_replicate *dst_replicate,
405  enum starpu_data_access_mode mode,
406  struct starpu_task *task, enum starpu_is_prefetch is_prefetch,
407  unsigned async,
408  void (*callback_func)(void *), void *callback_arg, int prio, const char *origin);
409 
410 void _starpu_redux_init_data_replicate(starpu_data_handle_t handle, struct _starpu_data_replicate *replicate, int workerid);
411 void _starpu_data_start_reduction_mode(starpu_data_handle_t handle);
412 void _starpu_data_end_reduction_mode(starpu_data_handle_t handle, int priority);
413 void _starpu_data_end_reduction_mode_terminate(starpu_data_handle_t handle);
414 
415 void _starpu_data_unmap(starpu_data_handle_t handle, unsigned node);
416 
417 void _starpu_data_set_unregister_hook(starpu_data_handle_t handle, _starpu_data_handle_unregister_hook func) STARPU_ATTRIBUTE_VISIBILITY_DEFAULT;
418 
419 #pragma GCC visibility pop
420 
421 #endif // __COHERENCY__H__
unsigned busy_count
Definition: coherency.h:165
void * mpi_data
Definition: coherency.h:320
struct _starpu_task_wrapper_list * post_sync_tasks
Definition: coherency.h:288
unsigned readonly
Definition: coherency.h:232
struct starpu_codelet * redux_cl
Definition: coherency.h:299
unsigned nactive_readonly_children
Definition: coherency.h:178
struct starpu_codelet * switch_cl
Definition: coherency.h:195
struct _starpu_data_requester_prio_list arbitered_req_list
Definition: coherency.h:331
_starpu_data_handle_unregister_hook unregister_hook
Definition: coherency.h:327
struct _starpu_data_request * request[STARPU_MAXNODES]
Definition: coherency.h:99
starpu_pthread_mutex_t unpartition_mutex
Definition: coherency.h:186
struct _starpu_data_request * write_invalidation_req
Definition: coherency.h:317
unsigned aliases
Definition: coherency.h:242
enum _starpu_cache_state state
Definition: coherency.h:59
unsigned active
Definition: coherency.h:209
enum starpu_data_access_mode current_mode
Definition: coherency.h:157
unsigned unlocking_reqs
Definition: coherency.h:153
starpu_data_handle_t * siblings
Definition: coherency.h:181
unsigned switch_cl_nparts
Definition: coherency.h:197
int _starpu_fetch_data_on_node(starpu_data_handle_t handle, int node, struct _starpu_data_replicate *replicate, enum starpu_data_access_mode mode, unsigned detached, struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
struct _starpu_data_state * father_handle
Definition: coherency.h:173
starpu_data_handle_t children
Definition: coherency.h:190
unsigned automatically_allocated
Definition: coherency.h:75
unsigned ooc
Definition: coherency.h:228
starpu_pthread_mutex_t sequential_consistency_mutex
Definition: coherency.h:267
unsigned nplans
Definition: coherency.h:193
struct _starpu_mem_chunk * mc
Definition: coherency.h:113
int mapped
Definition: coherency.h:87
int refcnt
Definition: coherency.h:54
unsigned relaxed_coherency
Definition: coherency.h:63
unsigned active_nchildren
Definition: coherency.h:175
enum starpu_data_access_mode last_submitted_mode
Definition: coherency.h:273
int last_locality
Definition: coherency.h:336
STARPU_ATTRIBUTE_WARN_UNUSED_RESULT int _starpu_fetch_task_input(struct starpu_task *task, struct _starpu_job *j, int async)
struct _starpu_data_replicate per_node[STARPU_MAXNODES]
Definition: coherency.h:214
struct _starpu_spinlock header_lock
Definition: coherency.h:159
starpu_data_handle_t readonly_dup_of
Definition: coherency.h:248
unsigned char lazy_unregister
Definition: coherency.h:263
unsigned busy_waiting
Definition: coherency.h:167
void * sched_data
Definition: coherency.h:349
unsigned * active_readonly_nchildren
Definition: coherency.h:177
unsigned map_write
Definition: coherency.h:82
unsigned partitioned
Definition: coherency.h:204
unsigned initialized
Definition: coherency.h:254
void * data_interface
Definition: coherency.h:51
struct _starpu_data_request * last_request[STARPU_MAXNODES]
Definition: coherency.h:101
int home_node
Definition: coherency.h:235
struct _starpu_data_request * _starpu_create_request_to_fetch_data(starpu_data_handle_t handle, struct _starpu_data_replicate *dst_replicate, enum starpu_data_access_mode mode, struct starpu_task *task, enum starpu_is_prefetch is_prefetch, unsigned async, void(*callback_func)(void *), void *callback_arg, int prio, const char *origin)
unsigned nb_tasks_prefetch
Definition: coherency.h:110
struct _starpu_data_state * root_handle
Definition: coherency.h:172
unsigned last_submitted_ghost_sync_id_is_valid
Definition: coherency.h:283
void _starpu_release_data_on_node(struct _starpu_data_state *state, uint32_t default_wt_mask, enum starpu_data_access_mode down_to_mode, struct _starpu_data_replicate *replicate)
void _starpu_push_task_output(struct _starpu_job *j)
starpu_data_handle_t readonly_dup
Definition: coherency.h:245
uint32_t wt_mask
Definition: coherency.h:238
unsigned depth
Definition: coherency.h:183
unsigned initialized
Definition: coherency.h:66
unsigned nsiblings
Definition: coherency.h:180
struct _starpu_data_requester_prio_list reduction_req_list
Definition: coherency.h:312
void * user_data
Definition: coherency.h:345
unsigned refcnt
Definition: coherency.h:151
unsigned dimensions
Definition: coherency.h:340
unsigned sequential_consistency
Definition: coherency.h:230
unsigned reduction_refcnt
Definition: coherency.h:307
uint32_t requested
Definition: coherency.h:96
unsigned is_not_important
Definition: coherency.h:226
unsigned allocated
Definition: coherency.h:69
starpu_data_handle_t * active_children
Definition: coherency.h:174
uint32_t footprint
Definition: coherency.h:220
unsigned part_readonly
Definition: coherency.h:206
Definition: coherency.h:47
Definition: coherency.h:145
Definition: coherency.h:119
Definition: coherency.h:133
Definition: coherency.h:126
#define STARPU_MAXNODES
Definition: starpu_config.h:217
Definition: starpu_spinlock.h:82
Definition: jobs.h:74
Definition: workers.h:155