corosync  2.4.3
sync.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2009-2012 Red Hat, Inc.
3  *
4  * All rights reserved.
5  *
6  * Author: Steven Dake (sdake@redhat.com)
7  *
8  * This software licensed under BSD license, the text of which follows:
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions are met:
12  *
13  * - Redistributions of source code must retain the above copyright notice,
14  * this list of conditions and the following disclaimer.
15  * - Redistributions in binary form must reproduce the above copyright notice,
16  * this list of conditions and the following disclaimer in the documentation
17  * and/or other materials provided with the distribution.
18  * - Neither the name of the MontaVista Software, Inc. nor the names of its
19  * contributors may be used to endorse or promote products derived from this
20  * software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32  * THE POSSIBILITY OF SUCH DAMAGE.
33  */
34 #include <config.h>
35 
36 #include <sys/types.h>
37 #include <sys/socket.h>
38 #include <sys/un.h>
39 #include <sys/ioctl.h>
40 #include <netinet/in.h>
41 #include <sys/uio.h>
42 #include <unistd.h>
43 #include <fcntl.h>
44 #include <stdlib.h>
45 #include <stdio.h>
46 #include <errno.h>
47 #include <time.h>
48 #include <arpa/inet.h>
49 
50 #include <corosync/corotypes.h>
51 #include <corosync/swab.h>
52 #include <corosync/totem/totempg.h>
53 #include <corosync/totem/totem.h>
54 #include <corosync/logsys.h>
55 #include <qb/qbipc_common.h>
56 #include "schedwrk.h"
57 #include "quorum.h"
58 #include "sync.h"
59 #include "main.h"
60 
61 LOGSYS_DECLARE_SUBSYS ("SYNC");
62 
63 #define MESSAGE_REQ_SYNC_BARRIER 0
64 #define MESSAGE_REQ_SYNC_SERVICE_BUILD 1
65 #define MESSAGE_REQ_SYNC_MEMB_DETERMINE 2
66 
71 };
72 
73 enum sync_state {
77 };
78 
79 struct service_entry {
81  void (*sync_init) (
82  const unsigned int *trans_list,
83  size_t trans_list_entries,
84  const unsigned int *member_list,
85  size_t member_list_entries,
86  const struct memb_ring_id *ring_id);
87  void (*sync_abort) (void);
88  int (*sync_process) (void);
89  void (*sync_activate) (void);
91  char name[128];
92 };
93 
95  int nodeid;
96  int received;
97 };
98 
100  struct qb_ipc_request_header header __attribute__((aligned(8)));
101  struct memb_ring_id ring_id __attribute__((aligned(8)));
102 };
103 
105  struct qb_ipc_request_header header __attribute__((aligned(8)));
106  struct memb_ring_id ring_id __attribute__((aligned(8)));
107  int service_list_entries __attribute__((aligned(8)));
108  int service_list[128] __attribute__((aligned(8)));
109 };
110 
112  struct qb_ipc_request_header header __attribute__((aligned(8)));
113  struct memb_ring_id ring_id __attribute__((aligned(8)));
114 };
115 
116 static enum sync_state my_state = SYNC_BARRIER;
117 
118 static struct memb_ring_id my_ring_id;
119 
120 static struct memb_ring_id my_memb_determine_ring_id;
121 
122 static int my_memb_determine = 0;
123 
124 static unsigned int my_memb_determine_list[PROCESSOR_COUNT_MAX];
125 
126 static unsigned int my_memb_determine_list_entries = 0;
127 
128 static int my_processing_idx = 0;
129 
130 static hdb_handle_t my_schedwrk_handle;
131 
132 static struct processor_entry my_processor_list[PROCESSOR_COUNT_MAX];
133 
134 static unsigned int my_member_list[PROCESSOR_COUNT_MAX];
135 
136 static unsigned int my_trans_list[PROCESSOR_COUNT_MAX];
137 
138 static size_t my_member_list_entries = 0;
139 
140 static size_t my_trans_list_entries = 0;
141 
142 static int my_processor_list_entries = 0;
143 
144 static struct service_entry my_service_list[SERVICES_COUNT_MAX];
145 
146 static int my_service_list_entries = 0;
147 
148 static void (*sync_synchronization_completed) (void);
149 
150 static void sync_deliver_fn (
151  unsigned int nodeid,
152  const void *msg,
153  unsigned int msg_len,
154  int endian_conversion_required);
155 
156 static int schedwrk_processor (const void *context);
157 
158 static void sync_process_enter (void);
159 
160 static struct totempg_group sync_group = {
161  .group = "sync",
162  .group_len = 4
163 };
164 
165 static void *sync_group_handle;
166 
168  int service_id,
169  struct sync_callbacks *callbacks);
170 
172  int (*sync_callbacks_retrieve) (
173  int service_id,
174  struct sync_callbacks *callbacks),
175  void (*synchronization_completed) (void))
176 {
177  unsigned int res;
178 
180  &sync_group_handle,
181  sync_deliver_fn,
182  NULL);
183  if (res == -1) {
185  "Couldn't initialize groups interface.");
186  return (-1);
187  }
188 
189  res = totempg_groups_join (
190  sync_group_handle,
191  &sync_group,
192  1);
193  if (res == -1) {
194  log_printf (LOGSYS_LEVEL_ERROR, "Couldn't join group.");
195  return (-1);
196  }
197 
198  sync_synchronization_completed = synchronization_completed;
199  my_sync_callbacks_retrieve = sync_callbacks_retrieve;
200 
201  return (0);
202 }
203 
204 static void sync_barrier_handler (unsigned int nodeid, const void *msg)
205 {
207  int i;
208  int barrier_reached = 1;
209 
210  if (memcmp (&my_ring_id, &req_exec_barrier_message->ring_id,
211  sizeof (struct memb_ring_id)) != 0) {
212 
213  log_printf (LOGSYS_LEVEL_DEBUG, "barrier for old ring - discarding");
214  return;
215  }
216  for (i = 0; i < my_processor_list_entries; i++) {
217  if (my_processor_list[i].nodeid == nodeid) {
218  my_processor_list[i].received = 1;
219  }
220  }
221  for (i = 0; i < my_processor_list_entries; i++) {
222  if (my_processor_list[i].received == 0) {
223  barrier_reached = 0;
224  }
225  }
226  if (barrier_reached) {
227  log_printf (LOGSYS_LEVEL_DEBUG, "Committing synchronization for %s",
228  my_service_list[my_processing_idx].name);
229  my_service_list[my_processing_idx].state = ACTIVATE;
230 
231  if (my_sync_callbacks_retrieve(my_service_list[my_processing_idx].service_id, NULL) != -1) {
232  my_service_list[my_processing_idx].sync_activate ();
233  }
234 
235  my_processing_idx += 1;
236  if (my_service_list_entries == my_processing_idx) {
237  my_memb_determine_list_entries = 0;
238  sync_synchronization_completed ();
239  } else {
240  sync_process_enter ();
241  }
242  }
243 }
244 
245 static void dummy_sync_init (
246  const unsigned int *trans_list,
247  size_t trans_list_entries,
248  const unsigned int *member_list,
249  size_t member_list_entries,
250  const struct memb_ring_id *ring_id)
251 {
252 }
253 
254 static void dummy_sync_abort (void)
255 {
256 }
257 
258 static int dummy_sync_process (void)
259 {
260  return (0);
261 }
262 
263 static void dummy_sync_activate (void)
264 {
265 }
266 
267 static int service_entry_compare (const void *a, const void *b)
268 {
269  const struct service_entry *service_entry_a = a;
270  const struct service_entry *service_entry_b = b;
271 
272  return (service_entry_a->service_id > service_entry_b->service_id);
273 }
274 
275 static void sync_memb_determine (unsigned int nodeid, const void *msg)
276 {
278  int found = 0;
279  int i;
280 
281  if (memcmp (&req_exec_memb_determine_message->ring_id,
282  &my_memb_determine_ring_id, sizeof (struct memb_ring_id)) != 0) {
283 
284  log_printf (LOGSYS_LEVEL_DEBUG, "memb determine for old ring - discarding");
285  return;
286  }
287 
288  my_memb_determine = 1;
289  for (i = 0; i < my_memb_determine_list_entries; i++) {
290  if (my_memb_determine_list[i] == nodeid) {
291  found = 1;
292  }
293  }
294  if (found == 0) {
295  my_memb_determine_list[my_memb_determine_list_entries] = nodeid;
296  my_memb_determine_list_entries += 1;
297  }
298 }
299 
300 static void sync_service_build_handler (unsigned int nodeid, const void *msg)
301 {
303  int i, j;
304  int barrier_reached = 1;
305  int found;
306  int qsort_trigger = 0;
307 
308  if (memcmp (&my_ring_id, &req_exec_service_build_message->ring_id,
309  sizeof (struct memb_ring_id)) != 0) {
310  log_printf (LOGSYS_LEVEL_DEBUG, "service build for old ring - discarding");
311  return;
312  }
313  for (i = 0; i < req_exec_service_build_message->service_list_entries; i++) {
314 
315  found = 0;
316  for (j = 0; j < my_service_list_entries; j++) {
317  if (req_exec_service_build_message->service_list[i] ==
318  my_service_list[j].service_id) {
319  found = 1;
320  break;
321  }
322  }
323  if (found == 0) {
324  my_service_list[my_service_list_entries].state =
325  INIT;
326  my_service_list[my_service_list_entries].service_id =
327  req_exec_service_build_message->service_list[i];
328  sprintf (my_service_list[my_service_list_entries].name,
329  "Unknown External Service (id = %d)\n",
330  req_exec_service_build_message->service_list[i]);
331  my_service_list[my_service_list_entries].sync_init =
332  dummy_sync_init;
333  my_service_list[my_service_list_entries].sync_abort =
334  dummy_sync_abort;
335  my_service_list[my_service_list_entries].sync_process =
336  dummy_sync_process;
337  my_service_list[my_service_list_entries].sync_activate =
338  dummy_sync_activate;
339  my_service_list_entries += 1;
340 
341  qsort_trigger = 1;
342  }
343  }
344  if (qsort_trigger) {
345  qsort (my_service_list, my_service_list_entries,
346  sizeof (struct service_entry), service_entry_compare);
347  }
348  for (i = 0; i < my_processor_list_entries; i++) {
349  if (my_processor_list[i].nodeid == nodeid) {
350  my_processor_list[i].received = 1;
351  }
352  }
353  for (i = 0; i < my_processor_list_entries; i++) {
354  if (my_processor_list[i].received == 0) {
355  barrier_reached = 0;
356  }
357  }
358  if (barrier_reached) {
359  sync_process_enter ();
360  }
361 }
362 
363 static void sync_deliver_fn (
364  unsigned int nodeid,
365  const void *msg,
366  unsigned int msg_len,
367  int endian_conversion_required)
368 {
369  struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
370 
371  switch (header->id) {
373  sync_barrier_handler (nodeid, msg);
374  break;
376  sync_service_build_handler (nodeid, msg);
377  break;
379  sync_memb_determine (nodeid, msg);
380  break;
381  }
382 }
383 
384 static void memb_determine_message_transmit (void)
385 {
386  struct iovec iovec;
387  struct req_exec_memb_determine_message req_exec_memb_determine_message;
388 
389  req_exec_memb_determine_message.header.size = sizeof (struct req_exec_memb_determine_message);
390  req_exec_memb_determine_message.header.id = MESSAGE_REQ_SYNC_MEMB_DETERMINE;
391 
392  memcpy (&req_exec_memb_determine_message.ring_id,
393  &my_memb_determine_ring_id,
394  sizeof (struct memb_ring_id));
395 
396  iovec.iov_base = (char *)&req_exec_memb_determine_message;
397  iovec.iov_len = sizeof (req_exec_memb_determine_message);
398 
399  (void)totempg_groups_mcast_joined (sync_group_handle,
400  &iovec, 1, TOTEMPG_AGREED);
401 }
402 
403 static void barrier_message_transmit (void)
404 {
405  struct iovec iovec;
406  struct req_exec_barrier_message req_exec_barrier_message;
407 
408  req_exec_barrier_message.header.size = sizeof (struct req_exec_barrier_message);
409  req_exec_barrier_message.header.id = MESSAGE_REQ_SYNC_BARRIER;
410 
411  memcpy (&req_exec_barrier_message.ring_id, &my_ring_id,
412  sizeof (struct memb_ring_id));
413 
414  iovec.iov_base = (char *)&req_exec_barrier_message;
415  iovec.iov_len = sizeof (req_exec_barrier_message);
416 
417  (void)totempg_groups_mcast_joined (sync_group_handle,
418  &iovec, 1, TOTEMPG_AGREED);
419 }
420 
421 static void service_build_message_transmit (struct req_exec_service_build_message *service_build_message)
422 {
423  struct iovec iovec;
424 
425  service_build_message->header.size = sizeof (struct req_exec_service_build_message);
426  service_build_message->header.id = MESSAGE_REQ_SYNC_SERVICE_BUILD;
427 
428  memcpy (&service_build_message->ring_id, &my_ring_id,
429  sizeof (struct memb_ring_id));
430 
431  iovec.iov_base = (void *)service_build_message;
432  iovec.iov_len = sizeof (struct req_exec_service_build_message);
433 
434  (void)totempg_groups_mcast_joined (sync_group_handle,
435  &iovec, 1, TOTEMPG_AGREED);
436 }
437 
438 static void sync_barrier_enter (void)
439 {
440  my_state = SYNC_BARRIER;
441  barrier_message_transmit ();
442 }
443 
444 static void sync_process_enter (void)
445 {
446  int i;
447 
448  my_state = SYNC_PROCESS;
449 
450  /*
451  * No sync services
452  */
453  if (my_service_list_entries == 0) {
454  my_state = SYNC_SERVICELIST_BUILD;
455  my_memb_determine_list_entries = 0;
456  sync_synchronization_completed ();
457  return;
458  }
459  for (i = 0; i < my_processor_list_entries; i++) {
460  my_processor_list[i].received = 0;
461  }
462  schedwrk_create (&my_schedwrk_handle,
463  schedwrk_processor,
464  NULL);
465 }
466 
467 static void sync_servicelist_build_enter (
468  const unsigned int *member_list,
469  size_t member_list_entries,
470  const struct memb_ring_id *ring_id)
471 {
472  struct req_exec_service_build_message service_build;
473  int i;
474  int res;
475  struct sync_callbacks sync_callbacks;
476 
477  my_state = SYNC_SERVICELIST_BUILD;
478  for (i = 0; i < member_list_entries; i++) {
479  my_processor_list[i].nodeid = member_list[i];
480  my_processor_list[i].received = 0;
481  }
482  my_processor_list_entries = member_list_entries;
483 
484  memcpy (my_member_list, member_list,
485  member_list_entries * sizeof (unsigned int));
486  my_member_list_entries = member_list_entries;
487 
488  my_processing_idx = 0;
489 
490  memset(my_service_list, 0, sizeof (struct service_entry) * SERVICES_COUNT_MAX);
491  my_service_list_entries = 0;
492 
493  for (i = 0; i < SERVICES_COUNT_MAX; i++) {
494  res = my_sync_callbacks_retrieve (i, &sync_callbacks);
495  if (res == -1) {
496  continue;
497  }
498  if (sync_callbacks.sync_init == NULL) {
499  continue;
500  }
501  my_service_list[my_service_list_entries].state = INIT;
502  my_service_list[my_service_list_entries].service_id = i;
503  strcpy (my_service_list[my_service_list_entries].name,
504  sync_callbacks.name);
505  my_service_list[my_service_list_entries].sync_init = sync_callbacks.sync_init;
506  my_service_list[my_service_list_entries].sync_process = sync_callbacks.sync_process;
507  my_service_list[my_service_list_entries].sync_abort = sync_callbacks.sync_abort;
508  my_service_list[my_service_list_entries].sync_activate = sync_callbacks.sync_activate;
509  my_service_list_entries += 1;
510  }
511 
512  for (i = 0; i < my_service_list_entries; i++) {
513  service_build.service_list[i] =
514  my_service_list[i].service_id;
515  }
516  service_build.service_list_entries = my_service_list_entries;
517 
518  service_build_message_transmit (&service_build);
519 }
520 
521 static int schedwrk_processor (const void *context)
522 {
523  int res = 0;
524 
525  if (my_service_list[my_processing_idx].state == INIT) {
526  unsigned int old_trans_list[PROCESSOR_COUNT_MAX];
527  size_t old_trans_list_entries = 0;
528  int o, m;
529  my_service_list[my_processing_idx].state = PROCESS;
530 
531  memcpy (old_trans_list, my_trans_list, my_trans_list_entries *
532  sizeof (unsigned int));
533  old_trans_list_entries = my_trans_list_entries;
534 
535  my_trans_list_entries = 0;
536  for (o = 0; o < old_trans_list_entries; o++) {
537  for (m = 0; m < my_member_list_entries; m++) {
538  if (old_trans_list[o] == my_member_list[m]) {
539  my_trans_list[my_trans_list_entries] = my_member_list[m];
540  my_trans_list_entries++;
541  break;
542  }
543  }
544  }
545 
546  if (my_sync_callbacks_retrieve(my_service_list[my_processing_idx].service_id, NULL) != -1) {
547  my_service_list[my_processing_idx].sync_init (my_trans_list,
548  my_trans_list_entries, my_member_list,
549  my_member_list_entries,
550  &my_ring_id);
551  }
552  }
553  if (my_service_list[my_processing_idx].state == PROCESS) {
554  my_service_list[my_processing_idx].state = PROCESS;
555  if (my_sync_callbacks_retrieve(my_service_list[my_processing_idx].service_id, NULL) != -1) {
556  res = my_service_list[my_processing_idx].sync_process ();
557  } else {
558  res = 0;
559  }
560  if (res == 0) {
561  sync_barrier_enter();
562  } else {
563  return (-1);
564  }
565  }
566  return (0);
567 }
568 
570  const unsigned int *member_list,
571  size_t member_list_entries,
572  const struct memb_ring_id *ring_id)
573 {
574  ENTER();
575  memcpy (&my_ring_id, ring_id, sizeof (struct memb_ring_id));
576 
577  if (my_memb_determine) {
578  my_memb_determine = 0;
579  sync_servicelist_build_enter (my_memb_determine_list,
580  my_memb_determine_list_entries, ring_id);
581  } else {
582  sync_servicelist_build_enter (member_list, member_list_entries,
583  ring_id);
584  }
585 }
586 
588  const unsigned int *member_list,
589  size_t member_list_entries,
590  const struct memb_ring_id *ring_id)
591 {
592  ENTER();
593  memcpy (my_trans_list, member_list, member_list_entries *
594  sizeof (unsigned int));
595  my_trans_list_entries = member_list_entries;
596 }
597 
598 void sync_abort (void)
599 {
600  ENTER();
601  if (my_state == SYNC_PROCESS) {
602  schedwrk_destroy (my_schedwrk_handle);
603  if (my_sync_callbacks_retrieve(my_service_list[my_processing_idx].service_id, NULL) != -1) {
604  my_service_list[my_processing_idx].sync_abort ();
605  }
606  }
607 
608  /* this will cause any "old" barrier messages from causing
609  * problems.
610  */
611  memset (&my_ring_id, 0, sizeof (struct memb_ring_id));
612 }
613 
614 void sync_memb_list_determine (const struct memb_ring_id *ring_id)
615 {
616  ENTER();
617  memcpy (&my_memb_determine_ring_id, ring_id,
618  sizeof (struct memb_ring_id));
619 
620  memb_determine_message_transmit ();
621 }
622 
624 {
625  ENTER();
626  my_memb_determine_list_entries = 0;
627  memset (&my_memb_determine_ring_id, 0, sizeof (struct memb_ring_id));
628 }
Definition: sync.c:69
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:569
void(* sync_abort)(void)
Definition: sync.h:47
Totem Single Ring Protocol.
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.h:39
void(* sync_activate)(void)
Definition: sync.c:89
#define MESSAGE_REQ_SYNC_MEMB_DETERMINE
Definition: sync.c:65
int nodeid
Definition: sync.c:95
struct message_header header
Definition: totemsrp.c:60
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition: totempg.c:1115
#define log_printf(level, format, args...)
Definition: logsys.h:319
int service_id
Definition: sync.c:80
void(* sync_activate)(void)
Definition: sync.h:46
void schedwrk_destroy(hdb_handle_t handle)
Definition: schedwrk.c:154
Definition: sync.c:79
Definition: sync.c:70
const char * name
Definition: sync.h:48
sync_process_state
Definition: sync.c:67
sync_state
Definition: sync.c:73
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:81
const void * group
Definition: totempg.h:56
#define TOTEMPG_AGREED
Definition: totempg.h:60
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:70
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:587
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition: totempg.c:1213
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
typedef __attribute__
void(* sync_abort)(void)
Definition: sync.c:87
Definition: sync.c:94
enum sync_process_state state
Definition: sync.c:90
#define ENTER
Definition: logsys.h:320
#define MESSAGE_REQ_SYNC_SERVICE_BUILD
Definition: sync.c:64
#define PROCESSOR_COUNT_MAX
Definition: coroapi.h:96
qb_handle_t hdb_handle_t
Definition: hdb.h:52
The memb_ring_id struct.
Definition: coroapi.h:122
Definition: sync.c:68
void sync_memb_list_abort(void)
Definition: sync.c:623
#define MESSAGE_REQ_SYNC_BARRIER
Definition: sync.c:63
#define SERVICES_COUNT_MAX
Definition: coroapi.h:463
int received
Definition: sync.c:96
int(* my_sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks)
Definition: sync.c:167
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition: totempg.c:1163
char name[128]
Definition: sync.c:91
LOGSYS_DECLARE_SUBSYS("SYNC")
unsigned int nodeid
Definition: coroapi.h:75
struct memb_ring_id ring_id
Definition: totemsrp.c:64
void sync_memb_list_determine(const struct memb_ring_id *ring_id)
Definition: sync.c:614
int(* sync_process)(void)
Definition: sync.c:88
int schedwrk_create(hdb_handle_t *handle, int(schedwrk_fn)(const void *), const void *context)
Definition: schedwrk.c:138
int(* sync_process)(void)
Definition: sync.h:45